diff --git a/.gitignore b/.gitignore index e0bd1ccf..14bbc054 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ - .DS_Store gen.bat gen_ref.bat train.bat __pycache__ */**/__pycache__ -logs \ No newline at end of file +logs +.ipynb_checkpoints \ No newline at end of file diff --git a/dreambooth_colab_joepenna.ipynb b/dreambooth_colab_joepenna.ipynb index 6396450c..2623fb68 100644 --- a/dreambooth_colab_joepenna.ipynb +++ b/dreambooth_colab_joepenna.ipynb @@ -1,32 +1,17 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false + }, "outputs": [], "source": [ "#@title Load repo (if needed)\n", "!git clone https://github.com/JoePenna/Dreambooth-Stable-Diffusion\n", "%cd Dreambooth-Stable-Diffusion" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -65,31 +50,36 @@ { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false + }, "outputs": [], "source": [ "#@title # Required - Navigate back to the directory.\n", "%cd Dreambooth-Stable-Diffusion" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "6tjx0HcjesFo" + }, + "outputs": [], "source": [ "#@markdown Hugging Face Login\n", "from huggingface_hub import notebook_login\n", "\n", "notebook_login()" - ], - "metadata": { - "id": "6tjx0HcjesFo" - }, - "execution_count": 1, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "O15vMMhCevib" + }, + "outputs": [], "source": [ "#@markdown Download the 1.4 sd model\n", "from IPython.display import clear_output\n", @@ -106,15 +96,15 @@ "!mv {actual_locations_of_model_blob[-1]} model.ckpt\n", "clear_output()\n", "print(\"✅ model.ckpt successfully downloaded\")\n" - ], - "metadata": { - "id": "O15vMMhCevib" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "N96aedTtfBjO" + }, + "outputs": [], "source": [ "#@title # Download Regularization Images\n", "#@markdown We’ve created the following image sets\n", @@ -129,15 +119,15 @@ "\n", "!mkdir -p regularization_images/{dataset}\n", "!mv -v Stable-Diffusion-Regularization-Images-{dataset}/{dataset}/*.* regularization_images/{dataset}" - ], - "metadata": { - "id": "N96aedTtfBjO" - }, - "execution_count": 2, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "A7hmdOdOfGzs" + }, + "outputs": [], "source": [ "#@title # Training Images\n", "#@markdown ## Upload your training images\n", @@ -166,15 +156,15 @@ " print(\"❌ no training images found. Please upload images to training_images\")\n", "else:\n", " print(\"✅ \" + str(len(training_images_file_paths)) + \" training images found\")\n" - ], - "metadata": { - "id": "A7hmdOdOfGzs" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "m2o_fFFvfxHi" + }, + "outputs": [], "source": [ "#@title # Training\n", "\n", @@ -205,15 +195,15 @@ " --class_word \"{class_word}\" \\\n", " --token \"{token}\" \\\n", " --no-test" - ], - "metadata": { - "id": "m2o_fFFvfxHi" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ll_ZIFNUulKJ" + }, + "outputs": [], "source": [ "#@title # Copy and name the checkpoint file\n", "\n", @@ -229,27 +219,44 @@ "!mv \"{last_checkpoint_file}\" \"trained_models/{file_name}\"\n", "\n", "print(\"Download your trained model file from trained_models/\" + file_name + \" and use in your favorite Stable Diffusion repo!\")" - ], - "metadata": { - "id": "Ll_ZIFNUulKJ" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mkidEm4evn1J" + }, + "outputs": [], "source": [ "#@title Save model in google drive\n", "from google.colab import drive\n", "drive.mount('/content/drive')\n", "\n", "!cp trained_models/{file_name} /content/drive/MyDrive/{file_name}" - ], - "metadata": { - "id": "mkidEm4evn1J" - }, - "execution_count": null, - "outputs": [] + ] } - ] + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.9.0 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + }, + "vscode": { + "interpreter": { + "hash": "0f8486a44be3db540a44064ef4647f290d210aa4269e54f181de788486e3e134" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/dreambooth_runpod_joepenna.ipynb b/dreambooth_runpod_joepenna.ipynb index 4e369153..707d42e9 100644 --- a/dreambooth_runpod_joepenna.ipynb +++ b/dreambooth_runpod_joepenna.ipynb @@ -1,475 +1,552 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "aa2c1ada", - "metadata": { - "id": "aa2c1ada" - }, - "source": [ - "# Dreambooth\n", - "### Notebook implementation by Joe Penna (@MysteryGuitarM on Twitter) - Improvements by David Bielejeski\n", - "\n", - "### Instructions\n", - "- Sign up for RunPod here: https://runpod.io/?ref=n8yfwyum\n", - " - Note: That's my personal referral link. Please don't use it if we are mortal enemies.\n", - "\n", - "- Click *Deploy* on either `SECURE CLOUD` or `COMMUNITY CLOUD`\n", - "\n", - "- Follow the rest of the instructions in this video: https://www.youtube.com/watch?v=7m__xadX0z0#t=5m33.1s\n", - "\n", - "Latest information on:\n", - "https://github.com/JoePenna/Dreambooth-Stable-Diffusion" - ] + "cells": [ + { + "cell_type": "markdown", + "id": "aa2c1ada", + "metadata": { + "id": "aa2c1ada" + }, + "source": [ + "# Dreambooth\n", + "### Notebook implementation by Joe Penna (@MysteryGuitarM on Twitter) - Improvements by David Bielejeski\n", + "\n", + "### Instructions\n", + "- Sign up for RunPod here: https://runpod.io/?ref=n8yfwyum\n", + " - Note: That's my personal referral link. Please don't use it if we are mortal enemies.\n", + "\n", + "- Click *Deploy* on either `SECURE CLOUD` or `COMMUNITY CLOUD`\n", + "\n", + "- Follow the rest of the instructions in this video: https://www.youtube.com/watch?v=7m__xadX0z0#t=5m33.1s\n", + "\n", + "Latest information on:\n", + "https://github.com/JoePenna/Dreambooth-Stable-Diffusion" + ] + }, + { + "cell_type": "markdown", + "id": "7b971cc0", + "metadata": { + "id": "7b971cc0" + }, + "source": [ + "## Build Environment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2AsGA1xpNQnb", + "metadata": { + "id": "2AsGA1xpNQnb" + }, + "outputs": [], + "source": [ + "# If running on Vast.AI, copy the code in this cell into a new notebook. Run it, then launch the `dreambooth_runpod_joepenna.ipynb` notebook from the jupyter interface.\n", + "!git clone https://github.com/JoePenna/Dreambooth-Stable-Diffusion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e1bc458-091b-42f4-a125-c3f0df20f29d", + "metadata": { + "id": "9e1bc458-091b-42f4-a125-c3f0df20f29d", + "scrolled": true + }, + "outputs": [], + "source": [ + "# BUILD ENV\n", + "!pip install omegaconf\n", + "!pip install einops\n", + "!pip install pytorch-lightning==1.6.5\n", + "!pip install test-tube\n", + "!pip install transformers\n", + "!pip install kornia\n", + "!pip install -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers\n", + "!pip install -e git+https://github.com/openai/CLIP.git@main#egg=clip\n", + "!pip install setuptools==59.5.0\n", + "!pip install pillow==9.0.1\n", + "!pip install torchmetrics==0.6.0\n", + "!pip install -e .\n", + "!pip install protobuf==3.20.1\n", + "!pip install gdown\n", + "!pip install -qq diffusers[\"training\"]==0.3.0 transformers ftfy\n", + "!pip install -qq \"ipywidgets>=7,<8\"\n", + "!pip install huggingface_hub\n", + "!pip install ipywidgets==7.7.1\n", + "!pip install captionizer==1.0.1" + ] + }, + { + "cell_type": "markdown", + "id": "3478e5a5", + "metadata": {}, + "source": [ + "### Try downloading hugging face model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16d69879", + "metadata": {}, + "outputs": [], + "source": [ + "# Hugging Face Login - can try this\n", + "from huggingface_hub import notebook_login\n", + "\n", + "notebook_login()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b65bd52", + "metadata": {}, + "outputs": [], + "source": [ + "# Download the 1.4 sd model\n", + "from IPython.display import clear_output\n", + "\n", + "from huggingface_hub import hf_hub_download\n", + "downloaded_model_path = hf_hub_download(\n", + " repo_id=\"CompVis/stable-diffusion-v-1-4-original\",\n", + " filename=\"sd-v1-4.ckpt\",\n", + " use_auth_token=True\n", + ")\n", + "\n", + "# Move the sd-v1-4.ckpt to the root of this directory as \"model.ckpt\"\n", + "actual_locations_of_model_blob = !readlink -f {downloaded_model_path}\n", + "!mv {actual_locations_of_model_blob[-1]} model.ckpt\n", + "clear_output()\n", + "print(\"✅ model.ckpt successfully downloaded\")" + ] + }, + { + "cell_type": "markdown", + "id": "789324c3", + "metadata": {}, + "source": [ + "If that doesn't work, download it from gdrive:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb0e9d30", + "metadata": {}, + "outputs": [], + "source": [ + "# Get model from google drive\n", + "# Place drive ID at end of the line below\n", + "!gdown https://drive.google.com/uc?id=EXAMPLEID" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3ca4a81", + "metadata": {}, + "outputs": [], + "source": [ + "# Rename model file\n", + "import glob, os\n", + "os.rename(glob.glob(\"*.ckpt\")[0], \"model.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "id": "17d1d11a", + "metadata": { + "id": "17d1d11a" + }, + "source": [ + "# Regularization Images (Skip this section if you are uploading your own or using the provided images)" + ] + }, + { + "cell_type": "markdown", + "id": "ed07a5df", + "metadata": { + "id": "ed07a5df" + }, + "source": [ + "Training teaches your new model both your token **but** re-trains your class simultaneously.\n", + "\n", + "From cursory testing, it does not seem like reg images affect the model too much. However, they do affect your class greatly, which will in turn affect your generations.\n", + "\n", + "You can either generate your images here, or use the repos below to quickly download 1500 images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67f9ff0c-b529-4c7c-8e26-8388d70a5d91", + "metadata": { + "id": "67f9ff0c-b529-4c7c-8e26-8388d70a5d91" + }, + "outputs": [], + "source": [ + "# GENERATE 200 images - Optional\n", + "self_generated_files_prompt = \"person\" #@param {type:\"string\"}\n", + "self_generated_files_count = 200 #@param {type:\"integer\"}\n", + "\n", + "!python scripts/stable_txt2img.py \\\n", + " --seed 10 \\\n", + " --ddim_eta 0.0 \\\n", + " --n_samples 1 \\\n", + " --n_iter {self_generated_files_count} \\\n", + " --scale 10.0 \\\n", + " --ddim_steps 50 \\\n", + " --ckpt model.ckpt \\\n", + " --prompt {self_generated_files_prompt}\n", + "\n", + "dataset=self_generated_files_prompt\n", + "\n", + "!mkdir -p regularization_images/{dataset}\n", + "!mv outputs/txt2img-samples/*.png regularization_images/{dataset}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d1c7e1c", + "metadata": { + "id": "3d1c7e1c" + }, + "outputs": [], + "source": [ + "# Zip up the files for downloading and reuse.\n", + "# Download this file locally so you can reuse during another training on this dataset\n", + "!apt-get install -y zip\n", + "!zip -r regularization_images.zip regularization_images/{dataset}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "# Download pre-generated regularization images\n", + "We've created the following image sets\n", + "\n", + "`man_euler` - provided by Niko Pueringer (Corridor Digital) - euler @ 40 steps, CFG 7.5\n", + "`man_unsplash` - pictures from various photographers\n", + "`person_ddim`\n", + "`woman_ddim` - provided by David Bielejeski - ddim @ 50 steps, CFG 10.0\n", + "`person_ddim` is recommended" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e7EydXCjOV1v", + "metadata": { + "id": "e7EydXCjOV1v" + }, + "outputs": [], + "source": [ + "#Download Regularization Images\n", + "\n", + "dataset=\"person_ddim\" #@param [\"man_euler\", \"man_unsplash\", \"person_ddim\", \"woman_ddim\", \"blonde_woman\"]\n", + "!git clone https://github.com/djbielejeski/Stable-Diffusion-Regularization-Images-{dataset}.git\n", + "\n", + "!mkdir -p regularization_images/{dataset}\n", + "!mv -v Stable-Diffusion-Regularization-Images-{dataset}/{dataset}/*.* regularization_images/{dataset}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Upload your training images\n", + "Upload 10-20 images of someone to\n", + "\n", + "```\n", + "/workspace/Dreambooth-Stable-Diffusion/training_images\n", + "```\n", + "\n", + "WARNING: Be sure to upload an *even* amount of images, otherwise the training inexplicably stops at 1500 steps.\n", + "\n", + "* 2-3 full body\n", + "* 3-5 upper body\n", + "* 5-12 close-up on face\n", + "\n", + "The images should be as close as possible to the kind of images you're trying to make (most of the time, that means no selfies).\n", + "\n", + "Images can be uploaded directly to the workspace directory, or they can be provided from URLs.\n", + "\n", + "**To upload images from URLs, set, run the block below only.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b242f027", + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#@markdown Add here the URLs to the images of the subject you are adding\n", + "image_from_url = True\n", + "urls = [\n", + " \"https://i.imgur.com/test1.png\",\n", + " \"https://i.imgur.com/test2.png\",\n", + " \"https://i.imgur.com/test3.png\",\n", + " \"https://i.imgur.com/test4.png\",\n", + " \"https://i.imgur.com/test5.png\",\n", + " # You can add additional images here -- about 20-30 images in different\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "e9c8b1d6", + "metadata": {}, + "source": [ + "**If you want to upload images directly to the workspace, change `path_to_images_folder` to the folder path where the images are stored and run the below block only.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1706511c", + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "\n", + "image_from_url = False\n", + "path_to_images_folder = \"my_images_folder/\"\n", + "\n", + "imagePaths = list()\n", + "for path in glob.glob(path_to_images_folder + \"*\"):\n", + " imagePaths.append(path)" + ] + }, + { + "cell_type": "markdown", + "id": "2cc02cce", + "metadata": {}, + "source": [ + "Run the below cell to finalise image imports and check the images that were just added." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cfd0355", + "metadata": {}, + "outputs": [], + "source": [ + "#@title Download and check the images you have just added\n", + "import os\n", + "import requests\n", + "from io import BytesIO\n", + "from PIL import Image\n", + "\n", + "def image_grid(imgs, rows, cols):\n", + " assert len(imgs) == rows*cols\n", + "\n", + " w, h = imgs[0].size\n", + " grid = Image.new('RGB', size=(cols*w, rows*h))\n", + " grid_w, grid_h = grid.size\n", + "\n", + " for i, img in enumerate(imgs):\n", + " grid.paste(img, box=(i%cols*w, i//cols*h))\n", + " return grid\n", + "\n", + "def download_image(url):\n", + " try:\n", + " response = requests.get(url)\n", + " except:\n", + " return None\n", + " return Image.open(BytesIO(response.content)).convert(\"RGB\")\n", + "\n", + "def read_image(path):\n", + " return Image.open(path).convert('RGB')\n", + "\n", + "# Assemble list of Image objects\n", + "if image_from_url:\n", + " images = list(filter(None,[download_image(url) for url in urls]))\n", + "else:\n", + " images = list(filter(None,[read_image(path) for path in imagePaths]))\n", + "\n", + "save_path = \"./training_samples\"\n", + "if not os.path.exists(save_path):\n", + " os.mkdir(save_path)\n", + "[image.save(f\"{save_path}/{i}.png\", format=\"png\") for i, image in enumerate(images)]\n", + "image_grid(images, 1, len(images))" + ] + }, + { + "cell_type": "markdown", + "id": "ad4e50df", + "metadata": { + "id": "ad4e50df" + }, + "source": [ + "## Training\n", + "\n", + "If training a person or subject, keep an eye on your project's `logs/{folder}/images/train/samples_scaled_gs-00xxxx` generations.\n", + "\n", + "If training a style, keep an eye on your project's `logs/{folder}/images/train/samples_gs-00xxxx` generations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fa5dd66-2ca0-4819-907e-802e25583ae6", + "metadata": { + "id": "6fa5dd66-2ca0-4819-907e-802e25583ae6", + "tags": [] + }, + "outputs": [], + "source": [ + "# Training\n", + "\n", + "# This isn't used for training, just to help you remember what your trained into the model.\n", + "project_name = \"project_name\"\n", + "\n", + "# MAX STEPS\n", + "# How many steps do you want to train for?\n", + "max_training_steps = 2000\n", + "\n", + "# Match class_word to the category of the regularization images you chose above.\n", + "class_word = \"person\" # typical uses are \"man\", \"person\", \"woman\"\n", + "\n", + "# This is the unique token you are incorporating into the stable diffusion model.\n", + "token = \"firstNameLastName\"\n", + "\n", + "\n", + "reg_data_root = \"/workspace/Dreambooth-Stable-Diffusion/regularization_images/\" + dataset\n", + "\n", + "!rm -rf training_images/.ipynb_checkpoints\n", + "!python \"main.py\" \\\n", + " --base configs/stable-diffusion/v1-finetune_unfrozen.yaml \\\n", + " -t \\\n", + " --actual_resume \"model.ckpt\" \\\n", + " --reg_data_root \"{reg_data_root}\" \\\n", + " -n \"{project_name}\" \\\n", + " --gpus 0, \\\n", + " --data_root \"/workspace/Dreambooth-Stable-Diffusion/training_images\" \\\n", + " --max_training_steps {max_training_steps} \\\n", + " --class_word \"{class_word}\" \\\n", + " --token \"{token}\" \\\n", + " --no-test" + ] + }, + { + "cell_type": "markdown", + "id": "dc49d0bd", + "metadata": {}, + "source": [ + "## Copy and name the checkpoint file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd160680", + "metadata": {}, + "outputs": [], + "source": [ + "# Copy the checkpoint into our `trained_models` folder\n", + "\n", + "directory_paths = !ls -d logs/*\n", + "last_checkpoint_file = directory_paths[-1] + \"/checkpoints/last.ckpt\"\n", + "training_images = !find training_images/*\n", + "date_string = !date +\"%Y-%m-%dT%H-%M-%S\"\n", + "file_name = date_string[-1] + \"_\" + project_name + \"_\" + str(len(training_images)) + \"_training_images_\" + str(max_training_steps) + \"_max_training_steps_\" + token + \"_token_\" + class_word + \"_class_word.ckpt\"\n", + "\n", + "file_name = file_name.replace(\" \", \"_\")\n", + "\n", + "!mkdir -p trained_models\n", + "!mv \"{last_checkpoint_file}\" \"trained_models/{file_name}\"\n", + "\n", + "print(\"Download your trained model file from trained_models/\" + file_name + \" and use in your favorite Stable Diffusion repo!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "# Optional - Upload to google drive\n", + "* run the following commands in a new `terminal` in the `Dreambooth-Stable-Diffusion` directory\n", + "* `chmod +x ./gdrive`\n", + "* `./gdrive about`\n", + "* `paste your token here after navigating to the link`\n", + "* `./gdrive upload trained_models/{file_name.ckpt}`" + ] + }, + { + "cell_type": "markdown", + "id": "9a90ac5c", + "metadata": {}, + "source": [ + "# Big Important Note!\n", + "\n", + "The way to use your token is ` ` ie `joepenna person` and not just `joepenna`" + ] + }, + { + "cell_type": "markdown", + "id": "d28d0139", + "metadata": {}, + "source": [ + "## Generate Images With Your Trained Model!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80ddb03b", + "metadata": {}, + "outputs": [], + "source": [ + "!python scripts/stable_txt2img.py \\\n", + " --ddim_eta 0.0 \\\n", + " --n_samples 1 \\\n", + " --n_iter 4 \\\n", + " --scale 7.0 \\\n", + " --ddim_steps 50 \\\n", + " --ckpt \"/workspace/Dreambooth-Stable-Diffusion/trained_models/\" + {file_name} \\\n", + " --prompt \"joepenna person as a masterpiece portrait painting by John Singer Sargent in the style of Rembrandt\"" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.9.0 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.0" + }, + "vscode": { + "interpreter": { + "hash": "0f8486a44be3db540a44064ef4647f290d210aa4269e54f181de788486e3e134" + } + } }, - { - "cell_type": "markdown", - "id": "7b971cc0", - "metadata": { - "id": "7b971cc0" - }, - "source": [ - "## Build Environment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2AsGA1xpNQnb", - "metadata": { - "id": "2AsGA1xpNQnb" - }, - "outputs": [], - "source": [ - "# If running on Vast.AI, copy the code in this cell into a new notebook. Run it, then launch the `dreambooth_runpod_joepenna.ipynb` notebook from the jupyter interface.\n", - "!git clone https://github.com/JoePenna/Dreambooth-Stable-Diffusion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e1bc458-091b-42f4-a125-c3f0df20f29d", - "metadata": { - "id": "9e1bc458-091b-42f4-a125-c3f0df20f29d", - "scrolled": true - }, - "outputs": [], - "source": [ - "# BUILD ENV\n", - "!pip install omegaconf\n", - "!pip install einops\n", - "!pip install pytorch-lightning==1.6.5\n", - "!pip install test-tube\n", - "!pip install transformers\n", - "!pip install kornia\n", - "!pip install -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers\n", - "!pip install -e git+https://github.com/openai/CLIP.git@main#egg=clip\n", - "!pip install setuptools==59.5.0\n", - "!pip install pillow==9.0.1\n", - "!pip install torchmetrics==0.6.0\n", - "!pip install -e .\n", - "!pip install protobuf==3.20.1\n", - "!pip install gdown\n", - "!pip install -qq diffusers[\"training\"]==0.3.0 transformers ftfy\n", - "!pip install -qq \"ipywidgets>=7,<8\"\n", - "!pip install huggingface_hub\n", - "!pip install ipywidgets==7.7.1\n", - "!pip install captionizer==1.0.1\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dae11c10", - "metadata": { - "id": "dae11c10" - }, - "outputs": [], - "source": [ - "# Hugging Face Login\n", - "from huggingface_hub import notebook_login\n", - "\n", - "notebook_login()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Download the 1.4 sd model\n", - "from IPython.display import clear_output\n", - "\n", - "from huggingface_hub import hf_hub_download\n", - "downloaded_model_path = hf_hub_download(\n", - " repo_id=\"CompVis/stable-diffusion-v-1-4-original\",\n", - " filename=\"sd-v1-4.ckpt\",\n", - " use_auth_token=True\n", - ")\n", - "\n", - "# Move the sd-v1-4.ckpt to the root of this directory as \"model.ckpt\"\n", - "actual_locations_of_model_blob = !readlink -f {downloaded_model_path}\n", - "!mv {actual_locations_of_model_blob[-1]} model.ckpt\n", - "clear_output()\n", - "print(\"✅ model.ckpt successfully downloaded\")" - ] - }, - { - "cell_type": "markdown", - "id": "17d1d11a", - "metadata": { - "id": "17d1d11a" - }, - "source": [ - "# Regularization Images (Skip this section if you are uploading your own or using the provided images)" - ] - }, - { - "cell_type": "markdown", - "id": "ed07a5df", - "metadata": { - "id": "ed07a5df" - }, - "source": [ - "Training teaches your new model both your token **but** re-trains your class simultaneously.\n", - "\n", - "From cursory testing, it does not seem like reg images affect the model too much. However, they do affect your class greatly, which will in turn affect your generations.\n", - "\n", - "You can either generate your images here, or use the repos below to quickly download 1500 images." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "67f9ff0c-b529-4c7c-8e26-8388d70a5d91", - "metadata": { - "id": "67f9ff0c-b529-4c7c-8e26-8388d70a5d91" - }, - "outputs": [], - "source": [ - "# GENERATE 200 images - Optional\n", - "self_generated_files_prompt = \"person\" #@param {type:\"string\"}\n", - "self_generated_files_count = 200 #@param {type:\"integer\"}\n", - "\n", - "!python scripts/stable_txt2img.py \\\n", - " --seed 10 \\\n", - " --ddim_eta 0.0 \\\n", - " --n_samples 1 \\\n", - " --n_iter {self_generated_files_count} \\\n", - " --scale 10.0 \\\n", - " --ddim_steps 50 \\\n", - " --ckpt model.ckpt \\\n", - " --prompt {self_generated_files_prompt}\n", - "\n", - "dataset=self_generated_files_prompt\n", - "\n", - "!mkdir -p regularization_images/{dataset}\n", - "!mv outputs/txt2img-samples/*.png regularization_images/{dataset}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3d1c7e1c", - "metadata": { - "id": "3d1c7e1c" - }, - "outputs": [], - "source": [ - "# Zip up the files for downloading and reuse.\n", - "# Download this file locally so you can reuse during another training on this dataset\n", - "!apt-get install -y zip\n", - "!zip -r regularization_images.zip regularization_images/{dataset}" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Download pre-generated regularization images\n", - "We've created the following image sets\n", - "\n", - "`man_euler` - provided by Niko Pueringer (Corridor Digital) - euler @ 40 steps, CFG 7.5\n", - "`man_unsplash` - pictures from various photographers\n", - "`person_ddim`\n", - "`woman_ddim` - provided by David Bielejeski - ddim @ 50 steps, CFG 10.0\n", - "`person_ddim` is recommended" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "e7EydXCjOV1v", - "metadata": { - "id": "e7EydXCjOV1v" - }, - "outputs": [], - "source": [ - "#Download Regularization Images\n", - "\n", - "dataset=\"person_ddim\" #@param [\"man_euler\", \"man_unsplash\", \"person_ddim\", \"woman_ddim\", \"blonde_woman\"]\n", - "!git clone https://github.com/djbielejeski/Stable-Diffusion-Regularization-Images-{dataset}.git\n", - "\n", - "!mkdir -p regularization_images/{dataset}\n", - "!mv -v Stable-Diffusion-Regularization-Images-{dataset}/{dataset}/*.* regularization_images/{dataset}" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Upload your training images\n", - "Upload 10-20 images of someone to\n", - "\n", - "```\n", - "/workspace/Dreambooth-Stable-Diffusion/training_images\n", - "```\n", - "\n", - "WARNING: Be sure to upload an *even* amount of images, otherwise the training inexplicably stops at 1500 steps.\n", - "\n", - "* 2-3 full body\n", - "* 3-5 upper body\n", - "* 5-12 close-up on face\n", - "\n", - "The images should be:\n", - "\n", - "- as close as possible to the kind of images you're trying to make" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "#@markdown Add here the URLs to the images of the subject you are adding\n", - "urls = [\n", - " \"https://i.imgur.com/test1.png\",\n", - " \"https://i.imgur.com/test2.png\",\n", - " \"https://i.imgur.com/test3.png\",\n", - " \"https://i.imgur.com/test4.png\",\n", - " \"https://i.imgur.com/test5.png\",\n", - " # You can add additional images here -- about 20-30 images in different\n", - "]" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "#@title Download and check the images you have just added\n", - "import os\n", - "import requests\n", - "from io import BytesIO\n", - "from PIL import Image\n", - "\n", - "\n", - "def image_grid(imgs, rows, cols):\n", - " assert len(imgs) == rows*cols\n", - "\n", - " w, h = imgs[0].size\n", - " grid = Image.new('RGB', size=(cols*w, rows*h))\n", - " grid_w, grid_h = grid.size\n", - "\n", - " for i, img in enumerate(imgs):\n", - " grid.paste(img, box=(i%cols*w, i//cols*h))\n", - " return grid\n", - "\n", - "def download_image(url):\n", - " try:\n", - " response = requests.get(url)\n", - " except:\n", - " return None\n", - " return Image.open(BytesIO(response.content)).convert(\"RGB\")\n", - "\n", - "images = list(filter(None,[download_image(url) for url in urls]))\n", - "save_path = \"./training_images\"\n", - "if not os.path.exists(save_path):\n", - " os.mkdir(save_path)\n", - "[image.save(f\"{save_path}/{i}.png\", format=\"png\") for i, image in enumerate(images)]\n", - "image_grid(images, 1, len(images))" - ] - }, - { - "cell_type": "markdown", - "id": "ad4e50df", - "metadata": { - "id": "ad4e50df" - }, - "source": [ - "## Training\n", - "\n", - "If training a person or subject, keep an eye on your project's `logs/{folder}/images/train/samples_scaled_gs-00xxxx` generations.\n", - "\n", - "If training a style, keep an eye on your project's `logs/{folder}/images/train/samples_gs-00xxxx` generations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6fa5dd66-2ca0-4819-907e-802e25583ae6", - "metadata": { - "id": "6fa5dd66-2ca0-4819-907e-802e25583ae6", - "tags": [] - }, - "outputs": [], - "source": [ - "# Training\n", - "\n", - "# This isn't used for training, just to help you remember what your trained into the model.\n", - "project_name = \"project_name\"\n", - "\n", - "# MAX STEPS\n", - "# How many steps do you want to train for?\n", - "max_training_steps = 2000\n", - "\n", - "# Match class_word to the category of the regularization images you chose above.\n", - "class_word = \"person\" # typical uses are \"man\", \"person\", \"woman\"\n", - "\n", - "# This is the unique token you are incorporating into the stable diffusion model.\n", - "token = \"firstNameLastName\"\n", - "\n", - "\n", - "reg_data_root = \"/workspace/Dreambooth-Stable-Diffusion/regularization_images/\" + dataset\n", - "\n", - "!rm -rf training_images/.ipynb_checkpoints\n", - "!python \"main.py\" \\\n", - " --base configs/stable-diffusion/v1-finetune_unfrozen.yaml \\\n", - " -t \\\n", - " --actual_resume \"model.ckpt\" \\\n", - " --reg_data_root \"{reg_data_root}\" \\\n", - " -n \"{project_name}\" \\\n", - " --gpus 0, \\\n", - " --data_root \"/workspace/Dreambooth-Stable-Diffusion/training_images\" \\\n", - " --max_training_steps {max_training_steps} \\\n", - " --class_word \"{class_word}\" \\\n", - " --token \"{token}\" \\\n", - " --no-test" - ] - }, - { - "cell_type": "markdown", - "id": "dc49d0bd", - "metadata": {}, - "source": [ - "## Copy and name the checkpoint file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Copy the checkpoint into our `trained_models` folder\n", - "\n", - "directory_paths = !ls -d logs/*\n", - "last_checkpoint_file = directory_paths[-1] + \"/checkpoints/last.ckpt\"\n", - "training_images = !find training_images/*\n", - "date_string = !date +\"%Y-%m-%dT%H-%M-%S\"\n", - "file_name = date_string[-1] + \"_\" + project_name + \"_\" + str(len(training_images)) + \"_training_images_\" + str(max_training_steps) + \"_max_training_steps_\" + token + \"_token_\" + class_word + \"_class_word.ckpt\"\n", - "\n", - "file_name = file_name.replace(\" \", \"_\")\n", - "\n", - "!mkdir -p trained_models\n", - "!mv \"{last_checkpoint_file}\" \"trained_models/{file_name}\"\n", - "\n", - "print(\"Download your trained model file from trained_models/\" + file_name + \" and use in your favorite Stable Diffusion repo!\")" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Optional - Upload to google drive\n", - "* run the following commands in a new `terminal` in the `Dreambooth-Stable-Diffusion` directory\n", - "* `chmod +x ./gdrive`\n", - "* `./gdrive about`\n", - "* `paste your token here after navigating to the link`\n", - "* `./gdrive upload trained_models/{file_name.ckpt}`" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "markdown", - "id": "9a90ac5c", - "metadata": {}, - "source": [ - "# Big Important Note!\n", - "\n", - "The way to use your token is ` ` ie `joepenna person` and not just `joepenna`" - ] - }, - { - "cell_type": "markdown", - "id": "d28d0139", - "metadata": {}, - "source": [ - "## Generate Images With Your Trained Model!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "80ddb03b", - "metadata": {}, - "outputs": [], - "source": [ - "!python scripts/stable_txt2img.py \\\n", - " --ddim_eta 0.0 \\\n", - " --n_samples 1 \\\n", - " --n_iter 4 \\\n", - " --scale 7.0 \\\n", - " --ddim_steps 50 \\\n", - " --ckpt \"/workspace/Dreambooth-Stable-Diffusion/trained_models/{file_name}\" \\\n", - " --prompt \"joepenna person as a masterpiece portrait painting by John Singer Sargent in the style of Rembrandt\"" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - }, - "vscode": { - "interpreter": { - "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/scripts/stable_txt2img.py b/scripts/stable_txt2img.py index 93642b4c..1fffb244 100644 --- a/scripts/stable_txt2img.py +++ b/scripts/stable_txt2img.py @@ -1,4 +1,4 @@ -import argparse, os, sys, glob +import argparse, os, sys, glob, uuid import torch import numpy as np from omegaconf import OmegaConf @@ -50,23 +50,23 @@ def main(): type=str, nargs="?", default="a painting of a virus monster playing guitar", - help="the prompt to render" + help="the prompt to render", ) parser.add_argument( "--outdir", type=str, nargs="?", help="dir to write results to", - default="outputs/txt2img-samples" + default="outputs/txt2img-samples", ) parser.add_argument( "--skip_grid", - action='store_true', + action="store_true", help="do not save a grid, only individual samples. Helpful when evaluating lots of samples", ) parser.add_argument( "--skip_save", - action='store_true', + action="store_true", help="do not save individual samples. For speed measurements.", ) parser.add_argument( @@ -77,17 +77,17 @@ def main(): ) parser.add_argument( "--plms", - action='store_true', + action="store_true", help="use plms sampling", ) parser.add_argument( "--laion400m", - action='store_true', + action="store_true", help="uses the LAION400M model", ) parser.add_argument( "--fixed_code", - action='store_true', + action="store_true", help="if enabled, uses the same starting code across samples ", ) parser.add_argument( @@ -160,7 +160,7 @@ def main(): type=str, default="models/ldm/stable-diffusion-v1/model.ckpt", help="path to checkpoint of model", - ) + ) parser.add_argument( "--seed", type=int, @@ -172,14 +172,14 @@ def main(): type=str, help="evaluate at this precision", choices=["full", "autocast"], - default="autocast" + default="autocast", ) - parser.add_argument( - "--embedding_path", - type=str, - help="Path to a pre-trained embedding manager checkpoint") + "--embedding_path", + type=str, + help="Path to a pre-trained embedding manager checkpoint", + ) opt = parser.parse_args() @@ -193,7 +193,7 @@ def main(): config = OmegaConf.load(f"{opt.config}") model = load_model_from_config(config, f"{opt.ckpt}") - #model.embedding_manager.load(opt.embedding_path) + # model.embedding_manager.load(opt.embedding_path) device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") model = model.to(device) @@ -226,9 +226,11 @@ def main(): start_code = None if opt.fixed_code: - start_code = torch.randn([opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f], device=device) + start_code = torch.randn( + [opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f], device=device + ) - precision_scope = autocast if opt.precision=="autocast" else nullcontext + precision_scope = autocast if opt.precision == "autocast" else nullcontext with torch.no_grad(): with precision_scope("cuda"): with model.ema_scope(): @@ -243,24 +245,31 @@ def main(): prompts = list(prompts) c = model.get_learned_conditioning(prompts) shape = [opt.C, opt.H // opt.f, opt.W // opt.f] - samples_ddim, _ = sampler.sample(S=opt.ddim_steps, - conditioning=c, - batch_size=opt.n_samples, - shape=shape, - verbose=False, - unconditional_guidance_scale=opt.scale, - unconditional_conditioning=uc, - eta=opt.ddim_eta, - x_T=start_code) + samples_ddim, _ = sampler.sample( + S=opt.ddim_steps, + conditioning=c, + batch_size=opt.n_samples, + shape=shape, + verbose=False, + unconditional_guidance_scale=opt.scale, + unconditional_conditioning=uc, + eta=opt.ddim_eta, + x_T=start_code, + ) x_samples_ddim = model.decode_first_stage(samples_ddim) - x_samples_ddim = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0) + x_samples_ddim = torch.clamp( + (x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0 + ) if not opt.skip_save: for x_sample in x_samples_ddim: - x_sample = 255. * rearrange(x_sample.cpu().numpy(), 'c h w -> h w c') + x_sample = 255.0 * rearrange( + x_sample.cpu().numpy(), "c h w -> h w c" + ) Image.fromarray(x_sample.astype(np.uint8)).save( - os.path.join(sample_path, f"{base_count:05}.jpg")) + os.path.join(sample_path, f"{base_count:05}.jpg") + ) base_count += 1 if not opt.skip_grid: @@ -269,23 +278,29 @@ def main(): if not opt.skip_grid: # additionally, save as grid grid = torch.stack(all_samples, 0) - grid = rearrange(grid, 'n b c h w -> (n b) c h w') - + grid = rearrange(grid, "n b c h w -> (n b) c h w") + + batch_uuid = uuid.uuid4() for i in range(grid.size(0)): - save_image(grid[i, :, :, :], os.path.join(outpath,opt.prompt[:180]+'_{}.png'.format(i))) + file_name = f"{batch_uuid.hex[:10]}_{opt.prompt[:30]}_{i}.png" + save_image(grid[i, :, :, :], os.path.join(outpath, file_name)) grid = make_grid(grid, nrow=n_rows) # to image - grid = 255. * rearrange(grid, 'c h w -> h w c').cpu().numpy() - Image.fromarray(grid.astype(np.uint8)).save(os.path.join(outpath, f'{prompt[:180].replace(" ", "-")}-{grid_count:04}.jpg')) + grid = 255.0 * rearrange(grid, "c h w -> h w c").cpu().numpy() + Image.fromarray(grid.astype(np.uint8)).save( + os.path.join( + outpath, + f'{batch_uuid.hex[:10]}-{prompt.replace(" ", "-")[:30]}-{grid_count:04}.jpg', + ) + ) grid_count += 1 - - toc = time.time() - print(f"Your samples are ready and waiting for you here: \n{outpath} \n" - f" \nEnjoy.") + print( + f"Your samples are ready and waiting for you here: \n{outpath} \n" f" \nEnjoy." + ) if __name__ == "__main__":