ysharma/style-aligned-controlnet

1{2 "cells": [3  {4   "cell_type": "markdown",5   "id": "50fa980f-1bae-40c1-a1f3-f5f89bef60d3",6   "metadata": {7    "pycharm": {8     "name": "#%% md\n"9    }10   },11   "source": [12    "## Copyright 2023 Google LLC"13   ]14  },15  {16   "cell_type": "code",17   "execution_count": null,18   "id": "5da5f038-057f-4475-a783-95660f98238c",19   "metadata": {20    "pycharm": {21     "name": "#%%\n"22    }23   },24   "outputs": [],25   "source": [26    "# Copyright 2023 Google LLC\n",27    "#\n",28    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",29    "# you may not use this file except in compliance with the License.\n",30    "# You may obtain a copy of the License at\n",31    "#\n",32    "#      http://www.apache.org/licenses/LICENSE-2.0\n",33    "#\n",34    "# Unless required by applicable law or agreed to in writing, software\n",35    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",36    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",37    "# See the License for the specific language governing permissions and\n",38    "# limitations under the License."39   ]40  },41  {42   "cell_type": "markdown",43   "id": "c3a7c069-c441-4204-a905-59cbd9edc13a",44   "metadata": {45    "pycharm": {46     "name": "#%% md\n"47    }48   },49   "source": [50    "# MultiDiffusion with StyleAligned over SD v2"51   ]52  },53  {54   "cell_type": "code",55   "execution_count": null,56   "id": "14178de7-d4c8-4881-ac1d-ff84bae57c6f",57   "metadata": {58    "pycharm": {59     "name": "#%%\n"60    }61   },62   "outputs": [],63   "source": [64    "import torch\n",65    "from diffusers import StableDiffusionPanoramaPipeline, DDIMScheduler\n",66    "import mediapy\n",67    "import sa_handler\n",68    "import pipeline_calls"69   ]70  },71  {72   "cell_type": "code",73   "execution_count": null,74   "id": "738cee0e-4d6e-4875-b4df-eadff6e27e7f",75   "metadata": {76    "pycharm": {77     "name": "#%%\n"78    }79   },80   "outputs": [],81   "source": [82    "# init models\n",83    "model_ckpt = \"stabilityai/stable-diffusion-2-base\"\n",84    "scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder=\"scheduler\")\n",85    "pipeline = StableDiffusionPanoramaPipeline.from_pretrained(\n",86    "     model_ckpt, scheduler=scheduler, torch_dtype=torch.float16\n",87    ").to(\"cuda\")\n",88    "\n",89    "sa_args = sa_handler.StyleAlignedArgs(share_group_norm=True,\n",90    "                                      share_layer_norm=True,\n",91    "                                      share_attention=True,\n",92    "                                      adain_queries=True,\n",93    "                                      adain_keys=True,\n",94    "                                      adain_values=False,\n",95    "                                     )\n",96    "handler = sa_handler.Handler(pipeline)\n",97    "handler.register(sa_args)"98   ]99  },100  {101   "cell_type": "code",102   "execution_count": null,103   "id": "ea61e789-2814-4820-8ae7-234c3c6640a0",104   "metadata": {105    "pycharm": {106     "name": "#%%\n"107    }108   },109   "outputs": [],110   "source": [111    "# run MultiDiffusion with StyleAligned\n",112    "\n",113    "reference_prompt = \"a beautiful papercut art design\"\n",114    "target_prompts = [\"mountains in a beautiful papercut art design\", \"giraffes in a beautiful papercut art design\"]\n",115    "view_batch_size = 25  # adjust according to VRAM size\n",116    "reference_latent = torch.randn(1, 4, 64, 64,)\n",117    "for target_prompt in target_prompts:\n",118    "    images = pipeline_calls.panorama_call(pipeline, [reference_prompt, target_prompt], reference_latent=reference_latent, view_batch_size=view_batch_size)\n",119    "    mediapy.show_images(images, titles=[\"reference\", \"result\"])"120   ]121  },122  {123   "cell_type": "code",124   "execution_count": null,125   "id": "791a9b28-f0ce-4fd0-9f3c-594281c2ae56",126   "metadata": {127    "pycharm": {128     "name": "#%%\n"129    }130   },131   "outputs": [],132   "source": []133  }134 ],135 "metadata": {136  "kernelspec": {137   "display_name": "Python 3 (ipykernel)",138   "language": "python",139   "name": "python3"140  },141  "language_info": {142   "codemirror_mode": {143    "name": "ipython",144    "version": 3145   },146   "file_extension": ".py",147   "mimetype": "text/x-python",148   "name": "python",149   "nbconvert_exporter": "python",150   "pygments_lexer": "ipython3",151   "version": "3.11.5"152  }153 },154 "nbformat": 4,155 "nbformat_minor": 5156}