skyadmin/cog-webui-sd

1from collections import namedtuple2 3import numpy as np4from tqdm import trange5 6import modules.scripts as scripts7import gradio as gr8 9from modules import processing, shared, sd_samplers, prompt_parser, sd_samplers_common10from modules.processing import Processed11from modules.shared import opts, cmd_opts, state12 13import torch14import k_diffusion as K15 16from PIL import Image17from torch import autocast18from einops import rearrange, repeat19 20 21def find_noise_for_image(p, cond, uncond, cfg_scale, steps):22    x = p.init_latent23 24    s_in = x.new_ones([x.shape[0]])25    dnw = K.external.CompVisDenoiser(shared.sd_model)26    sigmas = dnw.get_sigmas(steps).flip(0)27 28    shared.state.sampling_steps = steps29 30    for i in trange(1, len(sigmas)):31        shared.state.sampling_step += 132 33        x_in = torch.cat([x] * 2)34        sigma_in = torch.cat([sigmas[i] * s_in] * 2)35        cond_in = torch.cat([uncond, cond])36 37        image_conditioning = torch.cat([p.image_conditioning] * 2)38        cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]}39 40        c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)]41        t = dnw.sigma_to_t(sigma_in)42 43        eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in)44        denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2)45 46        denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale47 48        d = (x - denoised) / sigmas[i]49        dt = sigmas[i] - sigmas[i - 1]50 51        x = x + d * dt52 53        sd_samplers_common.store_latent(x)54 55        # This shouldn't be necessary, but solved some VRAM issues56        del x_in, sigma_in, cond_in, c_out, c_in, t,57        del eps, denoised_uncond, denoised_cond, denoised, d, dt58 59    shared.state.nextjob()60 61    return x / x.std()62 63 64Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment"])65 66 67# Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/73668def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps):69    x = p.init_latent70 71    s_in = x.new_ones([x.shape[0]])72    dnw = K.external.CompVisDenoiser(shared.sd_model)73    sigmas = dnw.get_sigmas(steps).flip(0)74 75    shared.state.sampling_steps = steps76 77    for i in trange(1, len(sigmas)):78        shared.state.sampling_step += 179 80        x_in = torch.cat([x] * 2)81        sigma_in = torch.cat([sigmas[i - 1] * s_in] * 2)82        cond_in = torch.cat([uncond, cond])83 84        image_conditioning = torch.cat([p.image_conditioning] * 2)85        cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]}86 87        c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)]88 89        if i == 1:90            t = dnw.sigma_to_t(torch.cat([sigmas[i] * s_in] * 2))91        else:92            t = dnw.sigma_to_t(sigma_in)93 94        eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in)95        denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2)96 97        denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale98 99        if i == 1:100            d = (x - denoised) / (2 * sigmas[i])101        else:102            d = (x - denoised) / sigmas[i - 1]103 104        dt = sigmas[i] - sigmas[i - 1]105        x = x + d * dt106 107        sd_samplers_common.store_latent(x)108 109        # This shouldn't be necessary, but solved some VRAM issues110        del x_in, sigma_in, cond_in, c_out, c_in, t,111        del eps, denoised_uncond, denoised_cond, denoised, d, dt112 113    shared.state.nextjob()114 115    return x / sigmas[-1]116 117 118class Script(scripts.Script):119    def __init__(self):120        self.cache = None121 122    def title(self):123        return "img2img alternative test"124 125    def show(self, is_img2img):126        return is_img2img127 128    def ui(self, is_img2img):     129        info = gr.Markdown('''130        * `CFG Scale` should be 2 or lower.131        ''')132 133        override_sampler = gr.Checkbox(label="Override `Sampling method` to Euler?(this method is built for it)", value=True, elem_id=self.elem_id("override_sampler"))134 135        override_prompt = gr.Checkbox(label="Override `prompt` to the same value as `original prompt`?(and `negative prompt`)", value=True, elem_id=self.elem_id("override_prompt"))136        original_prompt = gr.Textbox(label="Original prompt", lines=1, elem_id=self.elem_id("original_prompt"))137        original_negative_prompt = gr.Textbox(label="Original negative prompt", lines=1, elem_id=self.elem_id("original_negative_prompt"))138 139        override_steps = gr.Checkbox(label="Override `Sampling Steps` to the same value as `Decode steps`?", value=True, elem_id=self.elem_id("override_steps"))140        st = gr.Slider(label="Decode steps", minimum=1, maximum=150, step=1, value=50, elem_id=self.elem_id("st"))141 142        override_strength = gr.Checkbox(label="Override `Denoising strength` to 1?", value=True, elem_id=self.elem_id("override_strength"))143 144        cfg = gr.Slider(label="Decode CFG scale", minimum=0.0, maximum=15.0, step=0.1, value=1.0, elem_id=self.elem_id("cfg"))145        randomness = gr.Slider(label="Randomness", minimum=0.0, maximum=1.0, step=0.01, value=0.0, elem_id=self.elem_id("randomness"))146        sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=False, elem_id=self.elem_id("sigma_adjustment"))147 148        return [149            info, 150            override_sampler,151            override_prompt, original_prompt, original_negative_prompt, 152            override_steps, st,153            override_strength,154            cfg, randomness, sigma_adjustment,155        ]156 157    def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment):158        # Override159        if override_sampler:160            p.sampler_name = "Euler"161        if override_prompt:162            p.prompt = original_prompt163            p.negative_prompt = original_negative_prompt164        if override_steps:165            p.steps = st166        if override_strength:167            p.denoising_strength = 1.0168 169        def sample_extra(conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts):170            lat = (p.init_latent.cpu().numpy() * 10).astype(int)171 172            same_params = self.cache is not None and self.cache.cfg_scale == cfg and self.cache.steps == st \173                                and self.cache.original_prompt == original_prompt \174                                and self.cache.original_negative_prompt == original_negative_prompt \175                                and self.cache.sigma_adjustment == sigma_adjustment176            same_everything = same_params and self.cache.latent.shape == lat.shape and np.abs(self.cache.latent-lat).sum() < 100177 178            if same_everything:179                rec_noise = self.cache.noise180            else:181                shared.state.job_count += 1182                cond = p.sd_model.get_learned_conditioning(p.batch_size * [original_prompt])183                uncond = p.sd_model.get_learned_conditioning(p.batch_size * [original_negative_prompt])184                if sigma_adjustment:185                    rec_noise = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st)186                else:187                    rec_noise = find_noise_for_image(p, cond, uncond, cfg, st)188                self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment)189 190            rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p)191            192            combined_noise = ((1 - randomness) * rec_noise + randomness * rand_noise) / ((randomness**2 + (1-randomness)**2) ** 0.5)193            194            sampler = sd_samplers.create_sampler(p.sampler_name, p.sd_model)195 196            sigmas = sampler.model_wrap.get_sigmas(p.steps)197            198            noise_dt = combined_noise - (p.init_latent / sigmas[0])199            200            p.seed = p.seed + 1201            202            return sampler.sample_img2img(p, p.init_latent, noise_dt, conditioning, unconditional_conditioning, image_conditioning=p.image_conditioning)203 204        p.sample = sample_extra205 206        p.extra_generation_params["Decode prompt"] = original_prompt207        p.extra_generation_params["Decode negative prompt"] = original_negative_prompt208        p.extra_generation_params["Decode CFG scale"] = cfg209        p.extra_generation_params["Decode steps"] = st210        p.extra_generation_params["Randomness"] = randomness211        p.extra_generation_params["Sigma Adjustment"] = sigma_adjustment212 213        processed = processing.process_images(p)214 215        return processed216 217