Add files

.gitignore

@@ -0,0 +1 @@
+gradio_cached_examples

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "multires_textual_inversion"]
+	path = multires_textual_inversion
+	url = https://github.com/giannisdaras/multires_textual_inversion

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+exclude: patch
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.2.0
+  hooks:
+  - id: check-executables-have-shebangs
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-toml
+  - id: check-yaml
+  - id: double-quote-string-fixer
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: ['--fix=lf']
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/myint/docformatter
+  rev: v1.4
+  hooks:
+  - id: docformatter
+    args: ['--in-place']
+- repo: https://github.com/pycqa/isort
+  rev: 5.10.1
+  hooks:
+    - id: isort
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.812
+  hooks:
+    - id: mypy
+      args: ['--ignore-missing-imports']
+- repo: https://github.com/google/yapf
+  rev: v0.32.0
+  hooks:
+  - id: yapf
+    args: ['--parallel', '--in-place']

.style.yapf

@@ -0,0 +1,5 @@
+[style]
+based_on_style = pep8
+blank_line_before_nested_class_or_def = false
+spaces_before_comment = 2
+split_before_logical_operator = true

app.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import os
+
+import gradio as gr
+
+from model import Model
+
+TITLE = '# Multiresolution Textual Inversion'
+DESCRIPTION = 'An unofficial demo for [https://github.com/giannisdaras/multires_textual_inversion](https://github.com/giannisdaras/multires_textual_inversion).'
+
+DETAILS = '''
+- To run the Semi Resolution-Dependent sampler, use the format: `<jane(number)>`.
+- To run the Fully Resolution-Dependent sampler, use the format: `<jane[number]>`.
+- To run the Fixed Resolution sampler, use the format: `<jane|number|>`.
+
+For this demo, only `<jane>`, `<gta5-artwork>` and `<cat-toy>` are available.
+Also, `number` should be an integer in [0, 9].
+'''
+FOOTER = '<img id="visitor-badge" src="https://visitor-badge.glitch.me/badge?page_id=hysts.multires-textual-inversion" alt="visitor badge" />'
+
+CACHE_EXAMPLES = os.getenv('SYSTEM') == 'spaces'
+
+
+def main():
+    model = Model()
+
+    with gr.Blocks(css='style.css') as demo:
+        gr.Markdown(TITLE)
+        gr.Markdown(DESCRIPTION)
+
+        with gr.Row():
+            with gr.Group():
+                with gr.Row():
+                    prompt = gr.Textbox(label='Prompt')
+                with gr.Row():
+                    num_images = gr.Slider(1,
+                                           9,
+                                           value=1,
+                                           step=1,
+                                           label='Number of images')
+                with gr.Row():
+                    num_steps = gr.Slider(1,
+                                          50,
+                                          value=10,
+                                          step=1,
+                                          label='Number of inference steps')
+                with gr.Row():
+                    seed = gr.Slider(0,
+                                     100000,
+                                     value=100,
+                                     step=1,
+                                     label='Seed')
+                with gr.Row():
+                    run_button = gr.Button('Run')
+
+            with gr.Column():
+                result = gr.Gallery(label='Result')
+
+        with gr.Row():
+            with gr.Group():
+                fn = lambda x: model.run(x, 2, 10, 100)
+                with gr.Row():
+                    gr.Examples(
+                        label='Examples 1',
+                        examples=[
+                            ['an image of <gta5-artwork(0)>'],
+                            ['an image of <jane(0)>'],
+                            ['an image of <jane(3)>'],
+                            ['an image of <cat-toy(0)>'],
+                        ],
+                        inputs=[prompt],
+                        outputs=[result],
+                        fn=fn,
+                        cache_examples=CACHE_EXAMPLES,
+                    )
+                with gr.Row():
+                    gr.Examples(
+                        label='Examples 2',
+                        examples=[
+                            [
+                                'an image of a cat in the style of <gta5-artwork(0)>'
+                            ],
+                            ['a painting of a dog in the style of <jane(0)>'],
+                            ['a painting of a dog in the style of <jane(5)>'],
+                            [
+                                'a painting of a <cat-toy(0)> in the style of <jane(3)>'
+                            ],
+                        ],
+                        inputs=[prompt],
+                        outputs=[result],
+                        fn=fn,
+                        cache_examples=CACHE_EXAMPLES,
+                    )
+                with gr.Row():
+                    gr.Examples(
+                        label='Examples 3',
+                        examples=[
+                            ['an image of <jane[0]>'],
+                            ['an image of <jane|0|>'],
+                            ['an image of <jane|3|>'],
+                        ],
+                        inputs=[prompt],
+                        outputs=[result],
+                        fn=fn,
+                        cache_examples=CACHE_EXAMPLES,
+                    )
+
+            prompt.submit(
+                fn=model.run,
+                inputs=[prompt, num_images, num_steps, seed],
+                outputs=[result],
+            )
+            run_button.click(
+                fn=model.run,
+                inputs=[prompt, num_images, num_steps, seed],
+                outputs=[result],
+            )
+
+        with gr.Accordion('About available prompts', open=False):
+            gr.Markdown(DETAILS)
+        gr.Markdown(FOOTER)
+    demo.launch(enable_queue=True, share=False)
+
+
+if __name__ == '__main__':
+    main()

model.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+import PIL.Image
+import torch
+from diffusers import DPMSolverMultistepScheduler
+
+if os.getenv('SYSTEM') == 'spaces':
+    with open('patch') as f:
+        subprocess.run('patch -p1'.split(),
+                       cwd='multires_textual_inversion',
+                       stdin=f)
+
+sys.path.insert(0, 'multires_textual_inversion')
+
+from pipeline import MultiResPipeline, load_learned_concepts
+
+HF_TOKEN = os.environ.get('HF_TOKEN')
+
+
+class Model:
+    def __init__(self):
+        self.device = torch.device(
+            'cuda:0' if torch.cuda.is_available() else 'cpu')
+        model_id = 'runwayml/stable-diffusion-v1-5'
+        if self.device.type == 'cpu':
+            pipe = MultiResPipeline.from_pretrained(model_id,
+                                                    use_auth_token=HF_TOKEN)
+        else:
+            pipe = MultiResPipeline.from_pretrained(model_id,
+                                                    torch_dtype=torch.float16,
+                                                    revision='fp16',
+                                                    use_auth_token=HF_TOKEN)
+        self.pipe = pipe.to(self.device)
+        self.pipe.scheduler = DPMSolverMultistepScheduler(
+            beta_start=0.00085,
+            beta_end=0.012,
+            beta_schedule='scaled_linear',
+            num_train_timesteps=1000,
+            trained_betas=None,
+            predict_epsilon=True,
+            thresholding=False,
+            algorithm_type='dpmsolver++',
+            solver_type='midpoint',
+            lower_order_final=True,
+        )
+        self.string_to_param_dict = load_learned_concepts(
+            self.pipe, 'textual_inversion_outputs/')
+
+    def run(self, prompt: str, n_images: int, n_steps: int,
+            seed: int) -> list[PIL.Image.Image]:
+        generator = torch.Generator(device=self.device).manual_seed(seed)
+        return self.pipe([prompt] * n_images,
+                         self.string_to_param_dict,
+                         num_inference_steps=n_steps,
+                         generator=generator)

multires_textual_inversion

@@ -0,0 +1 @@
+Subproject commit ebe79d70929f9f4fabde9d038d1e948a05b3027f

patch

@@ -0,0 +1,41 @@
+diff --git a/pipeline.py b/pipeline.py
+index 7c41e04..842c5b4 100644
+--- a/pipeline.py
++++ b/pipeline.py
+@@ -27,7 +27,7 @@ def load_learned_concepts(pipe, root_folder="selected_outputs/", num_scales=10):
+     for exp_name in os.listdir(root_folder):
+         # get everything up to the first numeric
+         pure_names.append(exp_name)
+-        encoder = torch.load(os.path.join(root_folder, exp_name, "text_encoder/pytorch_model.bin"))
++        encoder = torch.load(os.path.join(root_folder, exp_name, "text_encoder/pytorch_model.bin"), map_location=pipe.device)
+         embeddings = encoder["text_model.embeddings.token_embedding.weight"]
+         param_value = embeddings[-10:]
+ 
+@@ -36,23 +36,23 @@ def load_learned_concepts(pipe, root_folder="selected_outputs/", num_scales=10):
+             string_name = f"<{exp_name}|{t}|>"
+             tokens_to_add.append(string_name)
+             string_to_param_dict[string_name] = torch.nn.Parameter(param_value[t].unsqueeze(0).repeat([num_scales, 1]))
+-            
++
+             # Fully Resolution: use appropriate time embedding for the whole generation time.
+             string_name = f"<{exp_name}[{t}]>"
+             tokens_to_add.append(string_name)
+             repeats = t + 1
+             rep_param = param_value[t].unsqueeze(0).repeat([repeats, 1])
+             left = param_value[rep_param.shape[0]:]
+-            new_param = torch.cat([rep_param, left]) 
++            new_param = torch.cat([rep_param, left])
+             string_to_param_dict[string_name] = torch.nn.Parameter(new_param)
+ 
+             # Semi Resolution: use appropriate time embedding up to a certain time and then no conditioning.
+             string_name = f"<{exp_name}({t})>"
+             tokens_to_add.append(string_name)
+-            null_embedding = torch.zeros((param_value.shape[1],), device=param_value.device, dtype=param_value.dtype)
++            null_embedding = torch.zeros((param_value.shape[1],), device=pipe.device, dtype=param_value.dtype)
+             rep_param = null_embedding.unsqueeze(0).repeat([t + 1, 1])
+             left = param_value[rep_param.shape[0]:]
+-            new_param = torch.cat([rep_param, left]) 
++            new_param = torch.cat([rep_param, left])
+             string_to_param_dict[string_name] = torch.nn.Parameter(new_param)
+ 
+     pipe.tokenizer.add_tokens(tokens_to_add)

requirements.txt

@@ -0,0 +1,6 @@
+accelerate==0.12.0
+diffusers==0.9.0
+ftfy==6.1.1
+Pillow==9.2.0
+torch==1.12.1
+transformers==4.22.1

style.css

@@ -0,0 +1,7 @@
+h1 {
+  text-align: center;
+}
+img#visitor-badge {
+  display: block;
+  margin: auto;
+}

textual_inversion_outputs/cat-toy/text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float32",
+  "transformers_version": "4.24.0",
+  "vocab_size": 49418
+}

textual_inversion_outputs/cat-toy/text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98b628c646c93aeb917cfaba0d1af0dbf2a7348cc614acce5f3fddda597cbd2e
+size 492338807

textual_inversion_outputs/gta5-artwork/text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float32",
+  "transformers_version": "4.24.0",
+  "vocab_size": 49418
+}

textual_inversion_outputs/gta5-artwork/text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4812193a5df98edf62d3df13460f886572a71ab8585545520c44b92a7b8e1bd2
+size 492338807

textual_inversion_outputs/jane/text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.0",
+  "vocab_size": 49418
+}

textual_inversion_outputs/jane/text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8acb7b5f0e2b8c10a349cddec2ef577e34e91e74f9107683465508b8aab511da
+size 492338807