fix bugs, add 'kd-multi-view' extension

README.md

@@ -29,7 +29,6 @@
 🕒 Advanced prompt syntax <br>
 🕒 Prompt weighting <br>
 🕒 Checkpoint conversion <br>
-🕒 Checkpoint merging <br>
 🕒 ext. Area composition <br>
 
 

extensions/kd-multi-view/requirements.txt

@@ -0,0 +1,8 @@
+rembg==2.0.52
+pymatting==1.1.10
+altair==4.2.2
+numba==0.58.1
+llvmlite==0.41.1
+onnxruntime==1.16.1
+pooch==1.8.0
+git+https://github.com/facebookresearch/segment-anything@6fdee8f

extensions/kd-multi-view/setup_ext.py

@@ -0,0 +1,85 @@
+import torch
+import random
+import gradio as gr
+import numpy
+import rembg
+
+from PIL import Image
+from diffusers import (
+    DiffusionPipeline,
+    EulerAncestralDiscreteScheduler,
+    DDPMParallelScheduler,
+)
+
+title = "Multi View"
+
+
+def setup(kubin):
+    pipeline = None
+    source_image = gr.Image(type="pil", label="Source image")
+
+    def create_multiview(cache_dir, device, source_image, seed):
+        nonlocal pipeline
+
+        if pipeline is None:
+            pipeline = DiffusionPipeline.from_pretrained(
+                "sudo-ai/zero123plus-v1.1",
+                custom_pipeline="sudo-ai/zero123plus-pipeline",
+                torch_dtype=torch.float16,
+                cache_dir=cache_dir,
+            )
+
+            pipeline.scheduler = DDPMParallelScheduler.from_config(
+                pipeline.scheduler.config, timestep_spacing="trailing"
+            )
+
+            pipeline.to(device)
+
+        if seed == -1:
+            seed = random.randint(1, 1000000)
+
+        source_image = rembg.remove(source_image)
+        result = pipeline(
+            source_image,
+            num_inference_steps=75,
+            generator=torch.Generator(pipeline.device).manual_seed(int(seed)),
+        ).images[0]
+        return result
+
+    def multiview_ui(ui_shared, ui_tabs):
+        with gr.Row() as multiview_block:
+            with gr.Column(scale=1) as multiview_params_block:
+                with gr.Row():
+                    source_image.render()
+
+            with gr.Column(scale=1):
+                create_btn = gr.Button(
+                    "Generate multiview", label="Generate multiview", variant="primary"
+                )
+                multiview_output = gr.Image(label="Generated multiview")
+
+            kubin.ui_utils.click_and_disable(
+                create_btn,
+                fn=create_multiview,
+                inputs=[
+                    gr.State(kubin.params("general", "cache_dir")),
+                    gr.State(kubin.params("general", "device")),
+                    source_image,
+                    gr.State(-1),
+                ],
+                outputs=multiview_output,
+                js=[
+                    f"args => kubin.UI.taskStarted('{title}')",
+                    f"args => kubin.UI.taskFinished('{title}')",
+                ],
+            )
+
+            multiview_params_block.elem_classes = ["block-params"]
+        return multiview_block
+
+    return {
+        "send_to": f"📇 Send to {title}",
+        "title": title,
+        "tab_ui": lambda ui_s, ts: multiview_ui(ui_s, ts),
+        "send_target": source_image,
+    }

extensions/kd-multi-view/setup_ext.yaml

@@ -0,0 +1 @@
+pip_args: --no-deps

notebooks/kubin-colab.ipynb

@@ -82,6 +82,7 @@
     "image_browser = True #@param {type:\"boolean\"}\n",
     "interrogator = True #@param {type:\"boolean\"}\n",
     "mesh_gen = False #@param {type:\"boolean\"}\n",
+    "multi_view = False #@param {type:\"boolean\"}\n",
     "animation = True #@param {type:\"boolean\"}\n",
     "prompt_styles = True #@param {type:\"boolean\"}\n",
     "segmentation = False #@param {type:\"boolean\"}\n",
@@ -92,6 +93,7 @@
     "if not image_browser: disabled_extensions.append('kd-image-browser')\n",
     "if not interrogator: disabled_extensions.append('kd-interrogator')\n",
     "if not mesh_gen: disabled_extensions.append('kd-mesh-gen')\n",
+    "if not multi_view: disabled_extensions.append('kd-multi-view')\n",
     "if not animation: disabled_extensions.append('kd-animation')\n",
     "if not prompt_styles: disabled_extensions.append('kd-prompt-styles')\n",
     "if not segmentation: disabled_extensions.append('kd-segmentation')\n",
@@ -159,7 +161,7 @@
     "!pip install -r requirements.txt\n",
     "\n",
     "if use_flash_attention:\n",
-    "  !pip install /content/kubin/wheels/flash_attn-1.0.1+cu118torch2.0.0-cp310-cp310-linux_x86_64.whl\n",
+    "  !pip install /content/kubin/wheels/flash_attn-1.0.9+cu118torch2.1.0-cp310-cp310-linux_x86_64.whl\n",
     "\n",
     "if use_xformers:\n",
     "  !pip install xformers==0.0.20\n",

requirements.txt

@@ -1,6 +1,6 @@
 git+https://github.com/ai-forever/Kandinsky-2.git@33c3176
 git+https://github.com/openai/CLIP.git@a1d0717
-diffusers==0.19.3
+diffusers==0.20.2
 transformers==4.31.0
 accelerate==0.21.0
 opencv-python==4.8.0.74

src/extension/ext_registry.py

@@ -4,6 +4,7 @@
 import importlib.util
 import sys
 import yaml
+import platform
 
 from utils.logging import k_log
 
@@ -142,13 +143,24 @@ def register(self, kubin):
             open(postinstall_reqs_installed, "a").close()
 
     def install_pip_reqs(self, reqs_path, arguments=[]):
-        # TODO: venv should be activated, otherwise pip installs globally
-        # venv_activation_cmd = f"call venv\Scripts\activate.bat"
-        # subprocess.run(venv_activation_cmd, shell=True)
+        current_platform = platform.system()
 
-        subprocess.check_call(
-            [sys.executable, "-m", "pip", "install", "-r", f"{reqs_path}"] + arguments
-        )
+        if current_platform == "Windows":
+            venv_activation_path = os.path.join("venv", "Scripts", "activate.bat")
+        else:
+            venv_activation_path = os.path.join("venv", "bin", "activate")
+
+        if os.path.exists(venv_activation_path):
+            if current_platform == "Windows":
+                pip_install_cmd = f"call {venv_activation_path} && {sys.executable} -m pip install -r {reqs_path} {' '.join(arguments)}"
+            else:
+                pip_install_cmd = f". {venv_activation_path} && {sys.executable} -m pip install -r {reqs_path} {' '.join(arguments)}"
+        else:
+            pip_install_cmd = (
+                f"{sys.executable} -m pip install -r {reqs_path} {' '.join(arguments)}"
+            )
+
+        subprocess.check_call(pip_install_cmd, shell=True)
 
     def standalone(self):
         return list(

src/models/model_diffusers22/patched/patched.py

@@ -22,12 +22,8 @@
 
 from diffusers.models import UNet2DConditionModel, VQModel
 from diffusers.schedulers import DDPMScheduler
-from diffusers.utils import (
-    is_accelerate_available,
-    is_accelerate_version,
-    logging,
-    randn_tensor,
-)
+from diffusers.utils import is_accelerate_available, is_accelerate_version, logging
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
 
 

src/models/model_diffusers22/patched/patched_controlnet.py

@@ -26,8 +26,8 @@
     is_accelerate_available,
     is_accelerate_version,
     logging,
-    randn_tensor,
 )
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
 
 

src/models/model_diffusers22/patched/patched_controlnet_img2img.py

@@ -25,13 +25,8 @@
 
 from diffusers.models import UNet2DConditionModel, VQModel
 from diffusers.schedulers import DDPMScheduler
-from diffusers.utils import (
-    is_accelerate_available,
-    is_accelerate_version,
-    logging,
-    randn_tensor,
-    replace_example_docstring,
-)
+from diffusers.utils import is_accelerate_available, is_accelerate_version, logging
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
 
 

src/models/model_diffusers22/patched/patched_img2img.py

@@ -29,8 +29,8 @@
     is_accelerate_available,
     is_accelerate_version,
     logging,
-    randn_tensor,
 )
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
 
 

src/models/model_diffusers22/patched/patched_inpainting.py

@@ -32,9 +32,9 @@
     is_accelerate_available,
     is_accelerate_version,
     logging,
-    randn_tensor,
     replace_example_docstring,
 )
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
 
 

src/models/model_diffusers22/patched/patched_prior.py

@@ -33,8 +33,8 @@
     is_accelerate_available,
     is_accelerate_version,
     logging,
-    randn_tensor,
 )
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.kandinsky import KandinskyPriorPipelineOutput
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline
 

src/models/model_diffusers22/patched/patched_prior_emb2emb.py

@@ -33,8 +33,8 @@
     is_accelerate_available,
     is_accelerate_version,
     logging,
-    randn_tensor,
 )
+from diffusers.utils.torch_utils import randn_tensor
 from diffusers.pipelines.kandinsky import KandinskyPriorPipelineOutput
 from diffusers.pipelines.pipeline_utils import DiffusionPipeline