Skip to content

Commit

Permalink
Animation SDK (#224)
Browse files Browse the repository at this point in the history 
SDK support for creating animations with Stability Python SDK. Check out the [release notes](https://platform.stability.ai/docs/release-notes#stable-animation-sdk-release) on the platform site and full details on creating animations in the [platform docs](https://platform.stability.ai/docs/features/animation).

Co-authored-by: David Marx
Co-authored-by: Adam Letts
Co-authored-by: Dmitrii Tochilkin
  • Loading branch information
@pharmapsychotic
pharmapsychotic authored May 11, 2023
1 parent 5bb000c commit 1e85abd
Show file tree
Hide file tree
Showing 21 changed files with 3,915 additions and 179 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/unit_testing.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@ jobs:
cache: 'pip'
cache-dependency-path: |
**/setup.py
- name: Install dependencies
run: |
python -m pip install --upgrade pip
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Install package
run: |
pip install .[dev]
pip install .[dev,anim]
- name: Test with pytest
run: |
pytest -s --ignore=src/stability_sdk/interfaces
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
*.pyc
*.egg-info
*.png
.vscode/
build/
dist/
pyenv/
*venv/
.env
generation-*.pb.json
Pipfile*
10 changes: 9 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ It will generate and put PNGs in your current directory.
To upscale:
`python3 -m stability_sdk upscale -i "/path/to/image.png"`

## Animation UI

Install with
`pip install stability-sdk[anim_ui]`

Then run with
`python3 -m stability_sdk animate --gui`

## SDK Usage

Be sure to check out [Platform](https://platform.stability.ai) for comprehensive documentation on how to interact with our API.
Expand All @@ -57,7 +65,7 @@ options:
--width WIDTH, -W WIDTH
[512] width of image
--start_schedule START_SCHEDULE
[0.5] start schedule for init image (must be greater than 0, 1 is full strength
[0.5] start schedule for init image (must be greater than 0; 1 is full strength
text prompt, no trace of image)
--end_schedule END_SCHEDULE
[0.01] end schedule for init image
Expand Down
303 changes: 303 additions & 0 deletions nbs/animation.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,303 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "rWXrHouW3pq_"
},
"source": [
"# Animation SDK example"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "jCZ-IphH3prD"
},
"outputs": [],
"source": [
"#@title Mount Google Drive\n",
"try:\n",
" from google.colab import drive\n",
" drive.mount('/content/gdrive')\n",
" outputs_path = \"/content/gdrive/MyDrive/AI/StableAnimation\"\n",
" !mkdir -p $outputs_path\n",
"except:\n",
" outputs_path = \".\"\n",
"print(f\"Animations will be saved to {outputs_path}\")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"cellView": "form",
"id": "zj56t6tc3prF"
},
"outputs": [],
"source": [
"%%capture\n",
"#@title Connect to the Stability API\n",
"\n",
"# install Stability Animation SDK for Python\n",
"%pip install stability-sdk[anim]\n",
"\n",
"import datetime\n",
"import json\n",
"import os\n",
"import panel as pn\n",
"import param\n",
"import shutil\n",
"import sys\n",
"\n",
"from base64 import b64encode\n",
"from IPython import display\n",
"from pathlib import Path\n",
"from PIL import Image\n",
"from tqdm import tqdm\n",
"from types import SimpleNamespace\n",
"\n",
"from stability_sdk.api import Context\n",
"from stability_sdk.animation import AnimationArgs, Animator\n",
"from stability_sdk.utils import create_video_from_frames\n",
"\n",
"\n",
"# Enter your API key from dreamstudio.ai\n",
"STABILITY_HOST = \"grpc.stability.ai:443\" #@param {type:\"string\"}\n",
"STABILITY_KEY = \"\" #@param {type:\"string\"}\n",
"\n",
"# Connect to Stability API\n",
"api_context = Context(STABILITY_HOST, STABILITY_KEY)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "ldUAFmur3prH"
},
"outputs": [],
"source": [
"# @title Settings\n",
"\n",
"# @markdown Run this cell to reveal the settings UI. After entering values, move on to the next step.\n",
"\n",
"# @markdown To reset values to default, simply re-run this cell.\n",
"\n",
"# @markdown NB: Settings are grouped across several tabs.\n",
"\n",
"show_documentation = True # @param {type:'boolean'}\n",
"\n",
"# #@markdown ####**Resume:**\n",
"resume_timestring = \"\" #@param {type:\"string\"}\n",
"\n",
"#@markdown ####**Override Settings:**\n",
"override_settings_path = \"\" #@param {type:\"string\"}\n",
"\n",
"###################\n",
"\n",
"from stability_sdk.animation import (\n",
" AnimationArgs,\n",
" Animator,\n",
" AnimationSettings,\n",
" BasicSettings,\n",
" CoherenceSettings,\n",
" ColorSettings,\n",
" DepthSettings,\n",
" InpaintingSettings,\n",
" Rendering3dSettings,\n",
" CameraSettings,\n",
" VideoInputSettings,\n",
" VideoOutputSettings,\n",
")\n",
"\n",
"args_generation = BasicSettings()\n",
"args_animation = AnimationSettings()\n",
"args_camera = CameraSettings()\n",
"args_coherence = CoherenceSettings()\n",
"args_color = ColorSettings()\n",
"args_depth = DepthSettings()\n",
"args_render_3d = Rendering3dSettings()\n",
"args_inpaint = InpaintingSettings()\n",
"args_vid_in = VideoInputSettings()\n",
"args_vid_out = VideoOutputSettings()\n",
"arg_objs = (\n",
" args_generation,\n",
" args_animation,\n",
" args_camera,\n",
" args_coherence,\n",
" args_color,\n",
" args_depth,\n",
" args_render_3d,\n",
" args_inpaint,\n",
" args_vid_in,\n",
" args_vid_out,\n",
")\n",
"\n",
"def _show_docs(component):\n",
" cols = []\n",
" for k, v in component.param.objects().items():\n",
" if k == 'name':\n",
" continue\n",
" col = pn.Column(v, v.doc)\n",
" cols.append(col)\n",
" return pn.Column(*cols)\n",
"\n",
"def build(component):\n",
" if show_documentation:\n",
" component = _show_docs(component)\n",
" return pn.Row(component, width=1000)\n",
"\n",
"pn.extension()\n",
"\n",
"pn.Tabs(*[(a.name[:-5], build(a)) for a in arg_objs])"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "_SudvbZG3prI"
},
"source": [
"### Prompts"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "FT9slDSw3prJ"
},
"outputs": [],
"source": [
"animation_prompts = {\n",
" 0: \"a painting of a delicious cheeseburger\",\n",
" 24: \"a painting of the the answer to life the universe and everything\",\n",
"}\n",
"\n",
"negative_prompt = \"\"\n",
"negative_prompt_weight = -1.0\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "Rpqv6t303prJ"
},
"outputs": [],
"source": [
"#@title Render the animation\n",
"\n",
"args_d = {}\n",
"[args_d.update(a.param.values()) for a in arg_objs]\n",
"args=AnimationArgs(**args_d)\n",
"\n",
"\n",
"# load override settings if provided\n",
"if override_settings_path:\n",
" if not os.path.exists(override_settings_path):\n",
" raise ValueError(f\"Override settings file not found: {override_settings_path}\")\n",
" with open(override_settings_path, 'r') as f:\n",
" overrides = json.load(f)\n",
" args = vars(args)\n",
" for k in args.keys():\n",
" if k in overrides:\n",
" args[k] = overrides[k]\n",
" args = SimpleNamespace(**args)\n",
" animation_prompts = overrides.get('animation_prompts', animation_prompts)\n",
" animation_prompts = {int(k): v for k, v in animation_prompts.items()}\n",
" negative_prompt = overrides.get('negative_prompt', negative_prompt)\n",
" negative_prompt_weight = overrides.get('negative_prompt_weight', negative_prompt_weight)\n",
"\n",
"# create folder for frames output\n",
"if resume_timestring:\n",
" out_dir = os.path.join(outputs_path, resume_timestring)\n",
" if not os.path.exists(out_dir):\n",
" raise Exception(\"Can't resume {resume_timestring} because path {out_dir} doesn't exist. Please make sure the timestring is correct.\")\n",
" timestring = resume_timestring\n",
"else:\n",
" timestring = datetime.datetime.now().strftime('%Y%m%d%H%M%S')\n",
" out_dir = os.path.join(outputs_path, timestring)\n",
" os.makedirs(out_dir, exist_ok=True)\n",
"print(f\"Saving animation frames to {out_dir}...\")\n",
"\n",
"animator = Animator(\n",
" api_context=api_context,\n",
" animation_prompts=animation_prompts,\n",
" args=args,\n",
" out_dir=out_dir, \n",
" negative_prompt=negative_prompt,\n",
" negative_prompt_weight=negative_prompt_weight,\n",
" resume=len(resume_timestring) != 0,\n",
")\n",
"animator.save_settings(f\"{timestring}_settings.txt\")\n",
"\n",
"for frame in tqdm(animator.render(), initial=animator.start_frame_idx, total=args.max_frames):\n",
" display.clear_output(wait=True)\n",
" display.display(frame)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "aWhJnLNX3prL"
},
"outputs": [],
"source": [
"#@title Create video from frames\n",
"skip_video_for_run_all = False #@param {type: 'boolean'}\n",
"fps = 12 #@param {type:\"number\"}\n",
"\n",
"if skip_video_for_run_all == True:\n",
" print('Skipping video creation, uncheck skip_video_for_run_all if you want to run it')\n",
"else:\n",
" mp4_path = os.path.join(out_dir, f\"{timestring}.mp4\")\n",
" print(f\"Compiling animation frames to {mp4_path}...\")\n",
" create_video_from_frames(out_dir, mp4_path, fps)\n",
"\n",
" mp4 = open(mp4_path,'rb').read()\n",
" data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
" display.display( display.HTML(f'<video controls loop><source src=\"{data_url}\" type=\"video/mp4\"></video>') )"
]
}
],
"metadata": {
"colab": {
"collapsed_sections": [],
"provenance": []
},
"kernelspec": {
"display_name": "client",
"language": "python",
"name": "client"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "fb02550c4ef2b9a37ba5f7f381e893a74079cea154f791601856f87ae67cf67c"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit 1e85abd

Please sign in to comment.