diff --git a/cookbooks/cosmos3/generator/audiovisual/run_with_sglang.ipynb b/cookbooks/cosmos3/generator/audiovisual/run_with_sglang.ipynb index 86f742f8..002bc4d0 100644 --- a/cookbooks/cosmos3/generator/audiovisual/run_with_sglang.ipynb +++ b/cookbooks/cosmos3/generator/audiovisual/run_with_sglang.ipynb @@ -133,6 +133,8 @@ "SGLANG_ENDPOINTS = {\n", " \"Cosmos3-Nano\": os.environ.get(\"COSMOS3_SGLANG_NANO_BASE_URL\", DEFAULT_SGLANG_BASE_URL),\n", " \"Cosmos3-Super\": os.environ.get(\"COSMOS3_SGLANG_SUPER_BASE_URL\", DEFAULT_SGLANG_BASE_URL),\n", + " \"Cosmos3-Edge\": os.environ.get(\"COSMOS3_SGLANG_EDGE_BASE_URL\", DEFAULT_SGLANG_BASE_URL),\n", + " \"Cosmos3-Distil\": os.environ.get(\"COSMOS3_SGLANG_DISTIL_BASE_URL\", DEFAULT_SGLANG_BASE_URL),\n", "}\n", "\n", "os.environ[\"COSMOS3_AUDIOVISUAL_OUTPUT_ROOT\"] = str(COSMOS3_AUDIOVISUAL_OUTPUT_ROOT)\n", @@ -266,12 +268,30 @@ " \"prompt\": \"assets/prompts/text2image/robot_draping.json\",\n", " \"enable_sound\": False,\n", " },\n", + " \"t2i_edge\": {\n", + " \"model\": \"Cosmos3-Edge\",\n", + " \"mode\": \"text2image\",\n", + " \"prompt\": \"assets/prompts/text2image/robot_draping.json\",\n", + " \"enable_sound\": False,\n", + " },\n", + " \"t2i_distil\": {\n", + " \"model\": \"Cosmos3-Distil\",\n", + " \"mode\": \"text2image\",\n", + " \"prompt\": \"assets/prompts/text2image/robot_draping.json\",\n", + " \"enable_sound\": False,\n", + " },\n", " \"t2v_nano_noaudio\": {\n", " \"model\": \"Cosmos3-Nano\",\n", " \"mode\": \"text2video\",\n", " \"prompt\": \"assets/prompts/text2video/robot_kitchen.json\",\n", " \"enable_sound\": False,\n", " },\n", + " \"t2v_edge_noaudio\": {\n", + " \"model\": \"Cosmos3-Edge\",\n", + " \"mode\": \"text2video\",\n", + " \"prompt\": \"assets/prompts/text2video/robot_kitchen.json\",\n", + " \"enable_sound\": False,\n", + " },\n", " \"t2vs\": {\n", " \"model\": \"Cosmos3-Nano\",\n", " \"mode\": \"text2video\",\n", @@ -285,6 +305,20 @@ " \"image\": \"assets/images/image2video/car_driving.jpg\",\n", " \"enable_sound\": False,\n", " },\n", + " \"i2v_edge_noaudio\": {\n", + " \"model\": \"Cosmos3-Edge\",\n", + " \"mode\": \"image2video\",\n", + " \"prompt\": \"assets/prompts/image2video/car_driving.json\",\n", + " \"image\": \"assets/images/image2video/car_driving.jpg\",\n", + " \"enable_sound\": False,\n", + " },\n", + " \"i2v_distil_noaudio\": {\n", + " \"model\": \"Cosmos3-Distil\",\n", + " \"mode\": \"image2video\",\n", + " \"prompt\": \"assets/prompts/image2video/car_driving.json\",\n", + " \"image\": \"assets/images/image2video/car_driving.jpg\",\n", + " \"enable_sound\": False,\n", + " },\n", " \"i2vs\": {\n", " \"model\": \"Cosmos3-Nano\",\n", " \"mode\": \"image2video\",\n", @@ -315,6 +349,11 @@ " },\n", "}\n", "\n", + "def _is_distil(model_name: str) -> bool:\n", + " return \"Distil\" in model_name\n", + "\n", + "def _is_edge(model_name: str) -> bool:\n", + " return \"Edge\" in model_name\n", "\n", "def asset_path(relative_path: str) -> Path:\n", " path = COSMOS3_AUDIOVISUAL_ROOT / relative_path\n", @@ -327,27 +366,15 @@ " return json.dumps(json.loads(path.read_text()), ensure_ascii=True, separators=(\",\", \":\"))\n", "\n", "\n", - "def normalize_negative_prompt(value) -> str:\n", - " if value is None:\n", - " return \"\"\n", - " if isinstance(value, str):\n", - " return value\n", - " if isinstance(value, dict):\n", - " parts = []\n", - " for v in value.values():\n", - " if isinstance(v, str):\n", - " parts.append(v)\n", - " elif isinstance(v, list):\n", - " parts.extend(str(x) for x in v)\n", - " return \"\\n\".join(parts)\n", - " return str(value)\n", - "\n", - "\n", "def payload_dimensions(payload: dict) -> tuple[int, int]:\n", " if payload.get(\"resolution\") == \"720\" and payload.get(\"aspect_ratio\") == \"16,9\":\n", " return 720, 1280\n", - " if payload.get(\"resolution\") == \"256\" and payload.get(\"aspect_ratio\") == \"16,9\":\n", - " return 192, 320\n", + " if payload.get(\"resolution\") == \"480\" and payload.get(\"aspect_ratio\") == \"16,9\":\n", + " return 480, 832\n", + " if payload.get(\"resolution\") == \"640\" and payload.get(\"aspect_ratio\") == \"1,1\":\n", + " return 640, 640\n", + " if payload.get(\"resolution\") == \"256\" and payload.get(\"aspect_ratio\") == \"1,1\":\n", + " return 256, 256\n", " raise ValueError(f\"Unsupported payload resolution/aspect ratio: {payload.get('resolution')} {payload.get('aspect_ratio')}\")\n", "\n", "\n", @@ -368,8 +395,8 @@ " prompt_path = asset_path(spec[\"prompt\"])\n", " negative_prompt = \"\"\n", " if spec[\"mode\"] != \"text2image\":\n", - " negative_prompt_path = asset_path(f\"assets/negative_prompts/{spec['mode']}/neg_prompt.json\") if not spec.get(\"negative_prompt\") else spec[\"negative_prompt\"]\n", - " negative_prompt = normalize_negative_prompt(negative_prompt_path)\n", + " negative_prompt_path = asset_path(f\"assets/negative_prompts/{spec['mode']}/neg_prompt.json\") if not spec.get(\"negative_prompt\") else asset_path(spec[\"negative_prompt\"])\n", + " negative_prompt = compact_json_file(negative_prompt_path)\n", " payload_path = payload_dir / f\"{use_case}.json\"\n", " payload = {\n", " \"model_mode\": spec[\"mode\"],\n", @@ -387,6 +414,16 @@ " video_path = asset_path(spec[\"video\"])\n", " payload[\"vision_path\"] = os.path.relpath(video_path, payload_path.parent)\n", "\n", + " if _is_distil(spec[\"model\"]) or _is_edge(spec[\"model\"]):\n", + " payload.pop(\"guidance\", None)\n", + " payload.pop(\"shift\", None)\n", + "\n", + " if _is_distil(spec[\"model\"]):\n", + " payload.pop(\"num_steps\", None)\n", + "\n", + " if _is_edge(spec[\"model\"]):\n", + " payload[\"resolution\"] = \"480\"\n", + " \n", " payload_path.write_text(json.dumps(payload, indent=2) + \"\\n\")\n", "\n", " os.environ[f\"COSMOS3_{backend.upper()}_{use_case.upper()}_INPUT\"] = str(payload_path)\n", @@ -405,7 +442,7 @@ " video_display_path = resolve_payload_path(payload_path, payload[\"vision_path\"])\n", " print(f\"video: {video_display_path.relative_to(COSMOS_ROOT)}\")\n", " display(Video(filename=str(video_display_path), width=420))\n", - " print(json.dumps({k: payload[k] for k in [\"model_mode\", \"name\", \"enable_sound\", \"num_steps\", \"guidance\", \"shift\", \"fps\", \"num_frames\", \"resolution\", \"aspect_ratio\", \"seed\"]}, indent=2))\n", + " print(json.dumps({k: payload[k] for k in [\"model_mode\", \"name\", \"enable_sound\", \"num_steps\", \"guidance\", \"shift\", \"fps\", \"num_frames\", \"resolution\", \"aspect_ratio\", \"seed\"] if k in payload}, indent=2))\n", " return payload_path, output_dir, spec[\"model\"]\n", "\n", "\n", @@ -446,9 +483,9 @@ " \"size\": f\"{width}x{height}\",\n", " \"num_frames\": str(payload[\"num_frames\"]),\n", " \"fps\": str(payload[\"fps\"]),\n", - " \"num_inference_steps\": str(payload[\"num_steps\"]),\n", - " \"guidance_scale\": str(payload[\"guidance\"]),\n", - " \"flow_shift\": str(payload[\"shift\"]),\n", + " \"num_inference_steps\": str(payload[\"num_steps\"]) if \"num_steps\" in payload else None,\n", + " \"guidance_scale\": str(payload[\"guidance\"]) if \"guidance\" in payload else None,\n", + " \"flow_shift\": str(payload[\"shift\"]) if \"shift\" in payload else None,\n", " \"seed\": str(payload[\"seed\"]),\n", " \"extra_params\": json.dumps(extra_params, separators=(\",\", \":\")),\n", " }\n", @@ -465,9 +502,9 @@ " \"negative_prompt\": payload.get(\"negative_prompt\", \"\"),\n", " \"size\": f\"{width}x{height}\",\n", " \"n\": 1,\n", - " \"num_inference_steps\": payload[\"num_steps\"],\n", - " \"guidance_scale\": payload[\"guidance\"],\n", - " \"flow_shift\": payload[\"shift\"],\n", + " \"num_inference_steps\": payload[\"num_steps\"] if \"num_steps\" in payload else None,\n", + " \"guidance_scale\": payload[\"guidance\"] if \"guidance\" in payload else None,\n", + " \"flow_shift\": payload[\"shift\"] if \"shift\" in payload else None,\n", " \"seed\": payload[\"seed\"],\n", " \"response_format\": \"b64_json\",\n", " \"extra_params\": {\n", @@ -501,6 +538,8 @@ " cmd += [\"-H\", f\"Authorization: Bearer {api_key}\"]\n", "\n", " for key, value in build_sglang_video_form(payload).items():\n", + " if value is None:\n", + " continue\n", " cmd += [\"--form-string\", f\"{key}={value}\"]\n", "\n", " if payload[\"model_mode\"] == \"image2video\":\n", @@ -1182,11 +1221,336 @@ "source": [ "view_run(i2v_super_noaudio_output)\n" ] + }, + { + "cell_type": "markdown", + "id": "cd022d1d", + "metadata": {}, + "source": [ + "# Cosmos3-Edge Examples\n", + "\n", + "Use cases for the smaller `Cosmos3-Edge` model. This section is self-contained; you can run it without the other sections above.\n" + ] + }, + { + "cell_type": "markdown", + "id": "0b168305", + "metadata": {}, + "source": [ + "## Edge: Text to Image\n", + "\n", + "Edge text-to-image generation using the same structured JSON prompt.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af8ddac1", + "metadata": {}, + "outputs": [], + "source": [ + "t2i_edge_payload, t2i_edge_output, t2i_edge_model = create_payload(\"t2i_edge\", backend=\"sglang\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "a42cb79b", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5515508", + "metadata": {}, + "outputs": [], + "source": [ + "run_sglang_payload(t2i_edge_payload, t2i_edge_output, model=\"Cosmos3-Edge\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "0502a370", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59b155c8", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(t2i_edge_output)\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "c0592326", + "metadata": {}, + "source": [ + "## Edge: Text to Video Without Audio\n", + "\n", + "Edge text-to-video generation with audio disabled.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed85b5ff", + "metadata": {}, + "outputs": [], + "source": [ + "t2v_edge_noaudio_payload, t2v_edge_noaudio_output, t2v_edge_noaudio_model = create_payload(\"t2v_edge_noaudio\", backend=\"sglang\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "db0e6065", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c8e28db", + "metadata": {}, + "outputs": [], + "source": [ + "run_sglang_payload(t2v_edge_noaudio_payload, t2v_edge_noaudio_output, model=\"Cosmos3-Edge\")\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "63168603", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9702913c", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(t2v_edge_noaudio_output)\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "3c3b2d01", + "metadata": {}, + "source": [ + "## Edge: Image to Video Without Audio\n", + "\n", + "Edge image-to-video generation using its paired image asset, with audio disabled.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4640e93", + "metadata": {}, + "outputs": [], + "source": [ + "i2v_edge_noaudio_payload, i2v_edge_noaudio_output, i2v_edge_noaudio_model = create_payload(\"i2v_edge_noaudio\", backend=\"sglang\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "8d5d0873", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2721deaa", + "metadata": {}, + "outputs": [], + "source": [ + "run_sglang_payload(i2v_edge_noaudio_payload, i2v_edge_noaudio_output, model=\"Cosmos3-Edge\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "13ae89e3", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10f39756", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(i2v_edge_noaudio_output)\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "d862a1f6", + "metadata": {}, + "source": [ + "# Cosmos3-Distil Examples\n", + "\n", + "Use cases for the smaller `Cosmos3-Distil` model. This section is self-contained; you can run it without the other sections above.\n" + ] + }, + { + "cell_type": "markdown", + "id": "7bb59b25", + "metadata": {}, + "source": [ + "## Distil: Text to Image\n", + "\n", + "Distil text-to-image generation using the same structured JSON prompt.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3eade39c", + "metadata": {}, + "outputs": [], + "source": [ + "t2i_distil_payload, t2i_distil_output, t2i_distil_model = create_payload(\"t2i_distil\", backend=\"sglang\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "a3367e63", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5908dc43", + "metadata": {}, + "outputs": [], + "source": [ + "run_sglang_payload(t2i_distil_payload, t2i_distil_output, model=\"Cosmos3-Distil\")\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "2b23b582", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e67db0de", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(t2i_distil_output)\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "35481725", + "metadata": {}, + "source": [ + "## Distil: Image to Video Without Audio\n", + "\n", + "Distil image-to-video generation using its paired image asset, with audio disabled.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d737217e", + "metadata": {}, + "outputs": [], + "source": [ + "i2v_distil_noaudio_payload, i2v_distil_noaudio_output, i2v_distil_noaudio_model = create_payload(\"i2v_distil_noaudio\", backend=\"sglang\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "681f6f85", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3df984a", + "metadata": {}, + "outputs": [], + "source": [ + "run_sglang_payload(i2v_distil_noaudio_payload, i2v_distil_noaudio_output, model=\"Cosmos3-Distil\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "8ae59282", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06d4095c", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(i2v_distil_noaudio_output)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64c8ffe5", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": ".venv (3.12.3)", "language": "python", "name": "python3" },