diff --git a/backend/app/runners/api_runner.py b/backend/app/runners/api_runner.py index 1aeae8c..1869272 100644 --- a/backend/app/runners/api_runner.py +++ b/backend/app/runners/api_runner.py @@ -312,6 +312,82 @@ async def _call_fal_ai_action( return images[0]["url"] if images else None +async def _call_fal_ai_video( + action: str, + prompt: str, + api_key: str, + image_b64: Optional[str] = None, + fps: int = 16, + num_frames: int = 25, + motion_bucket_id: int = 127, +) -> str | None: + """Call Fal.ai video endpoints (Fast SVD for img2video, Luma/Kling for txt2video).""" + if not api_key: + raise APIRunnerError("FAL_KEY / IMAGE_API_KEY not set for Fal.ai video") + + headers = {"Authorization": f"Key {api_key}", "Content-Type": "application/json"} + + async with httpx.AsyncClient(timeout=180.0) as client: + if action == "img2video": + if not image_b64: + raise ValueError("Source image required for Fal.ai img2video") + endpoint = "https://fal.run/fal-ai/fast-svd/image-to-video" + payload = { + "image_url": f"data:image/png;base64,{image_b64}", + "motion_bucket_id": motion_bucket_id, + "fps": fps, + "cond_aug": 0.02, + "steps": 25, + } + else: # txt2video + endpoint = "https://fal.run/fal-ai/luma-dream-machine" + payload = { + "prompt": prompt, + "aspect_ratio": "16:9", + "loop": False, + } + + response = await client.post(endpoint, headers=headers, json=payload) + response.raise_for_status() + data = response.json() + if "video" in data and isinstance(data["video"], dict): + return data["video"].get("url") + return data.get("video_url") or data.get("url") + + +async def _call_siliconflow_video( + action: str, + prompt: str, + api_key: str, + image_b64: Optional[str] = None, +) -> str | None: + """Call SiliconFlow video generation endpoints (CogVideoX).""" + if not api_key: + raise APIRunnerError("SILICONFLOW_API_KEY not set for SiliconFlow video") + + headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + + async with httpx.AsyncClient(timeout=180.0) as client: + endpoint = "https://api.siliconflow.cn/v1/video/submit" + payload: Dict[str, Any] = { + "model": "THUDM/CogVideoX-5b", + "prompt": prompt, + } + if action == "img2video" and image_b64: + payload["image"] = f"data:image/png;base64,{image_b64}" + + response = await client.post(endpoint, headers=headers, json=payload) + response.raise_for_status() + data = response.json() + # In SiliconFlow async video, url is returned in uri or results + if "uri" in data: + return data["uri"] + if "data" in data and isinstance(data["data"], dict): + return data["data"].get("url") + return data.get("url") + + + # --------------------------------------------------------------------------- # Input node: passthrough (no API call needed) # --------------------------------------------------------------------------- diff --git a/backend/app/runners/creative_runner.py b/backend/app/runners/creative_runner.py index c423783..125221e 100644 --- a/backend/app/runners/creative_runner.py +++ b/backend/app/runners/creative_runner.py @@ -21,16 +21,20 @@ from app.runners.api_runner import ( _call_fal_ai, _call_fal_ai_action, + _call_fal_ai_video, _call_openai_images, _call_openai_inpaint, _call_siliconflow, + _call_siliconflow_video, ) from app.runners.comfy_runner import comfy_client from app.runners.macro_compiler import ( ASPECT_RATIO_DIMENSIONS, build_comfy_img2img_graph, + build_comfy_img2video_graph, build_comfy_inpaint_graph, build_comfy_txt2img_graph, + build_comfy_txt2video_graph, build_comfy_upscale_graph, ) from app.runners.webui_runner import WebUIRunner @@ -65,6 +69,10 @@ def compute_creative_cache_hash(req: CreativeActionRequest, input_hash: str = "" "mask_hash": mask_hash, "upscale_factor": req.upscale_factor, "upscaler_name": req.upscaler_name, + "fps": req.fps, + "num_frames": req.num_frames, + "motion_bucket_id": req.motion_bucket_id, + "duration_seconds": req.duration_seconds, } raw = json.dumps(canonical_payload, sort_keys=True) return hashlib.sha256(raw.encode("utf-8")).hexdigest() @@ -141,14 +149,19 @@ async def execute(self, req: CreativeActionRequest) -> CreativeActionResult: # Check deterministic cache cache_key = compute_creative_cache_hash(req, input_hash, mask_hash) cached_result = await cache_store.get_async(cache_key) + is_video = req.action in (CreativeActionType.TXT2VIDEO, CreativeActionType.IMG2VIDEO) + if cached_result: return CreativeActionResult( success=True, task_id=task_id, asset_id=cached_result.get("asset_id"), - image_url=cached_result.get("image_url"), + image_url=cached_result.get("image_url") if not is_video else None, + video_url=cached_result.get("video_url") or (cached_result.get("image_url") if is_video else None), width=cached_result.get("width", req.width), height=cached_result.get("height", req.height), + duration_seconds=cached_result.get("duration_seconds", req.duration_seconds if is_video else None), + fps=cached_result.get("fps", req.fps if is_video else None), provenance=GenerationProvenance.model_validate(cached_result["provenance"]), is_cached=True, ) @@ -164,6 +177,8 @@ async def execute(self, req: CreativeActionRequest) -> CreativeActionResult: # Dispatch based on engine try: if "webui" in req.engine_id: + if is_video: + raise ValueError("WebUI engine currently does not support native video generation. Use ComfyUI or Cloud.") runner = WebUIRunner() action_data = await runner.execute_action(req) asset_id = action_data["asset_id"] @@ -215,15 +230,22 @@ async def execute(self, req: CreativeActionRequest) -> CreativeActionResult: source_asset_id=req.input_image_id, mask_asset_id=req.mask_image_id, execution_time_ms=elapsed_ms, + fps=req.fps if is_video else None, + num_frames=req.num_frames if is_video else None, + duration_seconds=req.duration_seconds if is_video else None, + motion_bucket_id=req.motion_bucket_id if is_video else None, ) result = CreativeActionResult( success=True, task_id=task_id, asset_id=asset_id, - image_url=image_url, + image_url=image_url if not is_video else None, + video_url=image_url if is_video else None, width=out_w, height=out_h, + duration_seconds=req.duration_seconds if is_video else None, + fps=req.fps if is_video else None, provenance=provenance, is_cached=False, ) @@ -231,9 +253,12 @@ async def execute(self, req: CreativeActionRequest) -> CreativeActionResult: # Store in cache cache_payload = { "asset_id": asset_id, - "image_url": image_url, + "image_url": image_url if not is_video else None, + "video_url": image_url if is_video else None, "width": out_w, "height": out_h, + "duration_seconds": req.duration_seconds if is_video else None, + "fps": req.fps if is_video else None, "provenance": provenance.model_dump(), } await cache_store.set_async(cache_key, cache_payload) @@ -304,6 +329,34 @@ async def _run_comfy( image_filename=input_file.name, upscaler_model=req.upscaler_name or "RealESRGAN_x4plus.pth", ) + elif req.action == CreativeActionType.IMG2VIDEO: + if not input_file: + raise ValueError("Source image required for ComfyUI img2video") + prompt_graph = build_comfy_img2video_graph( + image_filename=input_file.name, + checkpoint=req.model if "svd" in req.model.lower() else "svd_xt.safetensors", + width=req.width, + height=req.height, + video_frames=req.num_frames, + fps=req.fps, + motion_bucket_id=req.motion_bucket_id, + seed=req.seed, + steps=req.steps, + cfg=req.cfg_scale, + ) + elif req.action == CreativeActionType.TXT2VIDEO: + prompt_graph = build_comfy_txt2video_graph( + prompt=req.prompt, + negative_prompt=req.negative_prompt, + checkpoint=req.model, + width=req.width, + height=req.height, + video_frames=req.num_frames, + fps=req.fps, + seed=req.seed, + steps=req.steps, + cfg=req.cfg_scale, + ) else: raise ValueError(f"Unsupported action: {req.action}") @@ -315,7 +368,7 @@ async def _run_comfy( outputs = await comfy_client.poll_history_outputs(prompt_id) if not outputs: - raise RuntimeError("ComfyUI finished with no output images") + raise RuntimeError("ComfyUI finished with no output media") first_img = outputs[0] # Download and store in local asset store @@ -324,7 +377,12 @@ async def _run_comfy( img_type = first_img.get("type", "output") view_url = f"{comfy_client.base_url}/view?filename={filename}&subfolder={subfolder}&type={img_type}" - asset = await asset_store.save_image_from_url(view_url, filename=filename or "comfy_output.png") + is_video = req.action in (CreativeActionType.TXT2VIDEO, CreativeActionType.IMG2VIDEO) + if is_video: + asset = await asset_store.save_media_from_url(view_url, filename=filename or "comfy_video.webp", media_type="video") + else: + asset = await asset_store.save_image_from_url(view_url, filename=filename or "comfy_output.png") + return { "asset_id": asset.id, "image_url": f"/api/v1/assets/{asset.id}/content", @@ -419,6 +477,48 @@ async def _run_cloud( denoise=req.denoise, ) + elif req.action == CreativeActionType.IMG2VIDEO: + if not image_b64: + raise ValueError("Source image is required for cloud img2video.") + key = credentials_manager.get_key(CloudProviderId.FAL) + if not key: + raise RuntimeError( + "Cloud img2video requires a Fal.ai BYOK key (Fast SVD). " + "Configure Fal.ai in Cloud Providers or use local ComfyUI." + ) + remote_url = await _call_fal_ai_video( + action="img2video", + prompt=req.prompt, + api_key=key, + image_b64=image_b64, + fps=req.fps, + num_frames=req.num_frames, + motion_bucket_id=req.motion_bucket_id, + ) + + elif req.action == CreativeActionType.TXT2VIDEO: + key_fal = credentials_manager.get_key(CloudProviderId.FAL) + if key_fal: + remote_url = await _call_fal_ai_video( + action="txt2video", + prompt=req.prompt, + api_key=key_fal, + fps=req.fps, + num_frames=req.num_frames, + ) + else: + key_sf = credentials_manager.get_key(CloudProviderId.SILICONFLOW) + if not key_sf: + raise RuntimeError( + "No cloud API key configured for video generation. " + "Configure a Fal.ai or SiliconFlow BYOK key in Cloud Settings." + ) + remote_url = await _call_siliconflow_video( + action="txt2video", + prompt=req.prompt, + api_key=key_sf, + ) + else: # TXT2IMG if "flux" in model or req.engine_id == "cloud_fal": @@ -447,9 +547,14 @@ async def _run_cloud( remote_url = await _call_siliconflow(req.prompt, req.width, req.height, key) if not remote_url: - raise RuntimeError(f"Cloud provider returned no image URL for action {req.action}.") + raise RuntimeError(f"Cloud provider returned no media URL for action {req.action}.") + + is_video = req.action in (CreativeActionType.TXT2VIDEO, CreativeActionType.IMG2VIDEO) + if is_video: + asset = await asset_store.save_media_from_url(remote_url, filename="cloud_video.mp4", media_type="video") + else: + asset = await asset_store.save_image_from_url(remote_url, filename="cloud_output.png") - asset = await asset_store.save_image_from_url(remote_url, filename="cloud_output.png") return { "asset_id": asset.id, "image_url": f"/api/v1/assets/{asset.id}/content", diff --git a/backend/app/runners/macro_compiler.py b/backend/app/runners/macro_compiler.py index b821cdf..686300e 100644 --- a/backend/app/runners/macro_compiler.py +++ b/backend/app/runners/macro_compiler.py @@ -294,3 +294,177 @@ def build_comfy_upscale_graph( } return graph + + +def build_comfy_img2video_graph( + image_filename: str, + checkpoint: str = "svd_xt.safetensors", + width: int = 1024, + height: int = 576, + video_frames: int = 25, + fps: int = 16, + motion_bucket_id: int = 127, + seed: Optional[int] = None, + steps: int = 20, + cfg: float = 2.5, +) -> Dict[str, Any]: + """Compile an image-to-video request using SVD into a ComfyUI prompt graph.""" + actual_seed = seed if seed is not None and seed >= 0 else random.randint(1, 1125899906842624) + graph: Dict[str, Any] = {} + + # Node 1: SVD Checkpoint Loader + graph["1"] = { + "class_type": "ImageOnlyCheckpointLoader", + "inputs": {"ckpt_name": checkpoint}, + } + + # Node 2: Input Image + graph["2"] = { + "class_type": "LoadImage", + "inputs": {"image": image_filename}, + } + + # Node 3: SVD Conditioning + graph["3"] = { + "class_type": "SVD_img2vid_Conditioning", + "inputs": { + "clip_vision": ["1", 1], + "init_image": ["2", 0], + "vae": ["1", 2], + "width": width, + "height": height, + "video_frames": video_frames, + "motion_bucket_id": motion_bucket_id, + "fps": fps, + "augmentation_level": 0.0, + }, + } + + # Node 4: KSampler + graph["4"] = { + "class_type": "KSampler", + "inputs": { + "seed": actual_seed, + "steps": steps, + "cfg": cfg, + "sampler_name": "euler", + "scheduler": "karras", + "denoise": 1.0, + "model": ["1", 0], + "positive": ["3", 0], + "negative": ["3", 1], + "latent_image": ["3", 2], + }, + } + + # Node 5: VAEDecode + graph["5"] = { + "class_type": "VAEDecode", + "inputs": {"samples": ["4", 0], "vae": ["1", 2]}, + } + + # Node 6: Save Animated Output + graph["6"] = { + "class_type": "SaveAnimatedWEBP", + "inputs": { + "filename_prefix": "Berry-Img2Vid", + "images": ["5", 0], + "fps": fps, + "lossless": False, + "quality": 85, + "method": "default", + }, + } + + return graph + + +def build_comfy_txt2video_graph( + prompt: str, + negative_prompt: str = "", + checkpoint: str = "v1-5-pruned-emaonly.safetensors", + animatediff_model: str = "mm_sd_v15_v2.ckpt", + width: int = 512, + height: int = 512, + video_frames: int = 16, + fps: int = 8, + seed: Optional[int] = None, + steps: int = 20, + cfg: float = 7.0, +) -> Dict[str, Any]: + """Compile a text-to-video request using AnimateDiff into a ComfyUI prompt graph.""" + actual_seed = seed if seed is not None and seed >= 0 else random.randint(1, 1125899906842624) + graph: Dict[str, Any] = {} + + # Node 1: Checkpoint Loader + graph["1"] = { + "class_type": "CheckpointLoaderSimple", + "inputs": {"ckpt_name": checkpoint}, + } + + # Node 2: AnimateDiff Loader + graph["2"] = { + "class_type": "AnimateDiffLoaderWithContext", + "inputs": { + "model": ["1", 0], + "model_name": animatediff_model, + "context_length": 16, + }, + } + + # Node 3: Positive CLIP + graph["3"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": prompt, "clip": ["1", 1]}, + } + + # Node 4: Negative CLIP + graph["4"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": negative_prompt, "clip": ["1", 1]}, + } + + # Node 5: Empty Latent + graph["5"] = { + "class_type": "EmptyLatentImage", + "inputs": {"width": width, "height": height, "batch_size": video_frames}, + } + + # Node 6: KSampler + graph["6"] = { + "class_type": "KSampler", + "inputs": { + "seed": actual_seed, + "steps": steps, + "cfg": cfg, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1.0, + "model": ["2", 0], + "positive": ["3", 0], + "negative": ["4", 0], + "latent_image": ["5", 0], + }, + } + + # Node 7: VAEDecode + graph["7"] = { + "class_type": "VAEDecode", + "inputs": {"samples": ["6", 0], "vae": ["1", 2]}, + } + + # Node 8: Save Animated Output + graph["8"] = { + "class_type": "SaveAnimatedWEBP", + "inputs": { + "filename_prefix": "Berry-Txt2Vid", + "images": ["7", 0], + "fps": fps, + "lossless": False, + "quality": 85, + "method": "default", + }, + } + + return graph + diff --git a/backend/app/schemas/creative.py b/backend/app/schemas/creative.py index 186da0a..789fd61 100644 --- a/backend/app/schemas/creative.py +++ b/backend/app/schemas/creative.py @@ -10,6 +10,8 @@ class CreativeActionType(str, Enum): IMG2IMG = "img2img" INPAINT = "inpaint" UPSCALE = "upscale" + TXT2VIDEO = "txt2video" + IMG2VIDEO = "img2video" class CreativeActionRequest(BaseModel): @@ -25,10 +27,15 @@ class CreativeActionRequest(BaseModel): cfg_scale: float = 7.0 seed: int = -1 # -1 for random denoise: float = 0.75 # For img2img and inpaint (0.0 to 1.0) - input_image_id: Optional[str] = None # Reference asset ID for img2img / inpaint / upscale + input_image_id: Optional[str] = None # Reference asset ID for img2img / inpaint / upscale / img2video mask_image_id: Optional[str] = None # Binary mask asset ID for inpaint upscale_factor: float = 2.0 # 2.0 or 4.0 upscaler_name: str = "R-ESRGAN 4x+" + # Video generation parameters (M7) + fps: int = 16 + num_frames: int = 25 + motion_bucket_id: int = 127 + duration_seconds: float = 3.0 class GenerationProvenance(BaseModel): @@ -45,6 +52,11 @@ class GenerationProvenance(BaseModel): source_asset_id: Optional[str] = None mask_asset_id: Optional[str] = None execution_time_ms: Optional[float] = None + # Video-specific provenance + fps: Optional[int] = None + num_frames: Optional[int] = None + duration_seconds: Optional[float] = None + motion_bucket_id: Optional[int] = None class CreativeActionResult(BaseModel): @@ -52,8 +64,11 @@ class CreativeActionResult(BaseModel): task_id: str asset_id: Optional[str] = None image_url: Optional[str] = None + video_url: Optional[str] = None width: int = 512 height: int = 512 + duration_seconds: Optional[float] = None + fps: Optional[int] = None provenance: Optional[GenerationProvenance] = None is_cached: bool = False error_message: Optional[str] = None diff --git a/backend/app/storage/asset_store.py b/backend/app/storage/asset_store.py index 96fe8e0..36df24a 100644 --- a/backend/app/storage/asset_store.py +++ b/backend/app/storage/asset_store.py @@ -80,13 +80,25 @@ async def save_image_from_url( timeout: float = 30.0, ) -> AssetRecord: """Download remote image and persist into managed asset store.""" + return await self.save_media_from_url(url, filename=filename, media_type="image", project_id=project_id, timeout=timeout) + + async def save_media_from_url( + self, + url: str, + filename: Optional[str] = None, + media_type: str = "image", + project_id: Optional[str] = None, + timeout: float = 60.0, + ) -> AssetRecord: + """Download remote image or video and persist into managed asset store.""" async with httpx.AsyncClient(timeout=timeout) as client: resp = await client.get(url) resp.raise_for_status() data = resp.content - name = filename or Path(url.split("?")[0]).name or f"image_{uuid.uuid4().hex[:8]}.png" - return await self.save_bytes(data=data, filename=name, media_type="image", project_id=project_id) + default_ext = ".mp4" if "video" in media_type else ".png" + name = filename or Path(url.split("?")[0]).name or f"media_{uuid.uuid4().hex[:8]}{default_ext}" + return await self.save_bytes(data=data, filename=name, media_type=media_type, project_id=project_id) async def get_asset(self, asset_id: str) -> Optional[AssetRecord]: conn = await self.manager.get_connection() diff --git a/backend/tests/test_m7_video.py b/backend/tests/test_m7_video.py new file mode 100644 index 0000000..fb7f3ec --- /dev/null +++ b/backend/tests/test_m7_video.py @@ -0,0 +1,235 @@ +"""Tests for M7 Video Generation: Schemas, Caching, Macro Compilers, and Execution.""" + +import asyncio +from unittest.mock import AsyncMock, patch +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.runners.creative_runner import compute_creative_cache_hash, creative_runner +from app.runners.macro_compiler import ( + build_comfy_img2video_graph, + build_comfy_txt2video_graph, +) +from app.schemas.creative import ( + CreativeActionRequest, + CreativeActionResult, + CreativeActionType, + GenerationProvenance, +) +from app.storage.asset_store import asset_store + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_video_schemas(): + """Verify video action schemas and default parameters.""" + req = CreativeActionRequest( + action=CreativeActionType.TXT2VIDEO, + prompt="A soaring eagle over a snowy mountain range", + fps=16, + num_frames=25, + motion_bucket_id=127, + duration_seconds=3.0, + ) + assert req.action == CreativeActionType.TXT2VIDEO + assert req.fps == 16 + assert req.num_frames == 25 + assert req.motion_bucket_id == 127 + assert req.duration_seconds == 3.0 + + res = CreativeActionResult( + success=True, + task_id="task_123", + video_url="/api/v1/assets/asset_video_1/content", + width=1024, + height=576, + duration_seconds=3.0, + fps=16, + ) + assert res.video_url == "/api/v1/assets/asset_video_1/content" + assert res.image_url is None + assert res.duration_seconds == 3.0 + assert res.fps == 16 + + +def test_video_cache_hashing(): + """Verify deterministic cache hashing incorporates video-specific parameters.""" + req1 = CreativeActionRequest( + action=CreativeActionType.IMG2VIDEO, + prompt="Ocean waves gently breaking on sandy beach", + model="svd_xt.safetensors", + fps=16, + num_frames=25, + motion_bucket_id=127, + duration_seconds=2.5, + ) + req2 = CreativeActionRequest( + action=CreativeActionType.IMG2VIDEO, + prompt="Ocean waves gently breaking on sandy beach", + model="svd_xt.safetensors", + fps=16, + num_frames=25, + motion_bucket_id=127, + duration_seconds=2.5, + ) + # Identical requests produce identical hash + hash1 = compute_creative_cache_hash(req1, input_hash="sha256_input_abc") + hash2 = compute_creative_cache_hash(req2, input_hash="sha256_input_abc") + assert hash1 == hash2 + + # Changing fps changes hash + req_diff_fps = req1.model_copy(update={"fps": 24}) + assert compute_creative_cache_hash(req_diff_fps, input_hash="sha256_input_abc") != hash1 + + # Changing motion bucket changes hash + req_diff_motion = req1.model_copy(update={"motion_bucket_id": 200}) + assert compute_creative_cache_hash(req_diff_motion, input_hash="sha256_input_abc") != hash1 + + # Changing num_frames changes hash + req_diff_frames = req1.model_copy(update={"num_frames": 16}) + assert compute_creative_cache_hash(req_diff_frames, input_hash="sha256_input_abc") != hash1 + + +def test_comfy_img2video_macro_graph(): + """Verify SVD image-to-video macro graph construction.""" + graph = build_comfy_img2video_graph( + image_filename="source_portrait.png", + checkpoint="svd_xt.safetensors", + width=1024, + height=576, + video_frames=25, + fps=16, + motion_bucket_id=127, + seed=42, + ) + + assert "1" in graph + assert graph["1"]["class_type"] == "ImageOnlyCheckpointLoader" + assert graph["1"]["inputs"]["ckpt_name"] == "svd_xt.safetensors" + + assert "2" in graph + assert graph["2"]["class_type"] == "LoadImage" + assert graph["2"]["inputs"]["image"] == "source_portrait.png" + + assert "3" in graph + assert graph["3"]["class_type"] == "SVD_img2vid_Conditioning" + assert graph["3"]["inputs"]["video_frames"] == 25 + assert graph["3"]["inputs"]["fps"] == 16 + assert graph["3"]["inputs"]["motion_bucket_id"] == 127 + + assert "4" in graph + assert graph["4"]["class_type"] == "KSampler" + assert graph["4"]["inputs"]["seed"] == 42 + + assert "6" in graph + assert graph["6"]["class_type"] == "SaveAnimatedWEBP" + assert graph["6"]["inputs"]["fps"] == 16 + + +def test_comfy_txt2video_macro_graph(): + """Verify AnimateDiff text-to-video macro graph construction.""" + graph = build_comfy_txt2video_graph( + prompt="Neon hologram dancer in rain", + negative_prompt="blurry", + checkpoint="v1-5-pruned-emaonly.safetensors", + animatediff_model="mm_sd_v15_v2.ckpt", + width=512, + height=512, + video_frames=16, + fps=8, + seed=100, + ) + + assert "1" in graph + assert graph["1"]["class_type"] == "CheckpointLoaderSimple" + + assert "2" in graph + assert graph["2"]["class_type"] == "AnimateDiffLoaderWithContext" + assert graph["2"]["inputs"]["model_name"] == "mm_sd_v15_v2.ckpt" + + assert "3" in graph + assert graph["3"]["class_type"] == "CLIPTextEncode" + assert graph["3"]["inputs"]["text"] == "Neon hologram dancer in rain" + + assert "5" in graph + assert graph["5"]["class_type"] == "EmptyLatentImage" + assert graph["5"]["inputs"]["batch_size"] == 16 + + assert "6" in graph + assert graph["6"]["class_type"] == "KSampler" + assert graph["6"]["inputs"]["seed"] == 100 + + assert "8" in graph + assert graph["8"]["class_type"] == "SaveAnimatedWEBP" + assert graph["8"]["inputs"]["fps"] == 8 + + +@pytest.mark.asyncio +async def test_cloud_video_execution(client): + """Verify execution of cloud text-to-video via REST API with mocked provider.""" + mock_url = "https://fal.media/files/monkey/sample_generated_video.mp4" + + with patch("app.runners.creative_runner.credentials_manager.get_key", return_value="fake_fal_key"), \ + patch("app.runners.creative_runner._call_fal_ai_video", new=AsyncMock(return_value=mock_url)), \ + patch("app.storage.asset_store.asset_store.save_media_from_url") as mock_save: + + class MockAsset: + id = "mock_video_asset_id_999" + file_path = "mock/path.mp4" + media_type = "video" + filename = "cloud_video.mp4" + + mock_save.return_value = MockAsset() + + payload = { + "action": "txt2video", + "prompt": "Hyperlapse of clouds over Tokyo skyline at dusk", + "engine_id": "cloud", + "fps": 16, + "num_frames": 25, + "duration_seconds": 3.0, + "seed": 9999, + } + + resp = client.post("/api/v1/creative/execute", json=payload) + assert resp.status_code == 200 + data = resp.json() + + assert data["success"] is True + assert data["video_url"] == "/api/v1/assets/mock_video_asset_id_999/content" + assert data["fps"] == 16 + assert data["duration_seconds"] == 3.0 + assert data["provenance"]["action"] == "txt2video" + assert data["provenance"]["fps"] == 16 + + +def test_webui_rejection_for_video(client): + """Verify WebUI rejects video generation requests with an actionable error message.""" + payload = { + "action": "txt2video", + "prompt": "Underwater coral reef with bioluminescent fish", + "engine_id": "managed_webui", + } + resp = client.post("/api/v1/creative/execute", json=payload) + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is False + assert "WebUI engine currently does not support native video generation" in data["error_message"] + + +def test_img2video_missing_source_image(client): + """Verify img2video errors if no input image is provided.""" + payload = { + "action": "img2video", + "prompt": "Animate camera moving forward", + "engine_id": "managed_comfyui", + } + resp = client.post("/api/v1/creative/execute", json=payload) + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is False + assert "Source image required" in data["error_message"] diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index f100a06..ae927ff 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -17,6 +17,7 @@ import { FlowCanvas } from './components/canvas/FlowCanvas'; import { CreationDock } from './components/canvas/CreationDock'; import { InpaintModal } from './components/canvas/InpaintModal'; import { UpscaleModal } from './components/canvas/UpscaleModal'; +import { VideoModal } from './components/canvas/VideoModal'; import { CloudSettingsModal } from './components/cloud/CloudSettingsModal'; import { EnvironmentManagerModal } from './components/manager/EnvironmentManagerModal'; import { useCanvasStore } from './stores/useCanvasStore'; @@ -238,6 +239,7 @@ export default function App() { {/* Contextual Action Modals */} + setShowCloudModal(false)} /> setShowManagerModal(false)} /> diff --git a/frontend/src/components/canvas/CreationDock.tsx b/frontend/src/components/canvas/CreationDock.tsx index fc6ce4a..a42000c 100644 --- a/frontend/src/components/canvas/CreationDock.tsx +++ b/frontend/src/components/canvas/CreationDock.tsx @@ -1,4 +1,4 @@ -import { Dices, Sparkles, X, Loader2 } from 'lucide-react'; +import { Dices, Sparkles, Video, X, Loader2 } from 'lucide-react'; import { useCreativeStore } from '../../stores/useCreativeStore'; export const CreationDock = () => { @@ -76,23 +76,35 @@ export const CreationDock = () => { /> - +
+ + +
{/* Controls Toolbar: Aspect Ratio, Engine, Seed */} diff --git a/frontend/src/components/canvas/ImageCardNode.tsx b/frontend/src/components/canvas/ImageCardNode.tsx index 66cbb9f..3dd5c7a 100644 --- a/frontend/src/components/canvas/ImageCardNode.tsx +++ b/frontend/src/components/canvas/ImageCardNode.tsx @@ -1,20 +1,23 @@ import { memo } from 'react'; import { NodeProps } from '@xyflow/react'; -import { Download, Maximize2, Paintbrush, Sparkles, Trash2 } from 'lucide-react'; +import { Download, Maximize2, Paintbrush, Sparkles, Trash2, Video } from 'lucide-react'; import { ImageCardData } from '../../types/creative'; import { useCreativeStore } from '../../stores/useCreativeStore'; import { useCanvasStore } from '../../stores/useCanvasStore'; export const ImageCardNode = memo(({ id, data, selected }: NodeProps) => { const cardData = data as unknown as ImageCardData; - const { setReferenceImage, openInpaint, openUpscale } = useCreativeStore(); + const { setReferenceImage, openInpaint, openUpscale, openImg2Video } = useCreativeStore(); const { nodes } = useCanvasStore(); + const isVideo = cardData.mediaType === 'video' || Boolean(cardData.videoUrl); + const handleExport = (e: React.MouseEvent) => { e.stopPropagation(); const link = document.createElement('a'); - link.href = cardData.imageUrl; - link.download = `${cardData.label || 'berry_image'}.png`; + link.href = cardData.videoUrl || cardData.imageUrl; + const ext = isVideo ? 'mp4' : 'png'; + link.download = `${cardData.label || 'berry_asset'}.${ext}`; document.body.appendChild(link); link.click(); document.body.removeChild(link); @@ -42,6 +45,11 @@ export const ImageCardNode = memo(({ id, data, selected }: NodeProps) => { openUpscale(cardData); }; + const handleAnimate = (e: React.MouseEvent) => { + e.stopPropagation(); + openImg2Video(cardData); + }; + const p = cardData.provenance; return ( @@ -53,27 +61,38 @@ export const ImageCardNode = memo(({ id, data, selected }: NodeProps) => { > {/* Floating Action Bar */}
- - - + {!isVideo && ( + <> + + + + + + )}
- {/* Image Preview Container */} + {/* Media Preview Container */}
- {cardData.label + {isVideo ? ( +
@@ -117,11 +148,16 @@ export const ImageCardNode = memo(({ id, data, selected }: NodeProps) => { {p.engine_id.replace('managed_', '')} - seed: {p.seed} + {p.action} - steps: {p.steps} + seed: {p.seed} + {p.fps && ( + + {p.fps} fps + + )} ) : (
Imported asset
@@ -132,3 +168,4 @@ export const ImageCardNode = memo(({ id, data, selected }: NodeProps) => { }); ImageCardNode.displayName = 'ImageCardNode'; + diff --git a/frontend/src/components/canvas/VideoModal.tsx b/frontend/src/components/canvas/VideoModal.tsx new file mode 100644 index 0000000..76aeac6 --- /dev/null +++ b/frontend/src/components/canvas/VideoModal.tsx @@ -0,0 +1,204 @@ +import { useState } from 'react'; +import { Clapperboard, Loader2, Video, X } from 'lucide-react'; +import { useCreativeStore } from '../../stores/useCreativeStore'; + +export const VideoModal = () => { + const { + videoModalOpen, + referenceImage, + closeModals, + executeCreativeAction, + isGenerating, + fps, + setFps, + numFrames, + setNumFrames, + motionBucketId, + setMotionBucketId, + engineId, + setEngineId, + } = useCreativeStore(); + + const [prompt, setPrompt] = useState( + 'Smooth camera pan, vivid natural motion, cinematic lighting' + ); + + if (!videoModalOpen || !referenceImage) return null; + + const currentW = referenceImage.width || 512; + const currentH = referenceImage.height || 512; + + const handleGenerateVideo = async () => { + if (!referenceImage?.assetId || isGenerating) return; + + const res = await executeCreativeAction({ + action: 'img2video', + prompt, + input_image_id: referenceImage.assetId, + fps, + num_frames: numFrames, + motion_bucket_id: motionBucketId, + width: currentW, + height: currentH, + }); + + if (res && res.success) { + closeModals(); + } + }; + + return ( +
+
+ {/* Header */} +
+
+ +

Animate Image (Image-to-Video)

+
+ +
+ + {/* Content Body */} +
+ {/* Source Thumbnail Preview */} +
+ Source +
+ {referenceImage.label || 'Source Asset'} + Resolution: {currentW} × {currentH} px + Output: {numFrames} frames @ {fps} fps +
+
+ + {/* Prompt */} +
+ +