From cc046f80ac6f157df693893349a5b0ba253d24c7 Mon Sep 17 00:00:00 2001 From: Fsocietyhhh <1211904451@qq.com> Date: Fri, 3 Jul 2026 13:03:15 -0700 Subject: [PATCH] feat(proxy): add video/speech/music/sound-effects media endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sidecar only exposed /v1/images/generations, so OpenAI-compatible clients (LiteLLM) could not reach the gateway's video/audio models — e.g. a call to xai/grok-imagine-video 404'd at the proxy even though the gateway supports it. - Add POST /v1/videos/generations, /v1/audio/speech, /v1/audio/generations, and /v1/audio/sound-effects, mirroring the existing image route. - Adapter: video/music/speech/sound_effect async wrappers with Base (dedicated VideoClient/MusicClient/SpeechClient) vs Solana (unified SolanaLLMClient) dispatch; sync SDK clients run in the shared thread pool. Requires blockrun-llm with Solana media support (SolanaLLMClient.video etc.). Validated live end-to-end on Solana mainnet via the adapter path. --- blockrun_litellm/_adapter.py | 159 +++++++++++++++++++++++++++++++++++ blockrun_litellm/proxy.py | 151 +++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) diff --git a/blockrun_litellm/_adapter.py b/blockrun_litellm/_adapter.py index 0425146..32b82cd 100644 --- a/blockrun_litellm/_adapter.py +++ b/blockrun_litellm/_adapter.py @@ -478,6 +478,158 @@ async def image_generation_async( return response.model_dump(exclude_none=True) +# --------------------------------------------------------------------------- +# Video / music / speech generation (Base dedicated clients vs Solana unified) +# --------------------------------------------------------------------------- +# On Base each medium has its own SDK client (VideoClient/MusicClient/ +# SpeechClient). On Solana every medium is a method on the one SolanaLLMClient +# (which get_image_client already builds + caches). All of these clients are +# sync-only, so async callers run them in the shared _image_executor thread +# pool — same pattern as image_generation_async. + +_media_clients: Dict[str, Any] = {} + + +def _get_base_media_client(base_cls: Any, api_url: Optional[str], private_key: Optional[str]) -> Any: + """Cache + return a Base dedicated media client (VideoClient/MusicClient/…).""" + key = f"{base_cls.__name__}::{_client_key(api_url, private_key)}" + with _lock: + client = _media_clients.get(key) + if client is None: + client = base_cls(private_key=private_key, api_url=api_url) + _media_clients[key] = client + return client + + +def _is_solana_client(client: Any) -> bool: + return _HAS_SOLANA and SolanaLLMClient is not None and isinstance(client, SolanaLLMClient) + + +def get_video_client(api_url: Optional[str] = None, private_key: Optional[str] = None) -> Any: + """VideoClient (Base) or the unified SolanaLLMClient (Solana).""" + if _is_solana_url(api_url): + return get_image_client(api_url=api_url, private_key=private_key) + from blockrun_llm import VideoClient + + return _get_base_media_client(VideoClient, api_url, private_key) + + +def get_music_client(api_url: Optional[str] = None, private_key: Optional[str] = None) -> Any: + """MusicClient (Base) or the unified SolanaLLMClient (Solana).""" + if _is_solana_url(api_url): + return get_image_client(api_url=api_url, private_key=private_key) + from blockrun_llm import MusicClient + + return _get_base_media_client(MusicClient, api_url, private_key) + + +def get_speech_client(api_url: Optional[str] = None, private_key: Optional[str] = None) -> Any: + """SpeechClient (Base) or the unified SolanaLLMClient (Solana). Serves both + TTS (speech) and sound-effects.""" + if _is_solana_url(api_url): + return get_image_client(api_url=api_url, private_key=private_key) + from blockrun_llm import SpeechClient + + return _get_base_media_client(SpeechClient, api_url, private_key) + + +def _run_media(func: Any) -> Any: + """Run a sync SDK media call in the shared executor.""" + loop = asyncio.get_event_loop() + return loop.run_in_executor(_image_executor, func) + + +async def video_generation_async( + prompt: str, + *, + model: Optional[str] = None, + api_url: Optional[str] = None, + private_key: Optional[str] = None, + **params: Any, +) -> Dict[str, Any]: + """Generate a video. Extra kwargs (image_url, duration_seconds, resolution, + aspect_ratio, generate_audio, seed, reference_image_urls, real_face_asset_id, + last_frame_url, watermark, return_last_frame, budget_seconds) forward to the + SDK. ``timeout`` is only honored on Solana (Base VideoClient has no such arg).""" + client = get_video_client(api_url=api_url, private_key=private_key) + params = {k: v for k, v in params.items() if v is not None} + if _is_solana_client(client): + response = await _run_media(lambda: client.video(prompt, model=model, **params)) + else: + params.pop("timeout", None) # Base VideoClient.generate has no timeout kwarg + response = await _run_media(lambda: client.generate(prompt, model=model, **params)) + return response.model_dump(exclude_none=True) + + +async def music_generation_async( + prompt: str, + *, + model: Optional[str] = None, + instrumental: bool = True, + lyrics: Optional[str] = None, + api_url: Optional[str] = None, + private_key: Optional[str] = None, +) -> Dict[str, Any]: + """Generate a music track.""" + client = get_music_client(api_url=api_url, private_key=private_key) + call = ( + (lambda: client.music(prompt, model=model, instrumental=instrumental, lyrics=lyrics)) + if _is_solana_client(client) + else ( + lambda: client.generate(prompt, model=model, instrumental=instrumental, lyrics=lyrics) + ) + ) + response = await _run_media(call) + return response.model_dump(exclude_none=True) + + +async def speech_generation_async( + input: str, + *, + model: Optional[str] = None, + voice: Optional[str] = None, + response_format: Optional[str] = None, + speed: Optional[float] = None, + api_url: Optional[str] = None, + private_key: Optional[str] = None, +) -> Dict[str, Any]: + """Synthesize speech (TTS).""" + client = get_speech_client(api_url=api_url, private_key=private_key) + kw = {"model": model, "voice": voice, "response_format": response_format, "speed": speed} + kw = {k: v for k, v in kw.items() if v is not None} + call = ( + (lambda: client.speech(input, **kw)) + if _is_solana_client(client) + else (lambda: client.generate(input, **kw)) + ) + response = await _run_media(call) + return response.model_dump(exclude_none=True) + + +async def sound_effect_async( + text: str, + *, + model: Optional[str] = None, + duration_seconds: Optional[float] = None, + prompt_influence: Optional[float] = None, + response_format: Optional[str] = None, + api_url: Optional[str] = None, + private_key: Optional[str] = None, +) -> Dict[str, Any]: + """Generate a cinematic sound effect.""" + client = get_speech_client(api_url=api_url, private_key=private_key) + kw = { + "model": model, + "duration_seconds": duration_seconds, + "prompt_influence": prompt_influence, + "response_format": response_format, + } + kw = {k: v for k, v in kw.items() if v is not None} + # Both SolanaLLMClient and SpeechClient expose .sound_effect with the same shape. + response = await _run_media(lambda: client.sound_effect(text, **kw)) + return response.model_dump(exclude_none=True) + + __all__ = [ "chat_completion_sync", "chat_completion_async", @@ -488,6 +640,13 @@ async def image_generation_async( "get_image_client", "image_generation_sync", "image_generation_async", + "get_video_client", + "get_music_client", + "get_speech_client", + "video_generation_async", + "music_generation_async", + "speech_generation_async", + "sound_effect_async", "APIError", "PaymentError", ] diff --git a/blockrun_litellm/proxy.py b/blockrun_litellm/proxy.py index 8a02434..6692d36 100644 --- a/blockrun_litellm/proxy.py +++ b/blockrun_litellm/proxy.py @@ -12,6 +12,11 @@ Anthropic SDK); verbatim x402-signed passthrough — tools/thinking preserved - ``POST /v1/messages/count_tokens`` — Anthropic token counting (passthrough) - ``POST /v1/images/generations`` — OpenAI Image Generations (DALL-E compatible) +- ``POST /v1/videos/generations`` — video generation (xai/grok-imagine-video, + Seedance); async submit+poll, settles only on completion +- ``POST /v1/audio/speech`` — OpenAI-compatible TTS (ElevenLabs voices) +- ``POST /v1/audio/generations`` — music generation (minimax/music-2.5+) +- ``POST /v1/audio/sound-effects`` — cinematic sound effects - ``GET /v1/models`` — passthrough to BlockRun's chat catalog - ``GET /healthz`` — liveness probe (no upstream call) @@ -563,6 +568,152 @@ async def image_generations(request: Request) -> Any: return result +# Optional video params forwarded verbatim to the SDK when present. +_VIDEO_PARAM_KEYS = ( + "image_url", + "last_frame_url", + "reference_image_urls", + "real_face_asset_id", + "duration_seconds", + "aspect_ratio", + "resolution", + "generate_audio", + "seed", + "watermark", + "return_last_frame", + "budget_seconds", + "timeout", +) + + +@app.post("/v1/videos/generations", dependencies=[Depends(_require_token)]) +async def video_generations(request: Request) -> Any: + """Generate a video (default model xai/grok-imagine-video). Submits an + async job and blocks until the clip is ready (typ. 60-180s); the SDK + polls and only settles on completion, so a timeout costs nothing.""" + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + prompt = body.get("prompt") + if not prompt: + raise HTTPException(400, "`prompt` is required") + + params = {k: body[k] for k in _VIDEO_PARAM_KEYS if body.get(k) is not None} + + async with _get_semaphore(): + try: + result = await _adapter.video_generation_async( + prompt=prompt, + model=body.get("model"), + **params, + ) + except ValueError as exc: + # e.g. mutually-exclusive image_url + real_face_asset_id + raise HTTPException(400, str(exc)) + except PaymentError as exc: + return JSONResponse(status_code=402, content=_payment_error_payload(exc)) + except APIError as exc: + status = exc.status_code if 400 <= getattr(exc, "status_code", 0) < 600 else 502 + return JSONResponse(status_code=status, content={"error": str(exc)}) + + return result + + +@app.post("/v1/audio/speech", dependencies=[Depends(_require_token)]) +async def audio_speech(request: Request) -> Any: + """Synthesize speech (OpenAI-compatible TTS). Accepts ``input`` (or + ``prompt``/``text``), ``model``, ``voice``, ``response_format``, ``speed``.""" + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + text = body.get("input") or body.get("prompt") or body.get("text") + if not text: + raise HTTPException(400, "`input` is required") + + async with _get_semaphore(): + try: + result = await _adapter.speech_generation_async( + input=text, + model=body.get("model"), + voice=body.get("voice"), + response_format=body.get("response_format"), + speed=body.get("speed"), + ) + except PaymentError as exc: + return JSONResponse(status_code=402, content=_payment_error_payload(exc)) + except APIError as exc: + status = exc.status_code if 400 <= getattr(exc, "status_code", 0) < 600 else 502 + return JSONResponse(status_code=status, content={"error": str(exc)}) + + return result + + +@app.post("/v1/audio/generations", dependencies=[Depends(_require_token)]) +async def audio_generations(request: Request) -> Any: + """Generate a music track (default model minimax/music-2.5+). Takes 1-3 + min; returns a CDN URL valid ~24h.""" + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + prompt = body.get("prompt") + if not prompt: + raise HTTPException(400, "`prompt` is required") + + async with _get_semaphore(): + try: + result = await _adapter.music_generation_async( + prompt=prompt, + model=body.get("model"), + instrumental=body.get("instrumental", True), + lyrics=body.get("lyrics"), + ) + except PaymentError as exc: + return JSONResponse(status_code=402, content=_payment_error_payload(exc)) + except APIError as exc: + status = exc.status_code if 400 <= getattr(exc, "status_code", 0) < 600 else 502 + return JSONResponse(status_code=status, content={"error": str(exc)}) + + return result + + +@app.post("/v1/audio/sound-effects", dependencies=[Depends(_require_token)]) +async def audio_sound_effects(request: Request) -> Any: + """Generate a cinematic sound effect (default elevenlabs/sound-effects). + Accepts ``text`` (or ``prompt``), ``duration_seconds``, ``prompt_influence``, + ``response_format``.""" + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + text = body.get("text") or body.get("prompt") + if not text: + raise HTTPException(400, "`text` is required") + + async with _get_semaphore(): + try: + result = await _adapter.sound_effect_async( + text=text, + model=body.get("model"), + duration_seconds=body.get("duration_seconds"), + prompt_influence=body.get("prompt_influence"), + response_format=body.get("response_format"), + ) + except PaymentError as exc: + return JSONResponse(status_code=402, content=_payment_error_payload(exc)) + except APIError as exc: + status = exc.status_code if 400 <= getattr(exc, "status_code", 0) < 600 else 502 + return JSONResponse(status_code=status, content={"error": str(exc)}) + + return result + + # --------------------------------------------------------------------------- # OpenAI Responses API bridge (/v1/responses) # ---------------------------------------------------------------------------