From 6df81e834271c335c9fce999a6df5b7ba6371411 Mon Sep 17 00:00:00 2001 From: Caleb Hunter <40677799+hello-caleb@users.noreply.github.com> Date: Mon, 2 Feb 2026 21:25:32 -0500 Subject: [PATCH 1/4] feat: Phase 2 Real-time (WebSocket Hook, Light Mode UI) --- public/worklets/downsampler.js | 21 +++++ src/components/Dashboard.tsx | 137 +++++++++++++++++++-------- src/hooks/use-gemini-client.ts | 167 +++++++++++++++++++++++++++++++++ src/types/gemini.ts | 47 ++++++++++ 4 files changed, 334 insertions(+), 38 deletions(-) create mode 100644 public/worklets/downsampler.js create mode 100644 src/hooks/use-gemini-client.ts create mode 100644 src/types/gemini.ts diff --git a/public/worklets/downsampler.js b/public/worklets/downsampler.js new file mode 100644 index 0000000..440018d --- /dev/null +++ b/public/worklets/downsampler.js @@ -0,0 +1,21 @@ +class PCMProcessor extends AudioWorkletProcessor { + process(inputs, outputs, parameters) { + const input = inputs[0]; + if (!input || input.length === 0) return true; + + const channelData = input[0]; // Mono + if (!channelData) return true; + + // Convert Float32 (-1.0 to 1.0) to Int16 + const pcm16 = new Int16Array(channelData.length); + for (let i = 0; i < channelData.length; i++) { + const s = Math.max(-1, Math.min(1, channelData[i])); + pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + + this.port.postMessage(pcm16); + return true; + } +} + +registerProcessor('pcm-processor', PCMProcessor); diff --git a/src/components/Dashboard.tsx b/src/components/Dashboard.tsx index fda983f..a3c3de8 100644 --- a/src/components/Dashboard.tsx +++ b/src/components/Dashboard.tsx @@ -1,75 +1,136 @@ 'use client'; -import React from 'react'; -import { Video, Mic, Globe, Info } from 'lucide-react'; +import React, { useState, useEffect, useRef } from 'react'; +import { Video, Mic, Globe, Info, Power, Mic as MicIcon, MicOff } from 'lucide-react'; +import { useGeminiClient } from '@/hooks/use-gemini-client'; export default function Dashboard() { + const [apiKey, setApiKey] = useState(process.env.NEXT_PUBLIC_GEMINI_API_KEY || ""); + const { isConnected, isRecording, transcript, connect, disconnect, startAudio } = useGeminiClient({ apiKey }); + const scrollRef = useRef(null); + + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = scrollRef.current.scrollHeight; + } + }, [transcript]); + + const handleToggleConnect = () => { + if (isConnected) { + disconnect(); + } else { + connect(); + } + }; + return ( -
+
{/* Header */} -
-

HealthBridge

+
+

HealthBridge

-
-
- Live +
+
+ {isConnected ? 'Connected' : 'Disconnected'}
-
{/* Main Grid */} -
+
{/* Video Feeds Area (Left/Center) */} -
+
{/* Video Grid */} -
+
{/* Doctor Feed */} -
-
-
{/* Side Panel: Jargon & Insights (Right) */} -
diff --git a/src/hooks/use-gemini-client.ts b/src/hooks/use-gemini-client.ts index b93a04b..e5e9c42 100644 --- a/src/hooks/use-gemini-client.ts +++ b/src/hooks/use-gemini-client.ts @@ -2,25 +2,38 @@ import { useState, useRef, useCallback, useEffect } from 'react'; import { GeminiClientEvent, GeminiServerEvent } from '@/types/gemini'; +import { SYSTEM_INSTRUCTION } from '@/lib/prompts'; +import { detectMedicalTerms, MedicalTerm } from '@/lib/gemini-3-service'; interface UseGeminiClientProps { apiKey?: string; url?: string; } -export function useGeminiClient({ apiKey, url = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent" }: UseGeminiClientProps) { +export type { MedicalTerm }; + +export function useGeminiClient({ apiKey, url = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" }: UseGeminiClientProps) { const [isConnected, setIsConnected] = useState(false); const [isRecording, setIsRecording] = useState(false); const [transcript, setTranscript] = useState(""); + const [medicalTerms, setMedicalTerms] = useState([]); + const [error, setError] = useState(null); const wsRef = useRef(null); const audioContextRef = useRef(null); const streamRef = useRef(null); const workletNodeRef = useRef(null); + // Intelligence Loop Ref + const processingRef = useRef<{ lastLength: number; isProcessing: boolean }>({ + lastLength: 0, + isProcessing: false + }); + const connect = useCallback(async () => { + setError(null); if (!apiKey) { - console.error("No API key provided"); + setError("No API key provided"); return; } @@ -31,51 +44,112 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google ws.onopen = () => { console.log("WebSocket Connected"); setIsConnected(true); - // Send setup message + // Send setup message - Using gemini-2.0-flash-exp (v1alpha required for audio streaming) const setupMsg: GeminiClientEvent = { setup: { - model: "models/gemini-2.0-flash-exp", // Or appropriate model + model: "models/gemini-2.5-flash-native-audio-preview-12-2025", generationConfig: { - responseModalities: ["TEXT"], // Start with TEXT for simplicity, can add AUDIO + responseModalities: ["AUDIO"], + }, + systemInstruction: { + parts: [{ text: SYSTEM_INSTRUCTION }] }, }, }; ws.send(JSON.stringify(setupMsg)); }; - ws.onmessage = (event) => { + ws.onmessage = async (event) => { try { - const msg = JSON.parse(event.data) as GeminiServerEvent; + // Live API sends Blob data, not text + let data: string; + if (event.data instanceof Blob) { + data = await event.data.text(); + } else { + data = event.data; + } - if ('serverContent' in msg) { - const parts = msg.serverContent.modelTurn?.parts || []; - const text = parts.map(p => 'text' in p ? p.text : '').join(''); - if (text) { - setTranscript(prev => prev + text); + const msg = JSON.parse(data) as GeminiServerEvent; + console.log("Received message:", msg); + + // Handle Text Content from model turn + if ('serverContent' in msg && msg.serverContent?.modelTurn?.parts) { + const parts = msg.serverContent.modelTurn.parts; + for (const part of parts) { + if ('text' in part && part.text) { + console.log("Transcript text:", part.text); + setTranscript(prev => prev + part.text); + } } } + + // Note: WebSocket tool calls removed. Jargon detection is now handled via the Effect below. + } catch (e) { - console.error("Error parsing message", e); + console.error("Error parsing message", e, event.data); } }; - ws.onclose = () => { - console.log("WebSocket Closed"); + ws.onclose = (event) => { + console.log("WebSocket Closed", event.code, event.reason); setIsConnected(false); setIsRecording(false); + if (event.code !== 1000 && event.code !== 1005) { + setError(`Disconnected: Code ${event.code} ${event.reason || ""}`); + } }; ws.onerror = (err) => { console.error("WebSocket Error", err); + setError("WebSocket Error Check console for details."); }; wsRef.current = ws; - } catch (error) { + } catch (error: any) { console.error("Connection failed", error); + setError(error.toString()); } }, [apiKey, url]); + // Intelligence Loop: Poll Gemini 3 for medical terms + useEffect(() => { + const intervalId = setInterval(async () => { + if (!transcript || processingRef.current.isProcessing) return; + + const currentLength = transcript.length; + const lastLength = processingRef.current.lastLength; + + // Only process if we have significant new content (~30 chars) + if (currentLength - lastLength > 30) { + processingRef.current.isProcessing = true; + + const newSegment = transcript.slice(lastLength); + console.log(" analyzing segment:", newSegment); + + try { + const terms = await detectMedicalTerms(newSegment); + if (terms.length > 0) { + // Add timestamps manually since REST API doesn't give them + const timestampedTerms = terms.map(t => ({ + ...t, + timestamp: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) + })); + setMedicalTerms(prev => [...timestampedTerms, ...prev]); + } + // Update pointer only after successful process (or even if empty, to avoid stuck loop) + processingRef.current.lastLength = currentLength; + } catch (err) { + console.error("Jargon detection failed", err); + } finally { + processingRef.current.isProcessing = false; + } + } + }, 3000); // Check every 3 seconds + + return () => clearInterval(intervalId); + }, [transcript]); + const startAudio = useCallback(async () => { if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; @@ -83,7 +157,12 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, channelCount: 1 } }); const audioContext = new AudioContext({ sampleRate: 16000 }); - await audioContext.audioWorklet.addModule('/worklets/downsampler.js'); + try { + await audioContext.audioWorklet.addModule('/worklets/downsampler.js'); + } catch (e) { + setError("Failed to load audio worklet. Check /worklets/downsampler.js"); + return; + } const source = audioContext.createMediaStreamSource(stream); const worklet = new AudioWorkletNode(audioContext, 'pcm-processor'); @@ -93,11 +172,6 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google const pcm16 = event.data as Int16Array; // Convert to base64 - // Note: For large buffers effectively, simple btoa on string might stack overflow, - // but for worklet chunks (typically 128 frames) it's small. - // However, standard Worklet size is 128 frames ~ 8ms, which is very frequent. - // We buffer or send immediately. Let's send immediately for low latency. - const buffer = new Uint8Array(pcm16.buffer); let binary = ''; const len = buffer.byteLength; @@ -119,8 +193,6 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google }; source.connect(worklet); - // Worklet doesn't need to connect to destination if we don't want to hear ourselves. - // source.connect(worklet).connect(audioContext.destination); streamRef.current = stream; audioContextRef.current = audioContext; @@ -128,8 +200,9 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google setIsRecording(true); console.log("Audio started"); - } catch (err) { + } catch (err: any) { console.error("Error starting audio", err); + setError(`Audio Error: ${err.message}`); } }, []); @@ -148,6 +221,8 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google } setIsConnected(false); setIsRecording(false); + // Reset intelligence loop + processingRef.current = { lastLength: 0, isProcessing: false }; }, []); useEffect(() => { @@ -160,6 +235,8 @@ export function useGeminiClient({ apiKey, url = "wss://generativelanguage.google isConnected, isRecording, transcript, + medicalTerms, + error, connect, disconnect, startAudio diff --git a/src/lib/gemini-3-service.ts b/src/lib/gemini-3-service.ts new file mode 100644 index 0000000..3d14fc8 --- /dev/null +++ b/src/lib/gemini-3-service.ts @@ -0,0 +1,53 @@ +import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai"; + +const API_KEY = process.env.NEXT_PUBLIC_GEMINI_API_KEY || ""; +const genAI = new GoogleGenerativeAI(API_KEY); + +const model = genAI.getGenerativeModel({ + model: "gemini-3-flash-preview", + generationConfig: { + responseMimeType: "application/json", + responseSchema: { + type: SchemaType.ARRAY, + items: { + type: SchemaType.OBJECT, + properties: { + term: { type: SchemaType.STRING }, + definition: { type: SchemaType.STRING }, + }, + required: ["term", "definition"], + }, + }, + }, + // @ts-ignore - dangerouslyAllowBrowser is valid but missing from type definitions in 0.24.1 +}, { dangerouslyAllowBrowser: true }); + +export type MedicalTerm = { + term: string; + definition: string; +}; + +export async function detectMedicalTerms(text: string): Promise { + if (!text || text.trim().length < 10) return []; + + const prompt = ` + You are a medical communication assistant. + Analyze the following transcript segment for complex medical terms that a layperson might not understand (Grade 6 level). + If found, provide a simple, short definition. + + Transcript: + "${text}" + + Return ONLY a JSON array of objects. + `; + + try { + const result = await model.generateContent(prompt); + const response = result.response; + const json = response.text(); + return JSON.parse(json) as MedicalTerm[]; + } catch (error) { + console.error("Gemini 3 Jargon Detection Error:", error); + return []; + } +} diff --git a/src/lib/prompts.ts b/src/lib/prompts.ts new file mode 100644 index 0000000..e241498 --- /dev/null +++ b/src/lib/prompts.ts @@ -0,0 +1,20 @@ +export const SYSTEM_INSTRUCTION = ` +You are HealthBridge, a medical communication assistant. +Your goal is to facilitate understanding between a Doctor and a Patient. + +**CORE RESPONSIBILITIES**: +1. **Transcribe**: Provide real-time, accurate transcription of the conversation. +2. **Simplify (Jargon Detection)**: + - Listen for complex medical terms (e.g., "Hypertension", "Myocardial Infarction", "Titrate"). + - When a term is used, you MUST emit a JSON tool call or structured event defining it in simple Grade 6 language. + - Format: { "term": "Term", "definition": "Simple definition", "timestamp": "HH:MM" } + +**TONE**: +- Calm, professional, supportive. +- Do NOT interrupt the speakers unless clarifying a critical misunderstanding. +- Keep definitions brief (1-2 sentences). + +**SAFETY**: +- Do not provide medical advice. +- If the user asks for a diagnosis, state: "I am a communication aid, please consult your doctor." +`; diff --git a/src/types/gemini.ts b/src/types/gemini.ts index e6d2595..dc70af3 100644 --- a/src/types/gemini.ts +++ b/src/types/gemini.ts @@ -7,6 +7,14 @@ export type GeminiContent = { parts: GeminiPart[]; }; +export type GeminiTool = { + functionDeclarations: { + name: string; + description: string; + parameters?: any; + }[]; +}; + export type GeminiServerEvent = | { serverContent: { modelTurn: { parts: GeminiPart[] }; turnComplete?: boolean } } | { toolCall: { functionCalls: { name: string; args: any }[] } } @@ -26,6 +34,10 @@ export type GeminiClientEvent = }; }; }; + systemInstruction?: { + parts: GeminiPart[]; + }; + tools?: GeminiTool[]; }; } | { @@ -44,4 +56,12 @@ export type GeminiClientEvent = data: string; // Base64 }[]; }; + } + | { + toolResponse: { + functionResponses: { + name: string; + response: any; + }[]; + } }; From 9d7d2558585f42c612c0abb9780fbc08dd6ac19b Mon Sep 17 00:00:00 2001 From: Caleb Hunter <40677799+hello-caleb@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:25:12 -0500 Subject: [PATCH 3/4] feat: add video preview, caption size toggle, and 503 retry logic - Add use-video-stream.ts hook for local camera preview - Add camera toggle button to Patient Feed - Add caption size toggle (A/A+/A++) in Live Captions - Add MedicalTerm timestamp field - Add exponential backoff retry for Gemini 3 API 503 errors --- src/components/Dashboard.tsx | 57 +++++++++++++++++++++-------- src/hooks/use-video-stream.ts | 67 +++++++++++++++++++++++++++++++++++ src/lib/gemini-3-service.ts | 29 ++++++++++----- 3 files changed, 131 insertions(+), 22 deletions(-) create mode 100644 src/hooks/use-video-stream.ts diff --git a/src/components/Dashboard.tsx b/src/components/Dashboard.tsx index c8f36ab..194e602 100644 --- a/src/components/Dashboard.tsx +++ b/src/components/Dashboard.tsx @@ -3,11 +3,14 @@ import React, { useState, useEffect, useRef } from 'react'; import { Video, Mic, Globe, Info, Power, Mic as MicIcon, MicOff } from 'lucide-react'; import { useGeminiClient, MedicalTerm } from '@/hooks/use-gemini-client'; +import { useVideoStream } from '@/hooks/use-video-stream'; export default function Dashboard() { const [apiKey, setApiKey] = useState(process.env.NEXT_PUBLIC_GEMINI_API_KEY || ""); const { isConnected, isRecording, transcript, medicalTerms, error, connect, disconnect, startAudio } = useGeminiClient({ apiKey }); + const { videoRef, isActive: isVideoActive, startVideo, stopVideo } = useVideoStream(); const scrollRef = useRef(null); + const [captionSize, setCaptionSize] = useState<'sm' | 'md' | 'lg'>('md'); useEffect(() => { if (scrollRef.current) { @@ -93,23 +96,43 @@ export default function Dashboard() { {/* Patient Feed */}
-
- - Patient Feed -

Local Microphone

-
+ {/* Video Preview or Placeholder */} + {isVideoActive ? ( +
@@ -123,9 +146,15 @@ export default function Dashboard() { Live Captions {isRecording && LISTENING} +
-

+

{transcript || Captions will appear here...}

diff --git a/src/hooks/use-video-stream.ts b/src/hooks/use-video-stream.ts new file mode 100644 index 0000000..838a32f --- /dev/null +++ b/src/hooks/use-video-stream.ts @@ -0,0 +1,67 @@ +'use client'; + +import { useState, useRef, useCallback, useEffect } from 'react'; + +export function useVideoStream() { + const [isActive, setIsActive] = useState(false); + const [error, setError] = useState(null); + const videoRef = useRef(null); + const streamRef = useRef(null); + + const startVideo = useCallback(async () => { + try { + setError(null); + const stream = await navigator.mediaDevices.getUserMedia({ + video: { + width: { ideal: 1280 }, + height: { ideal: 720 }, + facingMode: 'user' + } + }); + + streamRef.current = stream; + setIsActive(true); // This triggers the video element to render + } catch (err: any) { + console.error('Failed to start video:', err); + setError(err.message || 'Failed to access camera'); + } + }, []); + + // Effect to attach stream to video element once it mounts + useEffect(() => { + if (isActive && streamRef.current && videoRef.current) { + videoRef.current.srcObject = streamRef.current; + videoRef.current.play().catch(console.error); + } + }, [isActive]); + + const stopVideo = useCallback(() => { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + } + + if (videoRef.current) { + videoRef.current.srcObject = null; + } + + setIsActive(false); + }, []); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + } + }; + }, []); + + return { + videoRef, + isActive, + error, + startVideo, + stopVideo + }; +} diff --git a/src/lib/gemini-3-service.ts b/src/lib/gemini-3-service.ts index 3d14fc8..585aa46 100644 --- a/src/lib/gemini-3-service.ts +++ b/src/lib/gemini-3-service.ts @@ -25,6 +25,7 @@ const model = genAI.getGenerativeModel({ export type MedicalTerm = { term: string; definition: string; + timestamp?: string; }; export async function detectMedicalTerms(text: string): Promise { @@ -41,13 +42,25 @@ export async function detectMedicalTerms(text: string): Promise { Return ONLY a JSON array of objects. `; - try { - const result = await model.generateContent(prompt); - const response = result.response; - const json = response.text(); - return JSON.parse(json) as MedicalTerm[]; - } catch (error) { - console.error("Gemini 3 Jargon Detection Error:", error); - return []; + // Retry with exponential backoff for 503 errors + const maxRetries = 3; + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + const result = await model.generateContent(prompt); + const response = result.response; + const json = response.text(); + return JSON.parse(json) as MedicalTerm[]; + } catch (error: any) { + const is503 = error?.message?.includes('503') || error?.message?.includes('overloaded'); + if (is503 && attempt < maxRetries - 1) { + // Wait before retry: 1s, 2s, 4s + await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt))); + console.log(`Gemini 3 API overloaded, retrying (${attempt + 1}/${maxRetries})...`); + continue; + } + console.error("Gemini 3 Jargon Detection Error:", error); + return []; + } } + return []; } From 4838f15f2769c84aeb763c2aa9cef5f8fa1e35c0 Mon Sep 17 00:00:00 2001 From: Caleb Hunter <40677799+hello-caleb@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:39:39 -0500 Subject: [PATCH 4/4] feat: integrate LiveKit cloud video for Doctor feed - Add livekit-client, livekit-server-sdk, @livekit/components-react - Create /api/livekit-token API route for JWT generation - Create use-livekit.ts hook for token management - Create DoctorVideoRoom component for remote video display - Create /doctor page for Doctor to join with camera - Update Dashboard with Join Video Room button and embedded feed --- package-lock.json | 230 ++++++++++++++++++++++++++++- package.json | 3 + src/app/api/livekit-token/route.ts | 32 ++++ src/app/doctor/page.tsx | 112 ++++++++++++++ src/components/Dashboard.tsx | 65 ++++++-- src/components/DoctorVideoRoom.tsx | 64 ++++++++ src/hooks/use-livekit.ts | 50 +++++++ 7 files changed, 545 insertions(+), 11 deletions(-) create mode 100644 src/app/api/livekit-token/route.ts create mode 100644 src/app/doctor/page.tsx create mode 100644 src/components/DoctorVideoRoom.tsx create mode 100644 src/hooks/use-livekit.ts diff --git a/package-lock.json b/package-lock.json index 219065e..f8aaa7c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,10 @@ "dependencies": { "@google/genai": "^1.39.0", "@google/generative-ai": "^0.24.1", + "@livekit/components-react": "^2.9.19", + "@livekit/components-styles": "^1.2.0", "livekit-client": "^2.17.0", + "livekit-server-sdk": "^2.15.0", "lucide-react": "^0.563.0", "next": "16.1.6", "react": "19.2.3", @@ -464,6 +467,31 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@floating-ui/core": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.4.tgz", + "integrity": "sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg==", + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.10" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.4.tgz", + "integrity": "sha512-OOchDgh4F2CchOX94cRVqhvy7b3AFb+/rQXyswmzmGakRfkMgoWVjfnLWkRirfLEfuD4ysVW16eXzwt3jHIzKA==", + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.7.3", + "@floating-ui/utils": "^0.2.10" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.10.tgz", + "integrity": "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==", + "license": "MIT" + }, "node_modules/@google/genai": { "version": "1.39.0", "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.39.0.tgz", @@ -1080,6 +1108,74 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@livekit/components-core": { + "version": "0.12.12", + "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.12.12.tgz", + "integrity": "sha512-DQ+lOAMPvum37Ta4lQLETxQe7ZxhivI78ZfE4nnWP0AcnwNByNR2vVLp9VGvw577HmvgHEkbjBbGBJBSZEBEZA==", + "license": "Apache-2.0", + "dependencies": { + "@floating-ui/dom": "1.7.4", + "loglevel": "1.9.1", + "rxjs": "7.8.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "livekit-client": "^2.15.14", + "tslib": "^2.6.2" + } + }, + "node_modules/@livekit/components-core/node_modules/loglevel": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.1.tgz", + "integrity": "sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" + } + }, + "node_modules/@livekit/components-react": { + "version": "2.9.19", + "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.9.19.tgz", + "integrity": "sha512-VEa4SsiwCHreuIdjxVPrqoWY+Ezq36fHc6swawDXcROPZMCPOeAovaxw83yZqqCKDpzT6oWoeBFDMbxbMytqRw==", + "license": "Apache-2.0", + "dependencies": { + "@livekit/components-core": "0.12.12", + "clsx": "2.1.1", + "events": "^3.3.0", + "jose": "^6.0.12", + "usehooks-ts": "3.1.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@livekit/krisp-noise-filter": "^0.2.12 || ^0.3.0", + "livekit-client": "^2.16.0", + "react": ">=18", + "react-dom": ">=18", + "tslib": "^2.6.2" + }, + "peerDependenciesMeta": { + "@livekit/krisp-noise-filter": { + "optional": true + } + } + }, + "node_modules/@livekit/components-styles": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@livekit/components-styles/-/components-styles-1.2.0.tgz", + "integrity": "sha512-74/rt0lDh6aHmOPmWAeDE9C4OrNW9RIdmhX/YRbovQBVNGNVWojRjl3FgQZ5LPFXO6l1maKB4JhXcBFENVxVvw==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/@livekit/mutex": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz", @@ -2745,6 +2841,36 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-8.0.0.tgz", + "integrity": "sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/camelcase-keys": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-9.1.3.tgz", + "integrity": "sha512-Rircqi9ch8AnZscQcsA1C47NFdaO3wukpmIRzYcDOrmvgt78hM/sj5pZhZNec2NM12uk5vTwRHZ4anGcrC4ZTg==", + "license": "MIT", + "dependencies": { + "camelcase": "^8.0.0", + "map-obj": "5.0.0", + "quick-lru": "^6.1.1", + "type-fest": "^4.3.2" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001767", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001767.tgz", @@ -2788,6 +2914,15 @@ "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", "license": "MIT" }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -5280,6 +5415,7 @@ "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.17.0.tgz", "integrity": "sha512-BD1QUS44ancVTBdnAher0aO7DV5holFYH2lYradYT/HgXtn6R8xPyvtDAH3UH40jGcesDo9fEopCFwEdOgrIhg==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@livekit/mutex": "1.1.1", "@livekit/protocol": "1.42.2", @@ -5296,6 +5432,39 @@ "@types/dom-mediacapture-record": "^1" } }, + "node_modules/livekit-server-sdk": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/livekit-server-sdk/-/livekit-server-sdk-2.15.0.tgz", + "integrity": "sha512-HmzjWnwEwwShu8yUf7VGFXdc+BuMJR5pnIY4qsdlhqI9d9wDgq+4cdTEHg0NEBaiGnc6PCOBiaTYgmIyVJ0S9w==", + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "^1.10.1", + "@livekit/protocol": "^1.43.1", + "camelcase-keys": "^9.0.0", + "jose": "^5.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/livekit-server-sdk/node_modules/@livekit/protocol": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.44.0.tgz", + "integrity": "sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==", + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "^1.10.0" + } + }, + "node_modules/livekit-server-sdk/node_modules/jose": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz", + "integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -5312,6 +5481,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -5380,6 +5555,18 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/map-obj": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-5.0.0.tgz", + "integrity": "sha512-2L3MIgJynYrZ3TYMriLDLWocz15okFakV6J12HXvMXDHui2x/zgChzg1u9mFFGbbGWE+GsLpQByt4POb9Or+uA==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -6012,6 +6199,18 @@ ], "license": "MIT" }, + "node_modules/quick-lru": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-6.1.2.tgz", + "integrity": "sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/react": { "version": "19.2.3", "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", @@ -6182,7 +6381,6 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", - "optional": true, "dependencies": { "tslib": "^2.1.0" } @@ -6952,7 +7150,8 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "license": "0BSD", + "peer": true }, "node_modules/type-check": { "version": "0.4.0", @@ -6967,6 +7166,18 @@ "node": ">= 0.8.0" } }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", @@ -7194,6 +7405,21 @@ "punycode": "^2.1.0" } }, + "node_modules/usehooks-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/usehooks-ts/-/usehooks-ts-3.1.1.tgz", + "integrity": "sha512-I4diPp9Cq6ieSUH2wu+fDAVQO43xwtulo+fKEidHUwZPnYImbtkTjzIJYcDcJqxgmX31GVqNFURodvcgHcW0pA==", + "license": "MIT", + "dependencies": { + "lodash.debounce": "^4.0.8" + }, + "engines": { + "node": ">=16.15.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc" + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", diff --git a/package.json b/package.json index bebf48f..29ac930 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,10 @@ "dependencies": { "@google/genai": "^1.39.0", "@google/generative-ai": "^0.24.1", + "@livekit/components-react": "^2.9.19", + "@livekit/components-styles": "^1.2.0", "livekit-client": "^2.17.0", + "livekit-server-sdk": "^2.15.0", "lucide-react": "^0.563.0", "next": "16.1.6", "react": "19.2.3", diff --git a/src/app/api/livekit-token/route.ts b/src/app/api/livekit-token/route.ts new file mode 100644 index 0000000..4153868 --- /dev/null +++ b/src/app/api/livekit-token/route.ts @@ -0,0 +1,32 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { AccessToken } from 'livekit-server-sdk'; + +export async function POST(req: NextRequest) { + const { room, identity } = await req.json(); + + if (!room || !identity) { + return NextResponse.json({ error: 'room and identity are required' }, { status: 400 }); + } + + const apiKey = process.env.LIVEKIT_API_KEY; + const apiSecret = process.env.LIVEKIT_API_SECRET; + + if (!apiKey || !apiSecret) { + return NextResponse.json({ error: 'LiveKit credentials not configured' }, { status: 500 }); + } + + const token = new AccessToken(apiKey, apiSecret, { + identity, + ttl: '1h', + }); + + token.addGrant({ + room, + roomJoin: true, + canPublish: true, + canSubscribe: true, + }); + + const jwt = await token.toJwt(); + return NextResponse.json({ token: jwt }); +} diff --git a/src/app/doctor/page.tsx b/src/app/doctor/page.tsx new file mode 100644 index 0000000..1f51bb4 --- /dev/null +++ b/src/app/doctor/page.tsx @@ -0,0 +1,112 @@ +'use client'; + +import { useState, useEffect, useCallback } from 'react'; +import { + LiveKitRoom, + VideoTrack, + useTracks, + useLocalParticipant, + ControlBar, +} from '@livekit/components-react'; +import { Track, LocalVideoTrack } from 'livekit-client'; +import '@livekit/components-styles'; + +const ROOM_NAME = 'healthbridge-session'; +const LIVEKIT_URL = process.env.NEXT_PUBLIC_LIVEKIT_URL || ''; + +function DoctorView() { + const { localParticipant } = useLocalParticipant(); + const tracks = useTracks([Track.Source.Camera], { onlySubscribed: false }); + + const localVideoTrack = tracks.find( + (trackRef) => trackRef.participant.isLocal === true + ); + + return ( +
+
+

HealthBridge - Doctor View

+

Your video is being shared with the patient

+
+ +
+
+ {localVideoTrack ? ( + + ) : ( +
+

Enable your camera to start sharing

+
+ )} +
+
+ +
+ +
+
+ ); +} + +export default function DoctorPage() { + const [token, setToken] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + async function getToken() { + try { + const res = await fetch('/api/livekit-token', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + room: ROOM_NAME, + identity: 'doctor-' + Math.random().toString(36).substring(7) + }), + }); + + if (!res.ok) throw new Error('Failed to get token'); + + const data = await res.json(); + setToken(data.token); + } catch (err: any) { + setError(err.message); + } finally { + setIsLoading(false); + } + } + + getToken(); + }, []); + + if (isLoading) { + return ( +
+
Connecting...
+
+ ); + } + + if (error || !token) { + return ( +
+
Error: {error || 'Failed to connect'}
+
+ ); + } + + return ( + + + + ); +} diff --git a/src/components/Dashboard.tsx b/src/components/Dashboard.tsx index 194e602..1c5d4cb 100644 --- a/src/components/Dashboard.tsx +++ b/src/components/Dashboard.tsx @@ -1,9 +1,10 @@ 'use client'; import React, { useState, useEffect, useRef } from 'react'; -import { Video, Mic, Globe, Info, Power, Mic as MicIcon, MicOff } from 'lucide-react'; +import { Video, Mic, Globe, Info, Power, Mic as MicIcon, MicOff, ExternalLink } from 'lucide-react'; import { useGeminiClient, MedicalTerm } from '@/hooks/use-gemini-client'; import { useVideoStream } from '@/hooks/use-video-stream'; +import { DoctorVideoRoom } from '@/components/DoctorVideoRoom'; export default function Dashboard() { const [apiKey, setApiKey] = useState(process.env.NEXT_PUBLIC_GEMINI_API_KEY || ""); @@ -12,6 +13,12 @@ export default function Dashboard() { const scrollRef = useRef(null); const [captionSize, setCaptionSize] = useState<'sm' | 'md' | 'lg'>('md'); + // LiveKit state + const [livekitToken, setLivekitToken] = useState(null); + const [isLivekitConnecting, setIsLivekitConnecting] = useState(false); + const LIVEKIT_URL = process.env.NEXT_PUBLIC_LIVEKIT_URL || ''; + const ROOM_NAME = 'healthbridge-session'; + useEffect(() => { if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; @@ -26,6 +33,25 @@ export default function Dashboard() { } }; + const joinLiveKitRoom = async () => { + setIsLivekitConnecting(true); + try { + const res = await fetch('/api/livekit-token', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ room: ROOM_NAME, identity: 'patient-' + Date.now() }), + }); + if (res.ok) { + const data = await res.json(); + setLivekitToken(data.token); + } + } catch (e) { + console.error('LiveKit token error:', e); + } finally { + setIsLivekitConnecting(false); + } + }; + return (
{/* Header */} @@ -83,14 +109,35 @@ export default function Dashboard() {
{/* Doctor Feed */}
- {/* Placeholder for real video */} -
-
-
- Doctor + {livekitToken ? ( + setLivekitToken(null)} + /> + ) : ( +
+
+ )} +
diff --git a/src/components/DoctorVideoRoom.tsx b/src/components/DoctorVideoRoom.tsx new file mode 100644 index 0000000..827ba25 --- /dev/null +++ b/src/components/DoctorVideoRoom.tsx @@ -0,0 +1,64 @@ +'use client'; + +import { + LiveKitRoom, + VideoTrack, + useTracks, + useParticipants, +} from '@livekit/components-react'; +import { Track } from 'livekit-client'; +import '@livekit/components-styles'; + +interface DoctorVideoRoomProps { + token: string; + serverUrl: string; + onDisconnect?: () => void; +} + +function RemoteVideoTrack() { + const tracks = useTracks([Track.Source.Camera], { onlySubscribed: true }); + const participants = useParticipants(); + + // Find remote participant's camera track (not local) + const remoteVideoTrack = tracks.find( + (trackRef) => trackRef.participant.isLocal === false + ); + + if (!remoteVideoTrack) { + return ( +
+
+

Waiting for Doctor...

+

+ {participants.length > 1 + ? 'Doctor connected, enabling camera...' + : 'Share the room link to join'} +

+
+
+ ); + } + + return ( + + ); +} + +export function DoctorVideoRoom({ token, serverUrl, onDisconnect }: DoctorVideoRoomProps) { + return ( + + + + ); +} diff --git a/src/hooks/use-livekit.ts b/src/hooks/use-livekit.ts new file mode 100644 index 0000000..d987998 --- /dev/null +++ b/src/hooks/use-livekit.ts @@ -0,0 +1,50 @@ +'use client'; + +import { useState, useCallback } from 'react'; + +const LIVEKIT_URL = process.env.NEXT_PUBLIC_LIVEKIT_URL || ''; + +export function useLiveKit() { + const [token, setToken] = useState(null); + const [isConnecting, setIsConnecting] = useState(false); + const [error, setError] = useState(null); + + const getToken = useCallback(async (room: string, identity: string) => { + setIsConnecting(true); + setError(null); + + try { + const res = await fetch('/api/livekit-token', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ room, identity }), + }); + + if (!res.ok) { + throw new Error('Failed to get LiveKit token'); + } + + const data = await res.json(); + setToken(data.token); + return data.token; + } catch (err: any) { + setError(err.message); + return null; + } finally { + setIsConnecting(false); + } + }, []); + + const disconnect = useCallback(() => { + setToken(null); + }, []); + + return { + token, + serverUrl: LIVEKIT_URL, + isConnecting, + error, + getToken, + disconnect, + }; +}