feat: add generic docstring extraction tools (Phase 2)

frontend/osa-chat-widget.js

@@ -1227,19 +1227,45 @@
   // Save chat history to localStorage
   let saveErrorShown = false;
   function saveHistory() {
+    if (!CONFIG.storageKey) {
+      console.warn('[OSA] Cannot save history - no storage key configured');
+      return;
+    }
+
     try {
-      localStorage.setItem(CONFIG.storageKey, JSON.stringify(messages));
+      const data = JSON.stringify(messages);
+      localStorage.setItem(CONFIG.storageKey, data);
       saveErrorShown = false;
     } catch (e) {
-      console.error('Failed to save chat history:', e);
-      // Show error once per session to avoid spam
-      if (!saveErrorShown) {
-        const container = document.querySelector('.osa-chat-widget');
-        if (container) {
-          showError(container, 'Chat history could not be saved. Storage may be full or disabled.');
-        }
-        saveErrorShown = true;
+      console.error('[OSA] localStorage save failed:', {
+        errorName: e.name,
+        errorMessage: e.message,
+        messageCount: messages.length,
+        isQuotaError: e.name === 'QuotaExceededError'
+      });
+
+      // Determine error type for better user messaging
+      let errorMsg = 'Chat history could not be saved';
+      const isQuotaError = e.name === 'QuotaExceededError';
+      const isSecurityError = e.name === 'SecurityError';
+
+      if (isQuotaError) {
+        errorMsg = 'Storage full - conversation NOT saved. Clear browser data or export chat.';
+      } else if (isSecurityError) {
+        errorMsg = 'Browser privacy settings prevent saving. Enable local storage.';
+      } else {
+        errorMsg = 'Storage unavailable - conversation will be lost on refresh.';
+      }
+
+      // Show error (not just once - user needs to know every time save fails)
+      const container = document.querySelector('.osa-chat-widget');
+      if (container && !saveErrorShown) {
+        showError(container, errorMsg);
+        saveErrorShown = true; // Show once per session to avoid spam
       }
+
+      // Re-throw so callers know save failed
+      throw e;
     }
   }
 
@@ -2091,10 +2117,16 @@
       throw error; // Re-throw to be handled by sendMessage
     } finally {
       // Always release the reader to free resources
-      try {
-        reader.releaseLock();
-      } catch (e) {
-        // Reader may already be closed, ignore errors
+      if (reader) {
+        try {
+          reader.releaseLock();
+        } catch (releaseError) {
+          // Log cleanup failures - they indicate serious issues
+          console.error('[OSA] Failed to release stream reader:', {
+            errorName: releaseError.name,
+            errorMessage: releaseError.message
+          });
+        }
       }
     }
   }
@@ -2104,7 +2136,12 @@
     if (isLoading || !question.trim()) return;
 
     isLoading = true;
+
+    // Track message indices to avoid corruption on error
+    const userMessageIndex = messages.length;
     messages.push({ role: 'user', content: question });
+    let assistantMessageCreated = false;
+
     renderMessages(container);
     renderSuggestions(container);
 
@@ -2157,6 +2194,7 @@
         method: 'POST',
         headers: headers,
         body: JSON.stringify(body),
+        signal: AbortSignal.timeout(120000), // 2 minute timeout for connection + streaming
       });
 
       if (!response.ok) {
@@ -2185,6 +2223,7 @@
       const contentType = response.headers.get('content-type') || '';
       if (CONFIG.streamingEnabled && contentType.includes('text/event-stream')) {
         // Handle streaming response
+        assistantMessageCreated = true; // handleStreamingResponse creates assistant message
         await handleStreamingResponse(response, container);
       } else if (CONFIG.streamingEnabled && !contentType.includes('text/event-stream')) {
         // Streaming was expected but not received - log for debugging
@@ -2243,19 +2282,30 @@
       console.error('[OSA] Send message error:', error);
       showError(container, userMessage);
 
-      // Check if we need to clean up messages
-      // If handleStreamingResponse threw and kept partial content, assistant message is already in array
-      // If handleStreamingResponse threw and had no content, it already popped the assistant message
-      // We need to remove the user message only if it's the last message
-      const lastMessage = messages[messages.length - 1];
-      if (lastMessage && lastMessage.role === 'user' && lastMessage.content === question) {
-        messages.pop();
+      // Clean up messages based on what was created
+      // If streaming was attempted, handleStreamingResponse manages its own assistant message
+      // We only need to remove the user message if no assistant response exists
+      if (assistantMessageCreated) {
+        // handleStreamingResponse created an assistant message
+        // If it has content (partial or complete), keep both user and assistant messages
+        // If it has no content, handleStreamingResponse already removed it, so remove user message too
+        const lastMessage = messages[messages.length - 1];
+        if (lastMessage && lastMessage.role === 'user' && messages.length === userMessageIndex + 1) {
+          // No assistant message remains, remove user message
+          messages.splice(userMessageIndex, 1);
+        }
+      } else {
+        // No streaming attempted, no assistant message created, remove user message
+        if (messages.length > userMessageIndex && messages[userMessageIndex].role === 'user') {
+          messages.splice(userMessageIndex, 1);
+        }
       }
 
       try {
         saveHistory();
       } catch (saveError) {
         console.error('[OSA] Failed to save history after error:', saveError);
+        // saveHistory already showed error to user
       }
       updateStatusDisplay(false);
     } finally {

src/api/routers/community.py

@@ -1060,14 +1060,40 @@ async def _stream_ask_response(
         sse_event = {"event": "done"}
         yield f"data: {json.dumps(sse_event)}\n\n"
 
+    except HTTPException:
+        # Don't catch our own HTTP exceptions - let them propagate
+        raise
+    except ValueError as e:
+        # Input validation errors - user's fault
+        logger.warning("Invalid input in streaming for community %s: %s", community_id, e)
+        sse_event = {
+            "event": "error",
+            "message": f"Invalid request: {str(e)}",
+            "retryable": False,
+        }
+        yield f"data: {json.dumps(sse_event)}\n\n"
     except Exception as e:
+        # Unexpected errors - log with full context
+        import uuid
+
+        error_id = str(uuid.uuid4())
         logger.error(
-            "Streaming error in ask endpoint for community %s: %s",
+            "Unexpected streaming error (ID: %s) in ask endpoint for community %s: %s",
+            error_id,
             community_id,
             e,
             exc_info=True,
+            extra={
+                "error_id": error_id,
+                "community_id": community_id,
+                "error_type": type(e).__name__,
+            },
         )
-        sse_event = {"event": "error", "message": str(e)}
+        sse_event = {
+            "event": "error",
+            "message": "An error occurred while generating the response. Please try again.",
+            "error_id": error_id,
+        }
         yield f"data: {json.dumps(sse_event)}\n\n"
 
 

src/cli/sync.py

@@ -18,6 +18,7 @@
 from src.assistants import registry
 from src.cli.config import load_config
 from src.knowledge.db import get_db_path, get_stats, init_db
+from src.knowledge.docstring_sync import sync_repo_docstrings
 from src.knowledge.github_sync import sync_repo, sync_repos
 from src.knowledge.papers_sync import (
     sync_all_papers,
@@ -330,6 +331,80 @@ def sync_papers(
     console.print(f"\n[green]Total papers synced for {community}: {total}[/green]")
 
 
+@sync_app.command("docstrings")
+def sync_docstrings(
+    community: Annotated[
+        str,
+        typer.Option("--community", "-c", help="Community ID to sync (e.g., hed, bids, eeglab)"),
+    ] = "hed",
+    language: Annotated[
+        str,
+        typer.Option("--language", "-l", help="Language: matlab or python"),
+    ] = "matlab",
+    repo: Annotated[
+        str | None,
+        typer.Option("--repo", "-r", help="Single repo to sync (owner/name format)"),
+    ] = None,
+    branch: Annotated[
+        str,
+        typer.Option("--branch", "-b", help="Branch to sync from"),
+    ] = "main",
+) -> None:
+    """Sync code docstrings from GitHub repositories.
+
+    Extracts docstrings from MATLAB (.m) or Python (.py) files and indexes them
+    for search. If --repo is specified, syncs that single repo. Otherwise, syncs
+    all repos configured for the community.
+    """
+    _require_admin()
+    _validate_community(community)
+
+    if language not in ("matlab", "python"):
+        console.print("[red]Error: Language must be 'matlab' or 'python'[/red]")
+        raise typer.Exit(1)
+
+    if not _safe_init_db(community):
+        raise typer.Exit(1)
+
+    if repo:
+        # Sync single repo
+        try:
+            count = sync_repo_docstrings(repo, language, project=community, branch=branch)
+            console.print(f"\n[green]✓ Synced {count} {language} docstrings from {repo}[/green]")
+        except Exception as e:
+            console.print(f"[red]Error syncing {repo}: {e}[/red]")
+            logger.exception("Failed to sync docstrings from %s", repo)
+            raise typer.Exit(1)
+    else:
+        # Sync all repos from community config
+        repos = _get_community_repos(community)
+        if not repos:
+            console.print(f"[yellow]No repos configured for community '{community}'[/yellow]")
+            console.print("[dim]Use --repo to specify a repository explicitly[/dim]")
+            raise typer.Exit(1)
+
+        console.print(f"[dim]Syncing {language} docstrings from {len(repos)} repos...[/dim]\n")
+        total = 0
+        failed = []
+
+        for repo_name in repos:
+            try:
+                count = sync_repo_docstrings(repo_name, language, project=community, branch=branch)
+                total += count
+                if count > 0:
+                    console.print(f"  ✓ {repo_name}: {count} docstrings")
+                else:
+                    console.print(f"  - {repo_name}: no {language} files found")
+            except Exception as e:
+                console.print(f"  ✗ {repo_name}: {e}")
+                logger.warning("Failed to sync docstrings from %s: %s", repo_name, e)
+                failed.append(repo_name)
+
+        console.print(f"\n[green]Total {language} docstrings synced: {total}[/green]")
+        if failed:
+            console.print(f"[yellow]Failed repos ({len(failed)}): {', '.join(failed)}[/yellow]")
+
+
 @sync_app.command("all")
 def sync_all(
     community: Annotated[