diff --git a/hindsight-control-plane/src/components/bank-selector.tsx b/hindsight-control-plane/src/components/bank-selector.tsx index e257e7d58..46b0f3308 100644 --- a/hindsight-control-plane/src/components/bank-selector.tsx +++ b/hindsight-control-plane/src/components/bank-selector.tsx @@ -46,6 +46,7 @@ import { import { toast } from "sonner"; import { useTheme } from "@/lib/theme-context"; import { useFeatures } from "@/lib/features-context"; +import { upsertPendingDocuments } from "@/lib/pending-documents"; import Image from "next/image"; import { Textarea } from "@/components/ui/textarea"; import { Checkbox } from "@/components/ui/checkbox"; @@ -72,6 +73,13 @@ function toIsoTimestamp(value: string): string { return value.includes("T") ? value : `${value}T00:00:00`; } +function createFileDocumentId(): string { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return `file_${crypto.randomUUID()}`; + } + return `file_${Date.now()}_${Math.random().toString(36).slice(2)}`; +} + function formatTimeAgo(isoDate: string): string { const diff = Date.now() - new Date(isoDate).getTime(); const seconds = Math.floor(diff / 1000); @@ -347,27 +355,53 @@ function BankSelectorInner() { try { setUploadProgress(`Uploading ${selectedFiles.length} file(s)...`); - const perFileMeta = filesMetadata.map((meta) => ({ - ...(meta.context && { context: meta.context }), - ...(meta.timestamp && { timestamp: toIsoTimestamp(meta.timestamp) }), - ...(meta.document_id && { document_id: meta.document_id }), - ...(meta.tags && { - tags: meta.tags - .split(",") - .map((t) => t.trim()) - .filter(Boolean), - }), - ...(meta.metadata && { metadata: parseMetadata(meta.metadata) }), - ...(meta.strategy && { strategy: meta.strategy }), - })); - - await client.uploadFiles({ + const filesToUpload = selectedFiles.map((file, index) => { + const meta = filesMetadata[index] ?? emptyFileMeta(file.name); + const documentId = meta.document_id.trim() || createFileDocumentId(); + const tags = meta.tags + ? meta.tags + .split(",") + .map((t) => t.trim()) + .filter(Boolean) + : []; + + return { + file, + documentId, + tags, + metadata: { + ...(meta.context && { context: meta.context }), + ...(meta.timestamp && { timestamp: toIsoTimestamp(meta.timestamp) }), + document_id: documentId, + ...(tags.length > 0 && { tags }), + ...(meta.metadata && { metadata: parseMetadata(meta.metadata) }), + ...(meta.strategy && { strategy: meta.strategy }), + }, + }; + }); + + const response = await client.uploadFiles({ bank_id: currentBank, - files: selectedFiles, + files: filesToUpload.map((item) => item.file), async: true, - files_metadata: perFileMeta, + files_metadata: filesToUpload.map((item) => item.metadata), }); + const operationIds = Array.isArray(response?.operation_ids) ? response.operation_ids : []; + + upsertPendingDocuments( + filesToUpload.map((item, index) => ({ + id: item.documentId, + bankId: currentBank, + filename: item.file.name, + size: item.file.size, + tags: item.tags, + operationId: operationIds[index], + createdAt: new Date().toISOString(), + status: "processing", + })) + ); + // Reset form and close dialog setDocDialogOpen(false); setSelectedFiles([]); diff --git a/hindsight-control-plane/src/components/documents-view.tsx b/hindsight-control-plane/src/components/documents-view.tsx index ed0b25847..0f4021fd6 100644 --- a/hindsight-control-plane/src/components/documents-view.tsx +++ b/hindsight-control-plane/src/components/documents-view.tsx @@ -1,11 +1,18 @@ "use client"; -import { useState, useEffect, useCallback } from "react"; +import { useState, useEffect, useCallback, useMemo } from "react"; import { useTranslations } from "next-intl"; import { toast } from "sonner"; import { client, LLMRequestEntry } from "@/lib/api"; import { useBank } from "@/lib/bank-context"; import { useFeatures } from "@/lib/features-context"; +import { + getPendingDocuments, + removePendingDocuments, + subscribePendingDocuments, + updatePendingDocumentStatus, + type PendingDocument, +} from "@/lib/pending-documents"; import { DataView } from "./data-view"; import { TraceDialog } from "./llm-requests-view"; import { Button } from "@/components/ui/button"; @@ -79,6 +86,19 @@ import { const ITEMS_PER_PAGE = 50; +type DocumentListItem = { + id: string; + created_at?: string | null; + updated_at?: string | null; + tags?: string[]; + document_metadata?: Record | null; + text_length?: number | null; + memory_unit_count?: number; + _pending?: false; +}; + +type DisplayDocument = (PendingDocument & { _pending: true }) | DocumentListItem; + function formatRelativeTime(dateStr: string): string { const now = Date.now(); const then = new Date(dateStr).getTime(); @@ -575,7 +595,8 @@ export function DocumentsView() { const tBank = useTranslations("bank"); const { currentBank } = useBank(); const { features } = useFeatures(); - const [documents, setDocuments] = useState([]); + const [documents, setDocuments] = useState([]); + const [pendingDocuments, setPendingDocuments] = useState([]); const [loading, setLoading] = useState(false); const [searchQuery, setSearchQuery] = useState(""); const [total, setTotal] = useState(0); @@ -631,26 +652,53 @@ export function DocumentsView() { null ); - const loadDocuments = async (page: number = 1) => { - if (!currentBank) return; + const loadDocuments = useCallback( + async (page: number = 1) => { + if (!currentBank) return; - setLoading(true); - try { - const pageOffset = (page - 1) * ITEMS_PER_PAGE; - const data: any = await client.listDocuments({ - bank_id: currentBank, - q: searchQuery, - limit: ITEMS_PER_PAGE, - offset: pageOffset, - }); - setDocuments(data.items || []); - setTotal(data.total || 0); - } catch (error) { - // Error toast is shown automatically by the API client interceptor - } finally { - setLoading(false); - } - }; + setLoading(true); + try { + const pageOffset = (page - 1) * ITEMS_PER_PAGE; + const data = (await client.listDocuments({ + bank_id: currentBank, + q: searchQuery, + limit: ITEMS_PER_PAGE, + offset: pageOffset, + })) as { items?: DocumentListItem[]; total?: number }; + const items = data.items || []; + setDocuments(items); + setTotal(data.total || 0); + removePendingDocuments(currentBank, items.map((doc) => doc.id).filter(Boolean)); + } catch (error) { + // Error toast is shown automatically by the API client interceptor + } finally { + setLoading(false); + } + }, + [currentBank, searchQuery] + ); + + const pendingRows = useMemo(() => { + const realDocumentIds = new Set(documents.map((doc) => doc.id)); + const normalizedQuery = searchQuery.trim().toLowerCase(); + + return pendingDocuments + .filter((doc) => !realDocumentIds.has(doc.id)) + .filter((doc) => { + if (!normalizedQuery) return true; + return ( + doc.id.toLowerCase().includes(normalizedQuery) || + doc.filename.toLowerCase().includes(normalizedQuery) + ); + }) + .map((doc) => ({ ...doc, _pending: true })); + }, [documents, pendingDocuments, searchQuery]); + + const visibleDocuments = useMemo( + () => [...pendingRows, ...documents], + [documents, pendingRows] + ); + const displayTotal = total + pendingRows.length; // Handle page change const handlePageChange = (newPage: number) => { @@ -851,6 +899,69 @@ export function DocumentsView() { } }, [currentBank]); + useEffect(() => { + if (!currentBank) { + setPendingDocuments([]); + return; + } + + const syncPendingDocuments = () => { + setPendingDocuments(getPendingDocuments(currentBank)); + }; + + syncPendingDocuments(); + return subscribePendingDocuments(syncPendingDocuments); + }, [currentBank]); + + useEffect(() => { + if (!currentBank) return; + + const tracked = pendingDocuments.filter( + (doc) => doc.status === "processing" && doc.operationId + ); + if (tracked.length === 0) return; + + let cancelled = false; + + const pollPendingOperations = async () => { + let shouldRefreshDocuments = false; + + await Promise.all( + tracked.map(async (doc) => { + try { + const op = await client.getOperationStatus(currentBank, doc.operationId!); + if (cancelled) return; + + if (op.status === "failed") { + updatePendingDocumentStatus( + currentBank, + doc.id, + "failed", + op.error_message || t("pendingUploadFailed") + ); + } else if (op.status === "completed") { + shouldRefreshDocuments = true; + } + } catch { + // Keep the pending row; the operations page remains the detailed source of truth. + } + }) + ); + + if (!cancelled && shouldRefreshDocuments) { + loadDocuments(currentPage); + } + }; + + pollPendingOperations(); + const interval = window.setInterval(pollPendingOperations, 5000); + + return () => { + cancelled = true; + window.clearInterval(interval); + }; + }, [currentBank, currentPage, loadDocuments, pendingDocuments, t]); + // Reload when search query changes (with debounce) useEffect(() => { if (!currentBank) return; @@ -861,7 +972,7 @@ export function DocumentsView() { }, 300); // 300ms debounce return () => clearTimeout(timeoutId); - }, [searchQuery]); + }, [currentBank, searchQuery]); const triggerDownload = (blob: Blob, filename: string) => { const url = URL.createObjectURL(blob); @@ -1083,16 +1194,18 @@ export function DocumentsView() { {/* Documents List Section */} - {loading ? ( + {loading && visibleDocuments.length === 0 ? (
⏳
{t("loadingDocuments")}
- ) : documents.length > 0 ? ( + ) : visibleDocuments.length > 0 ? ( <> -
{t("totalDocuments", { total })}
+
+ {t("totalDocuments", { total: displayTotal })} +
{/* Documents Table */}
@@ -1119,27 +1232,51 @@ export function DocumentsView() { - {documents.length > 0 ? ( - documents.map((doc) => ( + {visibleDocuments.map((doc) => { + const isPending = doc._pending === true; + return ( viewDocumentText(doc.id)} + key={`${isPending ? "pending" : "document"}-${doc.id}`} + className={ + isPending + ? "bg-muted/30" + : `cursor-pointer hover:bg-muted/50 ${selectedDocument?.id === doc.id ? "bg-primary/10" : ""}` + } + onClick={isPending ? undefined : () => viewDocumentText(doc.id)} > - {doc.id} +
{doc.id}
+ {isPending && ( +
+ {doc.filename} +
+ )} - {doc.created_at ? formatRelativeTime(doc.created_at) : "N/A"} + {isPending + ? formatRelativeTime(doc.createdAt) + : doc.created_at + ? formatRelativeTime(doc.created_at) + : "N/A"} - {doc.updated_at ? formatRelativeTime(doc.updated_at) : "N/A"} + {!isPending && doc.updated_at ? formatRelativeTime(doc.updated_at) : "-"} {doc.tags && doc.tags.length > 0 ? ( @@ -1163,28 +1300,43 @@ export function DocumentsView() { )} - {doc.document_metadata && - Object.keys(doc.document_metadata).length > 0 ? ( + {isPending ? ( + + {t("pendingUploadMetadata")} + + ) : doc.document_metadata && + Object.keys(doc.document_metadata).length > 0 ? ( ) : ( "-" )} - {formatBytes(doc.text_length || 0)} + {formatBytes(isPending ? doc.size : doc.text_length || 0)} - {doc.memory_unit_count} + {isPending ? ( + doc.status === "failed" ? ( + + + {t("pendingUploadFailedStatus")} + + ) : ( + + + {t("pendingUploadProcessingStatus")} + + ) + ) : ( + doc.memory_unit_count + )} - )) - ) : ( - - - {t("clickLoadDocumentsToView")} - - - )} + ); + })}
diff --git a/hindsight-control-plane/src/lib/pending-documents.ts b/hindsight-control-plane/src/lib/pending-documents.ts new file mode 100644 index 000000000..ad7cc16ac --- /dev/null +++ b/hindsight-control-plane/src/lib/pending-documents.ts @@ -0,0 +1,103 @@ +"use client"; + +const STORAGE_KEY = "hindsight.pendingDocuments.v1"; +const CHANGE_EVENT = "hindsight:pending-documents-changed"; +const MAX_AGE_MS = 24 * 60 * 60 * 1000; + +export type PendingDocumentStatus = "processing" | "failed"; + +export interface PendingDocument { + id: string; + bankId: string; + filename: string; + size: number; + tags: string[]; + operationId?: string; + createdAt: string; + status: PendingDocumentStatus; + error?: string; +} + +function canUseStorage() { + return typeof window !== "undefined" && typeof window.sessionStorage !== "undefined"; +} + +function readAll(): PendingDocument[] { + if (!canUseStorage()) return []; + + try { + const raw = window.sessionStorage.getItem(STORAGE_KEY); + if (!raw) return []; + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + + const now = Date.now(); + return parsed.filter((item): item is PendingDocument => { + if (!item || typeof item !== "object") return false; + if (typeof item.id !== "string" || typeof item.bankId !== "string") return false; + const createdAt = typeof item.createdAt === "string" ? Date.parse(item.createdAt) : NaN; + return Number.isFinite(createdAt) && now - createdAt <= MAX_AGE_MS; + }); + } catch { + return []; + } +} + +function writeAll(items: PendingDocument[]) { + if (!canUseStorage()) return; + window.sessionStorage.setItem(STORAGE_KEY, JSON.stringify(items)); + window.dispatchEvent(new Event(CHANGE_EVENT)); +} + +export function getPendingDocuments(bankId: string): PendingDocument[] { + return readAll().filter((item) => item.bankId === bankId); +} + +export function upsertPendingDocuments(items: PendingDocument[]) { + if (items.length === 0) return; + + const nextByKey = new Map(readAll().map((item) => [`${item.bankId}:${item.id}`, item])); + for (const item of items) { + nextByKey.set(`${item.bankId}:${item.id}`, item); + } + writeAll([...nextByKey.values()]); +} + +export function removePendingDocuments(bankId: string, documentIds: string[]) { + if (documentIds.length === 0) return; + + const ids = new Set(documentIds); + writeAll(readAll().filter((item) => item.bankId !== bankId || !ids.has(item.id))); +} + +export function updatePendingDocumentStatus( + bankId: string, + documentId: string, + status: PendingDocumentStatus, + error?: string +) { + let changed = false; + const next = readAll().map((item) => { + if (item.bankId !== bankId || item.id !== documentId) return item; + changed = true; + return { ...item, status, error }; + }); + + if (changed) writeAll(next); +} + +export function subscribePendingDocuments(listener: () => void) { + if (typeof window === "undefined") return () => {}; + + const onStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY) listener(); + }; + + window.addEventListener(CHANGE_EVENT, listener); + window.addEventListener("storage", onStorage); + + return () => { + window.removeEventListener(CHANGE_EVENT, listener); + window.removeEventListener("storage", onStorage); + }; +} diff --git a/hindsight-control-plane/src/messages/de.json b/hindsight-control-plane/src/messages/de.json index 6fe356bde..068dff9df 100644 --- a/hindsight-control-plane/src/messages/de.json +++ b/hindsight-control-plane/src/messages/de.json @@ -616,6 +616,10 @@ "refreshDocuments": "Dokumente aktualisieren", "noDocumentsMatchSearch": "Keine Dokumente entsprechen Ihrer Suche", "noDocumentsFound": "Keine Dokumente gefunden", + "pendingUploadMetadata": "Datei angenommen; Konvertierung ausstehend", + "pendingUploadProcessingStatus": "Wird verarbeitet", + "pendingUploadFailedStatus": "Fehlgeschlagen", + "pendingUploadFailed": "Dateikonvertierung fehlgeschlagen", "colDocumentId": "Dokument-ID", "colCreated": "Erstellt", "colUpdated": "Aktualisiert", diff --git a/hindsight-control-plane/src/messages/en.json b/hindsight-control-plane/src/messages/en.json index 90b937a1a..eb3174497 100644 --- a/hindsight-control-plane/src/messages/en.json +++ b/hindsight-control-plane/src/messages/en.json @@ -616,6 +616,10 @@ "refreshDocuments": "Refresh documents", "noDocumentsMatchSearch": "No documents match your search", "noDocumentsFound": "No documents found", + "pendingUploadMetadata": "File accepted; conversion pending", + "pendingUploadProcessingStatus": "Processing", + "pendingUploadFailedStatus": "Failed", + "pendingUploadFailed": "File conversion failed", "colDocumentId": "Document ID", "colCreated": "Created", "colUpdated": "Updated", diff --git a/hindsight-control-plane/src/messages/es.json b/hindsight-control-plane/src/messages/es.json index 8092077e5..ca4ab528e 100644 --- a/hindsight-control-plane/src/messages/es.json +++ b/hindsight-control-plane/src/messages/es.json @@ -616,6 +616,10 @@ "refreshDocuments": "Actualizar documentos", "noDocumentsMatchSearch": "Ningún documento coincide con tu búsqueda", "noDocumentsFound": "No se encontraron documentos", + "pendingUploadMetadata": "Archivo aceptado; conversión pendiente", + "pendingUploadProcessingStatus": "Procesando", + "pendingUploadFailedStatus": "Error", + "pendingUploadFailed": "Error al convertir el archivo", "colDocumentId": "ID del documento", "colCreated": "Creado", "colUpdated": "Actualizado", diff --git a/hindsight-control-plane/src/messages/fr.json b/hindsight-control-plane/src/messages/fr.json index cc28bec82..b66dd7d3d 100644 --- a/hindsight-control-plane/src/messages/fr.json +++ b/hindsight-control-plane/src/messages/fr.json @@ -616,6 +616,10 @@ "refreshDocuments": "Actualiser les documents", "noDocumentsMatchSearch": "Aucun document ne correspond à votre recherche", "noDocumentsFound": "Aucun document trouvé", + "pendingUploadMetadata": "Fichier accepté ; conversion en attente", + "pendingUploadProcessingStatus": "Traitement", + "pendingUploadFailedStatus": "Échec", + "pendingUploadFailed": "La conversion du fichier a échoué", "colDocumentId": "ID du document", "colCreated": "Créé", "colUpdated": "Mis à jour", diff --git a/hindsight-control-plane/src/messages/ja.json b/hindsight-control-plane/src/messages/ja.json index 08e97f5b0..157bf142c 100644 --- a/hindsight-control-plane/src/messages/ja.json +++ b/hindsight-control-plane/src/messages/ja.json @@ -616,6 +616,10 @@ "refreshDocuments": "ドキュメントを更新", "noDocumentsMatchSearch": "検索に一致するドキュメントがありません", "noDocumentsFound": "ドキュメントが見つかりません", + "pendingUploadMetadata": "ファイルを受け付けました。変換待ちです", + "pendingUploadProcessingStatus": "処理中", + "pendingUploadFailedStatus": "失敗", + "pendingUploadFailed": "ファイル変換に失敗しました", "colDocumentId": "ドキュメントID", "colCreated": "作成日", "colUpdated": "更新日", diff --git a/hindsight-control-plane/src/messages/ko.json b/hindsight-control-plane/src/messages/ko.json index 318901758..c73badab6 100644 --- a/hindsight-control-plane/src/messages/ko.json +++ b/hindsight-control-plane/src/messages/ko.json @@ -616,6 +616,10 @@ "refreshDocuments": "문서 새로고침", "noDocumentsMatchSearch": "검색과 일치하는 문서 없음", "noDocumentsFound": "문서를 찾을 수 없습니다", + "pendingUploadMetadata": "파일이 접수되었으며 변환 대기 중입니다", + "pendingUploadProcessingStatus": "처리 중", + "pendingUploadFailedStatus": "실패", + "pendingUploadFailed": "파일 변환 실패", "colDocumentId": "문서 ID", "colCreated": "생성일", "colUpdated": "수정일", diff --git a/hindsight-control-plane/src/messages/pt.json b/hindsight-control-plane/src/messages/pt.json index f71787da7..424df8b2f 100644 --- a/hindsight-control-plane/src/messages/pt.json +++ b/hindsight-control-plane/src/messages/pt.json @@ -616,6 +616,10 @@ "refreshDocuments": "Atualizar documentos", "noDocumentsMatchSearch": "Nenhum documento corresponde à sua pesquisa", "noDocumentsFound": "Nenhum documento encontrado", + "pendingUploadMetadata": "Arquivo aceito; conversão pendente", + "pendingUploadProcessingStatus": "Processando", + "pendingUploadFailedStatus": "Falhou", + "pendingUploadFailed": "Falha na conversão do arquivo", "colDocumentId": "ID do Documento", "colCreated": "Criado", "colUpdated": "Atualizado", diff --git a/hindsight-control-plane/src/messages/yue-Hant.json b/hindsight-control-plane/src/messages/yue-Hant.json index 9909ac4fd..e0b4f6d2c 100644 --- a/hindsight-control-plane/src/messages/yue-Hant.json +++ b/hindsight-control-plane/src/messages/yue-Hant.json @@ -616,6 +616,10 @@ "refreshDocuments": "重新整理文件", "noDocumentsMatchSearch": "沒有符合搜尋條件的文件", "noDocumentsFound": "找不到文件", + "pendingUploadMetadata": "文件已接收，正在轉換為文件", + "pendingUploadProcessingStatus": "處理中", + "pendingUploadFailedStatus": "失敗", + "pendingUploadFailed": "文件轉換失敗", "colDocumentId": "文件 ID", "colCreated": "建立時間", "colUpdated": "更新時間", diff --git a/hindsight-control-plane/src/messages/zh-CN.json b/hindsight-control-plane/src/messages/zh-CN.json index 253a078e2..1d455e4cf 100644 --- a/hindsight-control-plane/src/messages/zh-CN.json +++ b/hindsight-control-plane/src/messages/zh-CN.json @@ -616,6 +616,10 @@ "refreshDocuments": "刷新文档", "noDocumentsMatchSearch": "无文档匹配您的搜索", "noDocumentsFound": "未找到文档", + "pendingUploadMetadata": "文件已接收，正在转换为文档", + "pendingUploadProcessingStatus": "处理中", + "pendingUploadFailedStatus": "失败", + "pendingUploadFailed": "文件转换失败", "colDocumentId": "文档 ID", "colCreated": "创建时间", "colUpdated": "更新时间", diff --git a/hindsight-control-plane/src/messages/zh-TW.json b/hindsight-control-plane/src/messages/zh-TW.json index 0bbdad53d..a591761f0 100644 --- a/hindsight-control-plane/src/messages/zh-TW.json +++ b/hindsight-control-plane/src/messages/zh-TW.json @@ -616,6 +616,10 @@ "refreshDocuments": "重新整理文件", "noDocumentsMatchSearch": "沒有符合搜尋條件的文件", "noDocumentsFound": "找不到文件", + "pendingUploadMetadata": "文件已接收，正在轉換為文件", + "pendingUploadProcessingStatus": "處理中", + "pendingUploadFailedStatus": "失敗", + "pendingUploadFailed": "文件轉換失敗", "colDocumentId": "文件 ID", "colCreated": "建立時間", "colUpdated": "更新時間",