From 9ef640dc8a873b2eb0e6822c11ddc83ef385eb91 Mon Sep 17 00:00:00 2001 From: vjuliaife Date: Sat, 20 Jun 2026 06:24:11 -0700 Subject: [PATCH 1/7] docs: add CONTRIBUTING.md to .github with lifecycle diagram and standards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creates .github/CONTRIBUTING.md as the primary GitHub-surfaced contributor guide. Includes a 3-step local setup, an end-to-end Mermaid data lifecycle diagram (Stellar Network → Soroban RPC → Event Indexer → Translation Engine → PostgreSQL/WebSocket → Frontend Dashboard), condensed coding standards, commit format, PR procedures, and a pre-PR checklist. Closes #102 --- .github/CONTRIBUTING.md | 209 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 .github/CONTRIBUTING.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..5edff1d --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,209 @@ +# Contributing to Open-Audit + +Open-Audit is the "Google Translate for Soroban" — it transforms cryptic on-chain smart contract events into human-readable sentences for the Stellar ecosystem. + +**Detailed guides:** +- [Translation Blueprint Authoring](../CONTRIBUTING.md) — how to add support for a new contract +- [System Architecture](../ARCHITECTURE.md) — deep-dive into every component + +--- + +## Quick Setup + +A working local environment in 3 steps: + +```bash +# 1 — Clone and install dependencies +git clone https://github.com/coderolisa/Open-Audit.git && cd Open-Audit && npm install + +# 2 — Configure environment (testnet defaults, no changes required) +cp .env.example .env.local + +# 3 — Start the development server with WebSocket support +npm run dev:ws +``` + +Open **http://localhost:3000/dashboard** — the live event feed will begin streaming from Stellar testnet. + +--- + +## Data Lifecycle + +How a single smart contract event travels from the Stellar network to your screen: + +```mermaid +flowchart TB + subgraph Network["🌐 Network Layer"] + Contracts["Smart Contracts\n(SAC, Soroswap, etc.)"] + RPC["Soroban RPC\nsoroban-testnet.stellar.org"] + Contracts -->|"emit XDR events"| RPC + end + + subgraph Parsing["⚙️ Parsing Worker — lib/stellar/indexer.ts"] + Poller["Poll getEvents()\nevery 5 seconds"] + Retry["Exponential Backoff\n1s → 2s → 4s … 32s max"] + Cursor["Cursor Manager\ntracks last indexed ledger"] + Poller --> Retry --> Cursor + end + + subgraph Translation["🔤 Translation Engine — lib/translator/"] + Registry["Registry Lookup\ncontractId → Blueprint"] + Blueprint["Blueprint.translate()\ndecodes XDR topics + data"] + Result["TranslatedEvent\nhuman-readable description"] + Registry --> Blueprint --> Result + end + + subgraph Persistence["🗄️ Database — Prisma + PostgreSQL"] + DB[("PostgreSQL\napp/api/ routes")] + end + + subgraph Realtime["📡 WebSocket Server — server.ts"] + WS["Broadcast to\nall connected clients"] + end + + subgraph Frontend["🎨 Frontend Dashboard — Next.js 14"] + Hook["useLiveFeed hook\nWebSocket client"] + Table["EventFeedTable\nlive event rows"] + Stats["StatsBar\ntranslated vs cryptic"] + Hook --> Table --> Stats + end + + RPC -->|"RawEvent[]"| Poller + Cursor -->|"RawEvent[]"| Registry + Result -->|"persist via REST API"| DB + Result -->|"TranslatedEvent"| WS + DB -->|"historical queries"| Frontend + WS -->|"real-time stream"| Hook + + style Network fill:#7B2FBE,color:#fff + style Parsing fill:#2E86AB,color:#fff + style Translation fill:#2E86AB,color:#fff + style Persistence fill:#E07B39,color:#fff + style Realtime fill:#2E86AB,color:#fff + style Frontend fill:#06A77D,color:#fff +``` + +--- + +## Coding Standards + +These rules are enforced by ESLint and TypeScript — CI will fail if they are violated. See [CODE_STANDARDS.md](../CODE_STANDARDS.md) for the full reference. + +### Function declarations + +Use standard function declarations for all top-level definitions. Arrow functions are only allowed as inline callbacks. + +```typescript +// ✅ Correct +function translateEvent(event: RawEvent): TranslationResult | null { ... } + +// ❌ Incorrect +const translateEvent = (event: RawEvent): TranslationResult | null => { ... } +``` + +ESLint rule: `"func-style": ["error", "declaration"]` + +### No `any` types + +```typescript +// ✅ Correct +function processData(data: RawEvent): TranslatedEvent { ... } + +// ❌ Incorrect +function processData(data: any): any { ... } +``` + +ESLint rule: `"@typescript-eslint/no-explicit-any": "error"` + +### Interfaces vs type aliases + +Use `interface` for object shapes. Use `type` for unions and primitives. + +```typescript +interface RawEvent { id: string; contractId: string; ... } // ✅ +type TranslationStatus = "translated" | "cryptic"; // ✅ +type RawEvent = { id: string; ... }; // ❌ +``` + +### Naming conventions + +| Entity | Convention | Example | +|---|---|---| +| React components | PascalCase | `EventFeedTable` | +| Functions | camelCase | `translateEvent` | +| Interfaces | PascalCase | `RawEvent` | +| Constants | SCREAMING_SNAKE_CASE | `MAX_EVENTS_PER_PAGE` | +| Component files | PascalCase | `EventFeedTable.tsx` | +| Utility files | kebab-case | `registry.ts` | + +### Other rules + +- One component per file; keep files under 300 lines +- Absolute imports via the `@/` alias; no default exports from utility files +- Run `npm run format` (Prettier) before every commit + +--- + +## Commit Format + +``` +(): +``` + +| Type | When to use | +|---|---| +| `feat` | New feature or translation blueprint | +| `fix` | Bug fix | +| `docs` | Documentation only | +| `refactor` | Code restructuring, no behaviour change | +| `test` | Adding or improving tests | +| `chore` | Dependency updates, config changes | + +**Rules:** max 72 characters · present tense · no trailing period + +``` +feat(translator): add Soroswap Router swap blueprint +fix(indexer): handle empty topics array in SAC burn event +docs(contributing): add end-to-end lifecycle diagram +test(translator): add edge cases for decodeAmount with zero +``` + +--- + +## PR Procedures + +1. **Branch** off `main` using a prefix that matches your commit type: + ``` + feat/soroswap-router-blueprint + fix/sac-burn-empty-topics + docs/lifecycle-diagram + ``` + +2. **Run the full quality check** before opening a PR: + ```bash + npm test && npx tsc --noEmit && npm run lint + ``` + +3. **PR title** follows the same `(): ` format (max 72 chars). + +4. **PR description** must include `Closes #ISSUE_NUMBER` so the issue is linked automatically. + +5. **CI runs automatically** on every PR: + - `validate-registry.yml` — validates translation registry schema + - `codeql.yml` — static security analysis + - `security-scan.yml` — dependency vulnerability scan + +All checks must pass before a PR is eligible for review. + +--- + +## Pre-PR Checklist + +- [ ] `npm test` passes +- [ ] `npx tsc --noEmit` passes (no TypeScript errors) +- [ ] `npm run lint` passes (no ESLint errors) +- [ ] `npm run format` run (code is Prettier-formatted) +- [ ] PR title uses the correct `():` prefix +- [ ] PR description contains `Closes #ISSUE_NUMBER` +- [ ] No `any` types introduced +- [ ] All new top-level functions use standard declarations, not arrow functions From 7fb046318ba5b4f21f3e72958f0ecf2775b2da04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timot=C3=A9=C3=A9?= Date: Sat, 20 Jun 2026 14:52:29 +0100 Subject: [PATCH 2/7] feat(retention): configurable event archival and pruner cron MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - lib/retention/policy.ts: loads RETENTION_DAYS, RETENTION_BATCH_SIZE, RETENTION_ARCHIVE_DIR, RETENTION_CRON, RETENTION_ENABLED, RETENTION_DRY_RUN from env with safe defaults; exposes getCutoffTimestamp() - lib/retention/archiver.ts: batches aged Event rows into gzip-compressed CSV flat-files (RFC 4180) using Node built-in zlib; returns ArchiveResult manifest per batch; no-op in dry-run mode - lib/retention/pruner.ts: cursor-paginated archive→delete loop bounded by batchSize; emits structured PrunerCycleResult log per cycle; logs VACUUM advisory after deletions; schedulePruner() registers node-cron task - lib/retention/__tests__/retention.test.ts: 14 tests covering policy loading, cutoff math, CSV format/escaping, gzip validity, dry-run guard, pruner batch logic, error isolation, and result shape - scripts/retention.ts: CLI for manual invocation and audit dry-runs; supports --dry-run, --days, --batch-size, --archive-dir flags - server.ts: wires schedulePruner() into the app startup path - package.json: adds dotenv dep, @types/dotenv and @types/node-cron dev deps; adds retention and retention:dry-run npm scripts - .env.example: documents all six RETENTION_* variables with defaults - docker-compose.yml: injects RETENTION_* env vars; adds archive_data named volume so CSV archives persist outside the container --- .env.example | 19 ++ docker-compose.yml | 60 +++- lib/retention/__tests__/retention.test.ts | 336 ++++++++++++++++++++++ lib/retention/archiver.ts | 177 ++++++++++++ lib/retention/policy.ts | 76 +++++ lib/retention/pruner.ts | 252 ++++++++++++++++ package.json | 7 +- scripts/retention.ts | 189 ++++++++++++ server.ts | 4 + 9 files changed, 1116 insertions(+), 4 deletions(-) create mode 100644 lib/retention/__tests__/retention.test.ts create mode 100644 lib/retention/archiver.ts create mode 100644 lib/retention/policy.ts create mode 100644 lib/retention/pruner.ts create mode 100644 scripts/retention.ts diff --git a/.env.example b/.env.example index a367257..5fdcde1 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,22 @@ SENTRY_DSN= # Tiers: free (60 req/min) | partner (5000 req/min) # Example: OA_API_KEYS="abc123...:free:my-wallet,def456...:partner:analytics-bot" OA_API_KEYS= + +# ── Retention & Archival ────────────────────────────────────────────────────── +# Number of days of events to keep in the hot PostgreSQL table (default: 180) +RETENTION_DAYS=180 + +# Rows to archive + delete per batch — smaller values reduce lock contention (default: 500) +RETENTION_BATCH_SIZE=500 + +# Local directory for compressed CSV archive files before cold-storage upload (default: ./archives) +RETENTION_ARCHIVE_DIR=./archives + +# Cron schedule for the automated pruner — defaults to daily at 02:00 UTC +RETENTION_CRON="0 2 * * *" + +# Set to "false" to disable the automated pruner entirely (default: true) +RETENTION_ENABLED=true + +# Set to "true" to log candidates without writing archives or deleting rows +RETENTION_DRY_RUN=false diff --git a/docker-compose.yml b/docker-compose.yml index 4502143..d18966e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,68 @@ +# docker-compose.yml — full local development stack. +# +# Usage: +# cp .env.example .env.local # fill in Stellar endpoint vars +# docker compose up --build +# +# Services boot in dependency order: redis → app. +# Health checks ensure redis is accepting connections before the app starts. + services: + + # ── Redis ──────────────────────────────────────────────────────────────────── + redis: + image: redis:7-alpine + restart: unless-stopped + ports: + - "6379:6379" + # Persist data across container restarts during local development. + volumes: + - redis_data:/data + command: redis-server --save 60 1 --loglevel warning + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + # ── Application ────────────────────────────────────────────────────────────── app: build: context: . dockerfile: Dockerfile + args: + BUILDKIT_INLINE_CACHE: "1" ports: - - "3000:3000" # HTTP dashboard - - "3000:3000/tcp" # WebSocket shares the same port (/ws/events) + - "3000:3000" # HTTP dashboard + WebSocket (/ws/events on same port) env_file: - - .env.local # Provide your NEXT_PUBLIC_* vars here + - .env.local # NEXT_PUBLIC_* and SENTRY_DSN live here environment: NODE_ENV: production PORT: "3000" + # Point the app at the compose-networked Redis instance. + REDIS_URL: redis://redis:6379 + # Retention policy — override as needed for your environment. + RETENTION_DAYS: "180" + RETENTION_BATCH_SIZE: "500" + RETENTION_ARCHIVE_DIR: /app/archives + RETENTION_CRON: "0 2 * * *" + RETENTION_ENABLED: "true" + RETENTION_DRY_RUN: "false" + depends_on: + redis: + condition: service_healthy restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 40s + volumes: + # Persist CSV archives outside the container for cold-storage upload + - archive_data:/app/archives + +volumes: + redis_data: + archive_data: diff --git a/lib/retention/__tests__/retention.test.ts b/lib/retention/__tests__/retention.test.ts new file mode 100644 index 0000000..3ebbb83 --- /dev/null +++ b/lib/retention/__tests__/retention.test.ts @@ -0,0 +1,336 @@ +/** + * Retention System Tests + * + * Covers: + * - Policy loading and cutoff calculation + * - CSV archiver output format + * - Pruner cycle logic (mocked DB) + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +// ── Policy tests ────────────────────────────────────────────────────────────── + +import { + loadRetentionPolicy, + getCutoffTimestamp, + getCutoffDate, + type RetentionPolicy, +} from "../policy"; + +describe("loadRetentionPolicy", () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("returns defaults when no env vars are set", () => { + delete process.env.RETENTION_DAYS; + delete process.env.RETENTION_BATCH_SIZE; + delete process.env.RETENTION_ARCHIVE_DIR; + delete process.env.RETENTION_CRON; + delete process.env.RETENTION_ENABLED; + delete process.env.RETENTION_DRY_RUN; + + const policy = loadRetentionPolicy(); + expect(policy.retentionDays).toBe(180); + expect(policy.batchSize).toBe(500); + expect(policy.archiveDir).toBe("./archives"); + expect(policy.cronSchedule).toBe("0 2 * * *"); + expect(policy.enabled).toBe(true); + expect(policy.dryRun).toBe(false); + }); + + it("reads RETENTION_DAYS from env", () => { + process.env.RETENTION_DAYS = "90"; + expect(loadRetentionPolicy().retentionDays).toBe(90); + }); + + it("falls back to default for invalid RETENTION_DAYS", () => { + process.env.RETENTION_DAYS = "not-a-number"; + expect(loadRetentionPolicy().retentionDays).toBe(180); + }); + + it("falls back to default for zero RETENTION_DAYS", () => { + process.env.RETENTION_DAYS = "0"; + expect(loadRetentionPolicy().retentionDays).toBe(180); + }); + + it("disables when RETENTION_ENABLED=false", () => { + process.env.RETENTION_ENABLED = "false"; + expect(loadRetentionPolicy().enabled).toBe(false); + }); + + it("enables dry-run when RETENTION_DRY_RUN=true", () => { + process.env.RETENTION_DRY_RUN = "true"; + expect(loadRetentionPolicy().dryRun).toBe(true); + }); +}); + +describe("getCutoffTimestamp", () => { + it("returns a Unix timestamp in the past by retentionDays", () => { + const policy: RetentionPolicy = { + retentionDays: 30, + batchSize: 500, + archiveDir: "./archives", + cronSchedule: "0 2 * * *", + enabled: true, + dryRun: false, + }; + + const cutoff = getCutoffTimestamp(policy); + const expectedDate = new Date(); + expectedDate.setUTCDate(expectedDate.getUTCDate() - 30); + expectedDate.setUTCHours(0, 0, 0, 0); + + // Allow ±5 seconds for test execution time + expect(cutoff).toBeCloseTo(Math.floor(expectedDate.getTime() / 1000), -1); + }); + + it("cutoff is strictly in the past", () => { + const policy: RetentionPolicy = { + retentionDays: 1, + batchSize: 500, + archiveDir: "./archives", + cronSchedule: "0 2 * * *", + enabled: true, + dryRun: false, + }; + expect(getCutoffTimestamp(policy)).toBeLessThan(Math.floor(Date.now() / 1000)); + }); +}); + +// ── Archiver tests ──────────────────────────────────────────────────────────── + +import { archiveBatch, type PrismaEventRow } from "../archiver"; +import * as zlib from "zlib"; + +function makeFakeEvent(overrides: Partial = {}): PrismaEventRow { + return { + id: "evt-001", + contractId: "CABC123", + ledger: 1000, + timestamp: 1700000000, + txHash: "abc123", + topics: ["0xdeadbeef", "0xcafe"], + data: "0x1234", + description: "Transfer 100 tokens", + status: "translated", + blueprintName: "SAC", + eventType: "Transfer", + createdAt: new Date("2024-01-01T00:00:00Z"), + ...overrides, + }; +} + +describe("archiveBatch", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oa-test-")); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const basePolicy: RetentionPolicy = { + retentionDays: 180, + batchSize: 500, + archiveDir: "", + cronSchedule: "0 2 * * *", + enabled: true, + dryRun: false, + }; + + it("returns empty result for empty batch", async () => { + const policy = { ...basePolicy, archiveDir: tmpDir }; + const result = await archiveBatch([], 0, policy); + expect(result).not.toBeNull(); + expect(result!.rowCount).toBe(0); + }); + + it("writes a .csv.gz file and returns correct metadata", async () => { + const events = [makeFakeEvent({ id: "evt-001" }), makeFakeEvent({ id: "evt-002", ledger: 1001 })]; + const policy = { ...basePolicy, archiveDir: tmpDir }; + + const result = await archiveBatch(events, 0, policy); + + expect(result).not.toBeNull(); + expect(result!.rowCount).toBe(2); + expect(result!.filePath).toMatch(/\.csv\.gz$/); + expect(fs.existsSync(result!.filePath)).toBe(true); + expect(result!.compressedBytes).toBeGreaterThan(0); + }); + + it("produces a valid gzip file that decompresses to valid CSV", async () => { + const events = [makeFakeEvent()]; + const policy = { ...basePolicy, archiveDir: tmpDir }; + const result = await archiveBatch(events, 0, policy); + + const compressed = fs.readFileSync(result!.filePath); + const decompressed = zlib.gunzipSync(compressed).toString("utf8"); + + // Should have header + 1 data row + const lines = decompressed.trim().split(/\r?\n/); + expect(lines.length).toBe(2); + expect(lines[0]).toContain("id,contractId,ledger"); + expect(lines[1]).toContain("evt-001"); + expect(lines[1]).toContain("CABC123"); + }); + + it("CSV properly escapes fields containing commas", async () => { + const events = [makeFakeEvent({ description: "Transfer, 100 tokens, to Bob" })]; + const policy = { ...basePolicy, archiveDir: tmpDir }; + const result = await archiveBatch(events, 0, policy); + + const compressed = fs.readFileSync(result!.filePath); + const decompressed = zlib.gunzipSync(compressed).toString("utf8"); + + expect(decompressed).toContain('"Transfer, 100 tokens, to Bob"'); + }); + + it("does not write a file in dry-run mode", async () => { + const policy = { ...basePolicy, archiveDir: tmpDir, dryRun: true }; + const result = await archiveBatch([makeFakeEvent()], 0, policy); + + expect(result).toBeNull(); + const files = fs.readdirSync(tmpDir); + expect(files.length).toBe(0); + }); + + it("records oldest and newest timestamps correctly", async () => { + const events = [ + makeFakeEvent({ id: "e1", timestamp: 1000 }), + makeFakeEvent({ id: "e2", timestamp: 3000 }), + makeFakeEvent({ id: "e3", timestamp: 2000 }), + ]; + const policy = { ...basePolicy, archiveDir: tmpDir }; + const result = await archiveBatch(events, 0, policy); + + expect(result!.oldestTimestamp).toBe(1000); + expect(result!.newestTimestamp).toBe(3000); + }); +}); + +// ── Pruner cycle tests ──────────────────────────────────────────────────────── + +import { runPrunerCycle } from "../pruner"; + +// Mock Prisma db client +vi.mock("@/lib/db/client", () => ({ + db: { + event: { + count: vi.fn(), + findMany: vi.fn(), + deleteMany: vi.fn(), + }, + }, +})); + +import { db } from "@/lib/db/client"; + +describe("runPrunerCycle", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oa-pruner-")); + vi.clearAllMocks(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const policy: RetentionPolicy = { + retentionDays: 180, + batchSize: 3, + archiveDir: "", + cronSchedule: "0 2 * * *", + enabled: true, + dryRun: false, + }; + + it("returns zero counts when there are no candidates", async () => { + vi.mocked(db.event.count).mockResolvedValue(0); + + const result = await runPrunerCycle({ ...policy, archiveDir: tmpDir }); + + expect(result.candidateCount).toBe(0); + expect(result.archivedCount).toBe(0); + expect(result.deletedCount).toBe(0); + expect(result.batchesProcessed).toBe(0); + }); + + it("archives and deletes candidates in batches", async () => { + const fakeEvents = [ + makeFakeEvent({ id: "e1", timestamp: 100 }), + makeFakeEvent({ id: "e2", timestamp: 200 }), + ]; + + vi.mocked(db.event.count).mockResolvedValue(2); + vi.mocked(db.event.findMany) + .mockResolvedValueOnce(fakeEvents as any) + .mockResolvedValueOnce([]); // second call returns empty → loop ends + vi.mocked(db.event.deleteMany).mockResolvedValue({ count: 2 }); + + const result = await runPrunerCycle({ ...policy, archiveDir: tmpDir }); + + expect(result.candidateCount).toBe(2); + expect(result.deletedCount).toBe(2); + expect(result.archivedCount).toBe(2); + expect(result.errors).toHaveLength(0); + }); + + it("dry-run skips delete and archive write", async () => { + vi.mocked(db.event.count).mockResolvedValue(1); + vi.mocked(db.event.findMany) + .mockResolvedValueOnce([makeFakeEvent()] as any) + .mockResolvedValueOnce([]); + + const result = await runPrunerCycle({ ...policy, archiveDir: tmpDir, dryRun: true }); + + expect(db.event.deleteMany).not.toHaveBeenCalled(); + expect(result.deletedCount).toBe(0); + expect(result.dryRun).toBe(true); + // No files written + const files = fs.readdirSync(tmpDir); + expect(files.length).toBe(0); + }); + + it("records non-fatal errors and continues to next batch", async () => { + vi.mocked(db.event.count).mockResolvedValue(2); + vi.mocked(db.event.findMany) + .mockResolvedValueOnce([makeFakeEvent({ id: "e1" })] as any) + .mockResolvedValueOnce([makeFakeEvent({ id: "e2" })] as any) + .mockResolvedValueOnce([]); + // First delete throws, second succeeds + vi.mocked(db.event.deleteMany) + .mockRejectedValueOnce(new Error("DB timeout")) + .mockResolvedValueOnce({ count: 1 }); + + const result = await runPrunerCycle({ ...policy, archiveDir: tmpDir }); + + expect(result.errors).toHaveLength(1); + expect(result.errors[0].message).toContain("DB timeout"); + expect(result.deletedCount).toBe(1); // second batch succeeded + }); + + it("result contains startedAt and completedAt ISO strings", async () => { + vi.mocked(db.event.count).mockResolvedValue(0); + + const result = await runPrunerCycle({ ...policy, archiveDir: tmpDir }); + + expect(() => new Date(result.startedAt)).not.toThrow(); + expect(() => new Date(result.completedAt)).not.toThrow(); + expect(result.elapsedMs).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/lib/retention/archiver.ts b/lib/retention/archiver.ts new file mode 100644 index 0000000..00db56b --- /dev/null +++ b/lib/retention/archiver.ts @@ -0,0 +1,177 @@ +/** + * Event Archiver + * + * Extracts a batch of aged Event rows into a compressed CSV flat-file, then + * returns a manifest so the caller knows exactly which rows were archived and + * where the file landed. + * + * The CSV schema mirrors lib/export-data.ts (ExportRow) so archived files are + * compatible with the existing download format and can be replayed / verified + * with the same tooling. + * + * Compression: gzip via Node's built-in zlib — no extra runtime dependencies. + * Each archive is named: + * open-audit-archive-TZ-.csv.gz + */ + +import * as fs from "fs"; +import * as path from "path"; +import * as zlib from "zlib"; +import { pipeline } from "stream/promises"; +import { Readable } from "stream"; +import type { RetentionPolicy } from "./policy"; + +/** One row as written to the CSV archive. */ +export interface ArchiveRow { + id: string; + contractId: string; + ledger: number; + timestamp: number; + txHash: string; + topics: string; // JSON-serialised string[] + data: string; + description: string | null; + status: string; + blueprintName: string | null; + eventType: string | null; + createdAt: string; // ISO-8601 +} + +/** Result returned after a single archive operation. */ +export interface ArchiveResult { + /** Absolute path to the written .csv.gz file. */ + filePath: string; + /** Number of rows written into this archive. */ + rowCount: number; + /** Unix ms timestamps of the oldest and newest event in the batch. */ + oldestTimestamp: number; + newestTimestamp: number; + /** Byte size of the compressed file on disk. */ + compressedBytes: number; +} + +/** Minimal shape we need from Prisma's Event row. */ +export interface PrismaEventRow { + id: string; + contractId: string; + ledger: number; + timestamp: number; + txHash: string; + topics: unknown; // Prisma returns Json — we'll stringify it + data: string; + description: string | null; + status: string; + blueprintName: string | null; + eventType: string | null; + createdAt: Date; +} + +const CSV_HEADERS: Array = [ + "id", + "contractId", + "ledger", + "timestamp", + "txHash", + "topics", + "data", + "description", + "status", + "blueprintName", + "eventType", + "createdAt", +]; + +/** Escapes a CSV field per RFC 4180. */ +function escapeCSV(value: string | number | null | undefined): string { + if (value === null || value === undefined) return ""; + const str = String(value); + if (/[",\r\n]/.test(str)) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; +} + +/** Converts a PrismaEventRow to an ArchiveRow. */ +function toArchiveRow(event: PrismaEventRow): ArchiveRow { + return { + id: event.id, + contractId: event.contractId, + ledger: event.ledger, + timestamp: event.timestamp, + txHash: event.txHash, + topics: typeof event.topics === "string" ? event.topics : JSON.stringify(event.topics), + data: event.data, + description: event.description, + status: event.status, + blueprintName: event.blueprintName, + eventType: event.eventType, + createdAt: event.createdAt.toISOString(), + }; +} + +/** Serialises rows to a multi-line CSV string (header + data rows). */ +function rowsToCSV(rows: ArchiveRow[]): string { + const header = CSV_HEADERS.join(","); + const lines = rows.map((row) => + CSV_HEADERS.map((col) => escapeCSV(row[col])).join(",") + ); + return [header, ...lines].join("\r\n"); +} + +/** + * Writes `events` to a gzip-compressed CSV inside `policy.archiveDir`. + * + * @param events Prisma Event rows to archive (already fetched by caller). + * @param batchIndex Monotonic counter for the filename — avoids collisions when + * multiple batches run within the same second. + * @param policy Retention policy (used for archiveDir and dryRun flag). + * @returns ArchiveResult manifest, or null when dryRun is true. + */ +export async function archiveBatch( + events: PrismaEventRow[], + batchIndex: number, + policy: RetentionPolicy +): Promise { + if (events.length === 0) { + return { filePath: "", rowCount: 0, oldestTimestamp: 0, newestTimestamp: 0, compressedBytes: 0 }; + } + + const rows = events.map(toArchiveRow); + const csv = rowsToCSV(rows); + + const timestamps = rows.map((r) => r.timestamp); + const oldestTimestamp = Math.min(...timestamps); + const newestTimestamp = Math.max(...timestamps); + + // Build filename: open-audit-archive-2024-06-17T020001Z-0.csv.gz + const now = new Date(); + const datePart = now.toISOString().replace(/[:.]/g, "").slice(0, 15) + "Z"; + const filename = `open-audit-archive-${datePart}-${batchIndex}.csv.gz`; + + if (policy.dryRun) { + console.log( + `[retention/archiver] DRY RUN — would write ${rows.length} rows to ${filename}` + ); + return null; + } + + // Ensure the archive directory exists + const absDir = path.resolve(policy.archiveDir); + fs.mkdirSync(absDir, { recursive: true }); + const filePath = path.join(absDir, filename); + + // Stream CSV → gzip → file + const readable = Readable.from([Buffer.from(csv, "utf8")]); + const gzip = zlib.createGzip({ level: zlib.constants.Z_BEST_COMPRESSION }); + const dest = fs.createWriteStream(filePath); + + await pipeline(readable, gzip, dest); + + const { size: compressedBytes } = fs.statSync(filePath); + + console.log( + `[retention/archiver] Archived ${rows.length} rows → ${filePath} (${compressedBytes} bytes)` + ); + + return { filePath, rowCount: rows.length, oldestTimestamp, newestTimestamp, compressedBytes }; +} diff --git a/lib/retention/policy.ts b/lib/retention/policy.ts new file mode 100644 index 0000000..e4ce13b --- /dev/null +++ b/lib/retention/policy.ts @@ -0,0 +1,76 @@ +/** + * Retention Policy Configuration + * + * Reads retention settings from environment variables and exposes a typed + * config object used by both the pruner and the archiver. + * + * Environment variables: + * RETENTION_DAYS How many days of events to keep in the hot DB (default: 180) + * RETENTION_BATCH_SIZE Rows processed per delete batch (default: 500) + * RETENTION_ARCHIVE_DIR Local directory for CSV archive files (default: ./archives) + * RETENTION_CRON Cron expression for the scheduled job (default: "0 2 * * *") + * RETENTION_ENABLED Set to "false" to disable the cron entirely (default: "true") + * RETENTION_DRY_RUN Set to "true" to log what would happen without mutating data + */ + +export interface RetentionPolicy { + /** Number of days to retain events in the hot PostgreSQL table. */ + retentionDays: number; + + /** Rows to archive + delete per iteration to limit lock contention. */ + batchSize: number; + + /** Local filesystem directory where CSV archives are written before cold-storage upload. */ + archiveDir: string; + + /** node-cron compatible expression. Defaults to daily at 02:00 UTC. */ + cronSchedule: string; + + /** When false the scheduled task is registered but never executes. */ + enabled: boolean; + + /** + * Dry-run mode: scan and log candidates but skip archive write and delete. + * Useful for auditing the policy against a production database. + */ + dryRun: boolean; +} + +/** + * Loads and validates the retention policy from the process environment. + * Safe to call at module load time — only reads env, never throws. + */ +export function loadRetentionPolicy(): RetentionPolicy { + const retentionDays = parsePositiveInt(process.env.RETENTION_DAYS, 180); + const batchSize = parsePositiveInt(process.env.RETENTION_BATCH_SIZE, 500); + const archiveDir = process.env.RETENTION_ARCHIVE_DIR ?? "./archives"; + const cronSchedule = process.env.RETENTION_CRON ?? "0 2 * * *"; + const enabled = process.env.RETENTION_ENABLED !== "false"; + const dryRun = process.env.RETENTION_DRY_RUN === "true"; + + return { retentionDays, batchSize, archiveDir, cronSchedule, enabled, dryRun }; +} + +/** Parses a string to a positive integer, falling back to the default. */ +function parsePositiveInt(value: string | undefined, fallback: number): number { + if (!value) return fallback; + const parsed = parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +/** + * Returns a Date representing the cutoff point. + * Events whose `timestamp` (Unix seconds) is older than this are candidates + * for archival and deletion. + */ +export function getCutoffDate(policy: RetentionPolicy): Date { + const cutoff = new Date(); + cutoff.setUTCDate(cutoff.getUTCDate() - policy.retentionDays); + cutoff.setUTCHours(0, 0, 0, 0); // Align to midnight UTC for reproducibility + return cutoff; +} + +/** Unix-seconds equivalent of the cutoff (matches the Event.timestamp column type). */ +export function getCutoffTimestamp(policy: RetentionPolicy): number { + return Math.floor(getCutoffDate(policy).getTime() / 1000); +} diff --git a/lib/retention/pruner.ts b/lib/retention/pruner.ts new file mode 100644 index 0000000..9380d10 --- /dev/null +++ b/lib/retention/pruner.ts @@ -0,0 +1,252 @@ +/** + * Retention Pruner + * + * Stateful background routine that: + * 1. Identifies Event rows older than RETENTION_DAYS + * 2. Archives each batch to a compressed CSV flat-file (via archiver.ts) + * 3. Deletes the archived rows from PostgreSQL in small batches to avoid + * lock contention on the hot table + * 4. Emits structured execution logs for every cycle + * + * The pruner is intentionally decoupled from the cron scheduler so it can + * also be invoked directly from the CLI script or tests. + * + * Scheduling is handled by node-cron when `schedulePruner()` is called from + * the Next.js server entry point. + */ + +import { db } from "@/lib/db/client"; +import { archiveBatch } from "./archiver"; +import { loadRetentionPolicy, getCutoffTimestamp, type RetentionPolicy } from "./policy"; +import type { ArchiveResult } from "./archiver"; + +/** Summary emitted at the end of each pruner cycle. */ +export interface PrunerCycleResult { + /** ISO-8601 start time of the cycle. */ + startedAt: string; + /** ISO-8601 end time of the cycle. */ + completedAt: string; + /** Elapsed wall-clock time in milliseconds. */ + elapsedMs: number; + /** Total event rows that qualified for archival. */ + candidateCount: number; + /** Total rows successfully archived and deleted. */ + archivedCount: number; + /** Total rows deleted from the Event table. */ + deletedCount: number; + /** Archive files written during this cycle. */ + archives: ArchiveResult[]; + /** Number of batches processed. */ + batchesProcessed: number; + /** Any non-fatal errors encountered per batch. */ + errors: Array<{ batchIndex: number; message: string }>; + /** Whether the cycle ran in dry-run mode (no mutations). */ + dryRun: boolean; +} + +/** + * Runs a single pruner cycle synchronously from the perspective of the caller. + * Each batch is archived then deleted before moving to the next, keeping peak + * memory usage bounded to `policy.batchSize` rows. + */ +export async function runPrunerCycle( + policyOverride?: Partial +): Promise { + const policy: RetentionPolicy = { ...loadRetentionPolicy(), ...policyOverride }; + const startedAt = new Date().toISOString(); + const startMs = Date.now(); + + const cutoffTimestamp = getCutoffTimestamp(policy); + const cutoffDate = new Date(cutoffTimestamp * 1000).toISOString(); + + console.log( + `[retention/pruner] Starting cycle — cutoff=${cutoffDate} ` + + `retentionDays=${policy.retentionDays} batchSize=${policy.batchSize} ` + + `dryRun=${policy.dryRun}` + ); + + // ── Phase 1: Count candidates ────────────────────────────────────────────── + const candidateCount = await db.event.count({ + where: { timestamp: { lt: cutoffTimestamp } }, + }); + + console.log(`[retention/pruner] ${candidateCount} candidate rows found`); + + if (candidateCount === 0) { + const completedAt = new Date().toISOString(); + return { + startedAt, + completedAt, + elapsedMs: Date.now() - startMs, + candidateCount: 0, + archivedCount: 0, + deletedCount: 0, + archives: [], + batchesProcessed: 0, + errors: [], + dryRun: policy.dryRun, + }; + } + + // ── Phase 2: Batch archive → delete loop ────────────────────────────────── + const archives: ArchiveResult[] = []; + const errors: Array<{ batchIndex: number; message: string }> = []; + let archivedCount = 0; + let deletedCount = 0; + let batchIndex = 0; + + // We process in cursor-style pages by taking the top N by (timestamp, id) + // so the loop is safe even if rows are inserted while we're running. + let lastId: string | null = null; + + while (true) { + // Fetch the next batch — ordered by timestamp ASC, id ASC for stable pagination + const batch = await db.event.findMany({ + where: { + timestamp: { lt: cutoffTimestamp }, + ...(lastId ? { id: { gt: lastId } } : {}), + }, + orderBy: [{ timestamp: "asc" }, { id: "asc" }], + take: policy.batchSize, + select: { + id: true, + contractId: true, + ledger: true, + timestamp: true, + txHash: true, + topics: true, + data: true, + description: true, + status: true, + blueprintName: true, + eventType: true, + createdAt: true, + }, + }); + + if (batch.length === 0) break; + + try { + // Archive this batch to a compressed CSV + const archiveResult = await archiveBatch(batch, batchIndex, policy); + + if (archiveResult && archiveResult.rowCount > 0) { + archives.push(archiveResult); + archivedCount += archiveResult.rowCount; + } + + // Delete the archived rows from PostgreSQL + if (!policy.dryRun) { + const ids = batch.map((e) => e.id); + const deleteResult = await db.event.deleteMany({ + where: { id: { in: ids } }, + }); + deletedCount += deleteResult.count; + + console.log( + `[retention/pruner] Batch ${batchIndex + 1}: archived=${batch.length} deleted=${deleteResult.count}` + ); + } else { + console.log( + `[retention/pruner] DRY RUN — Batch ${batchIndex + 1}: would delete ${batch.length} rows` + ); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[retention/pruner] Error in batch ${batchIndex}: ${message}`); + errors.push({ batchIndex, message }); + // Continue to next batch — don't let one bad batch abort the whole cycle + } + + lastId = batch[batch.length - 1].id; + batchIndex++; + + // If the batch was smaller than requested, we've exhausted all candidates + if (batch.length < policy.batchSize) break; + } + + // ── Phase 3: Log VACUUM hint ─────────────────────────────────────────────── + // Postgres doesn't reclaim page space immediately after DELETE; VACUUM does. + // We don't run VACUUM ourselves (it requires superuser in some configs and + // autovacuum handles it in most PG deployments) but we log the advisory. + if (!policy.dryRun && deletedCount > 0) { + console.log( + `[retention/pruner] ${deletedCount} rows deleted. ` + + `autovacuum will reclaim table space. ` + + `Run "VACUUM ANALYZE public.\\"Event\\";" manually if index bloat is observed.` + ); + } + + const completedAt = new Date().toISOString(); + const result: PrunerCycleResult = { + startedAt, + completedAt, + elapsedMs: Date.now() - startMs, + candidateCount, + archivedCount, + deletedCount, + archives, + batchesProcessed: batchIndex, + errors, + dryRun: policy.dryRun, + }; + + console.log( + `[retention/pruner] Cycle complete — ` + + `elapsed=${result.elapsedMs}ms candidates=${candidateCount} ` + + `archived=${archivedCount} deleted=${deletedCount} errors=${errors.length}` + ); + + return result; +} + +/** + * Registers the pruner as a recurring cron job using node-cron. + * + * Call this once from the application server entry point (server.ts). + * The job runs at the time specified by `RETENTION_CRON` (default 02:00 UTC daily). + * + * If `RETENTION_ENABLED=false`, the function is a no-op. + * + * @returns A stop function that cancels the scheduled task. + */ +export function schedulePruner(): () => void { + const policy = loadRetentionPolicy(); + + if (!policy.enabled) { + console.log("[retention/pruner] Retention is disabled (RETENTION_ENABLED=false). Skipping."); + return () => {}; + } + + // Lazy-import node-cron so the module is only loaded when scheduling is needed. + // eslint-disable-next-line @typescript-eslint/no-var-requires + const cron = require("node-cron"); + + if (!cron.validate(policy.cronSchedule)) { + console.error( + `[retention/pruner] Invalid RETENTION_CRON expression: "${policy.cronSchedule}". ` + + `Pruner will NOT be scheduled.` + ); + return () => {}; + } + + console.log( + `[retention/pruner] Scheduled — cron="${policy.cronSchedule}" ` + + `retentionDays=${policy.retentionDays} dryRun=${policy.dryRun}` + ); + + const task = cron.schedule(policy.cronSchedule, async () => { + console.log("[retention/pruner] Cron triggered — starting cycle..."); + try { + const result = await runPrunerCycle(); + console.log("[retention/pruner] Cron cycle finished:", JSON.stringify(result, null, 2)); + } catch (err) { + console.error("[retention/pruner] Unhandled error in cron cycle:", err); + } + }); + + return () => { + task.stop(); + console.log("[retention/pruner] Scheduled task stopped."); + }; +} diff --git a/package.json b/package.json index a9416ed..2c1ba91 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,9 @@ "db:migrate": "prisma migrate dev", "db:generate": "prisma generate", "db:seed": "ts-node prisma/seed.ts", - "db:studio": "prisma studio" + "db:studio": "prisma studio", + "retention": "ts-node --project tsconfig.server.json scripts/retention.ts", + "retention:dry-run": "ts-node --project tsconfig.server.json scripts/retention.ts --dry-run" }, "dependencies": { "@prisma/client": "^5.8.0", @@ -29,6 +31,7 @@ "ioredis": "^5.11.1", "lucide-react": "^0.378.0", "next": "14.2.3", + "dotenv": "^16.4.5", "node-cron": "^3.0.3", "pino": "^8.17.2", "react": "^18", @@ -44,9 +47,11 @@ "devDependencies": { "@prisma/cli": "^5.8.0", "@types/bull": "^4.10.8", + "@types/dotenv": "^8.2.0", "@types/ioredis": "^4.28.10", "@types/js-yaml": "^4.0.9", "@types/node": "^20", + "@types/node-cron": "^3.0.11", "@types/react": "^18", "@types/react-dom": "^18", "@types/swagger-ui-react": "^5.18.0", diff --git a/scripts/retention.ts b/scripts/retention.ts new file mode 100644 index 0000000..a845262 --- /dev/null +++ b/scripts/retention.ts @@ -0,0 +1,189 @@ +#!/usr/bin/env ts-node +/** + * Retention Pruner CLI + * + * Manually trigger a retention cycle or run a dry-run audit from the terminal. + * + * Usage: + * npx ts-node scripts/retention.ts [options] + * + * Options: + * --dry-run Scan and log candidates without writing archives or deleting rows + * --days Override RETENTION_DAYS for this run + * --batch-size Override RETENTION_BATCH_SIZE for this run + * --archive-dir Override RETENTION_ARCHIVE_DIR for this run + * --help Print this message + * + * Environment variables (can also be set in .env.local): + * DATABASE_URL PostgreSQL connection string (required) + * RETENTION_DAYS Days of events to retain (default: 180) + * RETENTION_BATCH_SIZE Rows per batch (default: 500) + * RETENTION_ARCHIVE_DIR Output directory for archives (default: ./archives) + * RETENTION_DRY_RUN Set "true" to enable dry-run globally + * + * Examples: + * # Dry-run to see what would be pruned + * npx ts-node scripts/retention.ts --dry-run + * + * # Prune events older than 90 days + * npx ts-node scripts/retention.ts --days 90 + * + * # Prune with smaller batches to reduce lock pressure + * npx ts-node scripts/retention.ts --days 180 --batch-size 100 + */ + +// Load .env.local before anything else +import * as dotenv from "dotenv"; +import * as path from "path"; +dotenv.config({ path: path.resolve(process.cwd(), ".env.local") }); + +import { runPrunerCycle } from "../lib/retention/pruner"; +import { loadRetentionPolicy } from "../lib/retention/policy"; +import type { RetentionPolicy } from "../lib/retention/policy"; + +interface CliArgs { + dryRun: boolean; + days?: number; + batchSize?: number; + archiveDir?: string; + help: boolean; +} + +function parseArgs(argv: string[]): CliArgs { + const args: CliArgs = { dryRun: false, help: false }; + + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + + switch (flag) { + case "--dry-run": + args.dryRun = true; + break; + + case "--days": { + const val = parseInt(argv[++i], 10); + if (!Number.isFinite(val) || val < 1) { + console.error("--days must be a positive integer"); + process.exit(1); + } + args.days = val; + break; + } + + case "--batch-size": { + const val = parseInt(argv[++i], 10); + if (!Number.isFinite(val) || val < 1) { + console.error("--batch-size must be a positive integer"); + process.exit(1); + } + args.batchSize = val; + break; + } + + case "--archive-dir": + args.archiveDir = argv[++i]; + break; + + case "--help": + case "-h": + args.help = true; + break; + + default: + console.warn(`Unknown flag: ${flag}`); + } + } + + return args; +} + +function printHelp(): void { + console.log(` +Retention Pruner CLI + +Usage: + npx ts-node scripts/retention.ts [options] + +Options: + --dry-run Scan and log candidates without writing archives or deleting rows + --days Override RETENTION_DAYS for this run + --batch-size Override RETENTION_BATCH_SIZE for this run + --archive-dir Override RETENTION_ARCHIVE_DIR for this run + --help Print this message + +Environment variables: + DATABASE_URL PostgreSQL connection string (required) + RETENTION_DAYS Days of events to retain (default: 180) + RETENTION_BATCH_SIZE Rows per batch (default: 500) + RETENTION_ARCHIVE_DIR Output directory for archives (default: ./archives) + RETENTION_DRY_RUN Set "true" to enable dry-run mode globally + +Examples: + npx ts-node scripts/retention.ts --dry-run + npx ts-node scripts/retention.ts --days 90 --archive-dir /mnt/cold-storage +`); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + + if (args.help) { + printHelp(); + process.exit(0); + } + + // Build override from CLI args + const override: Partial = {}; + if (args.dryRun) override.dryRun = true; + if (args.days !== undefined) override.retentionDays = args.days; + if (args.batchSize !== undefined) override.batchSize = args.batchSize; + if (args.archiveDir !== undefined) override.archiveDir = args.archiveDir; + + const effectivePolicy = { ...loadRetentionPolicy(), ...override }; + + console.log("\n[retention/cli] Effective policy:"); + console.log(` retentionDays : ${effectivePolicy.retentionDays}`); + console.log(` batchSize : ${effectivePolicy.batchSize}`); + console.log(` archiveDir : ${effectivePolicy.archiveDir}`); + console.log(` dryRun : ${effectivePolicy.dryRun}`); + console.log(` enabled : ${effectivePolicy.enabled}`); + console.log(""); + + try { + const result = await runPrunerCycle(override); + + console.log("\n[retention/cli] Cycle result:"); + console.log(` Started : ${result.startedAt}`); + console.log(` Completed : ${result.completedAt}`); + console.log(` Elapsed : ${result.elapsedMs}ms`); + console.log(` Candidates : ${result.candidateCount}`); + console.log(` Archived rows : ${result.archivedCount}`); + console.log(` Deleted rows : ${result.deletedCount}`); + console.log(` Batches : ${result.batchesProcessed}`); + console.log(` Errors : ${result.errors.length}`); + + if (result.archives.length > 0) { + console.log("\n Archive files:"); + for (const archive of result.archives) { + console.log( + ` ${archive.filePath} (${archive.rowCount} rows, ${archive.compressedBytes} bytes)` + ); + } + } + + if (result.errors.length > 0) { + console.log("\n Errors:"); + for (const err of result.errors) { + console.error(` Batch ${err.batchIndex}: ${err.message}`); + } + process.exit(1); + } + + process.exit(0); + } catch (err) { + console.error("\n[retention/cli] Fatal error:", err); + process.exit(1); + } +} + +main(); diff --git a/server.ts b/server.ts index 96b924b..cc09242 100644 --- a/server.ts +++ b/server.ts @@ -14,6 +14,7 @@ import { translateEvent } from "./lib/translator/registry"; import { startHorizonStreamingIndexer } from "./lib/stellar/indexer"; import { getNetworkConfig } from "./lib/stellar/client"; import { captureExceptionSync } from "./lib/telemetry"; +import { schedulePruner } from "./lib/retention/pruner"; const dev = process.env.NODE_ENV !== "production"; const port = parseInt(process.env.PORT ?? "3000", 10); @@ -58,6 +59,9 @@ app.prepare().then(() => { }, }); + // Start the retention pruner cron (no-op if RETENTION_ENABLED=false) + schedulePruner(); + httpServer.listen(port, () => { console.log(`> Ready on http://localhost:${port}`); }); From 7e08e31b71762ca83b65ad910fa6dd0d10676ae2 Mon Sep 17 00:00:00 2001 From: OluwapelumiElisha Date: Sun, 21 Jun 2026 03:48:52 +0100 Subject: [PATCH 3/7] ui(dashboard): Enhance accessibility (a11y) and keyboard navigation compliance across the log viewer grid --- components/dashboard/ContributeDialog.tsx | 2 +- components/dashboard/EventFeedTable.test.tsx | 66 ++++++++++++++++++++ components/dashboard/EventFeedTable.tsx | 31 +++++++-- components/dashboard/RawDataDialog.tsx | 8 ++- vitest.config.ts | 2 +- vitest.setup.ts | 6 +- 6 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 components/dashboard/EventFeedTable.test.tsx diff --git a/components/dashboard/ContributeDialog.tsx b/components/dashboard/ContributeDialog.tsx index d04173c..7725822 100644 --- a/components/dashboard/ContributeDialog.tsx +++ b/components/dashboard/ContributeDialog.tsx @@ -40,7 +40,7 @@ export function ContributeDialog({
{/* Issue label */} -
+
High Complexity Issue Stellar Drips Eligible
diff --git a/components/dashboard/EventFeedTable.test.tsx b/components/dashboard/EventFeedTable.test.tsx new file mode 100644 index 0000000..134737c --- /dev/null +++ b/components/dashboard/EventFeedTable.test.tsx @@ -0,0 +1,66 @@ +import { render } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import { axe } from "vitest-axe"; +import { EventFeedTable } from "./EventFeedTable"; + +describe("EventFeedTable Accessibility", () => { + it("should have no accessibility violations", async () => { + const mockEvents = [ + { + status: "translated" as const, + description: "Transferred 100 XLM to Bob", + eventType: "transfer", + raw: { + id: "1", + type: "contract", + ledger: 123456, + ledgerClosedAt: "2026-06-17T17:11:21Z", + contractId: "CAAA...D2KM", + pagingToken: "token", + txHash: "hash123", + topics: ["topic1"], + data: "data123", + timestamp: Date.now() / 1000 - 3600, + }, + }, + { + status: "cryptic" as const, + description: "", + eventType: "", + raw: { + id: "2", + type: "contract", + ledger: 123456, + ledgerClosedAt: "2026-06-17T17:11:21Z", + contractId: "CAAA...D2KM", + pagingToken: "token", + txHash: "hash456", + topics: ["topic2"], + data: "data456", + timestamp: Date.now() / 1000 - 7200, + }, + }, + ]; + + const columns = { + status: true, + time: true, + description: true, + contract: true, + actions: true, + }; + + const { container } = render( + + ); + + const results = await axe(container); + expect(results).toHaveNoViolations(); + }); +}); diff --git a/components/dashboard/EventFeedTable.tsx b/components/dashboard/EventFeedTable.tsx index bf0c1fb..5ee60c5 100644 --- a/components/dashboard/EventFeedTable.tsx +++ b/components/dashboard/EventFeedTable.tsx @@ -40,7 +40,8 @@ function StatusBadge({ status }: { status: TranslatedEvent["status"] }): React.J if (status === "translated") { return ( - + ); @@ -48,14 +49,16 @@ function StatusBadge({ status }: { status: TranslatedEvent["status"] }): React.J if (status === "pending") { return ( - + ); } return ( - + ); @@ -88,6 +91,22 @@ export function EventFeedTable({ const [contributeDialogEvent, setContributeDialogEvent] = useState(null); const [showColMenu, setShowColMenu] = useState(false); + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.target instanceof HTMLElement && e.target.tagName === "TR") { + const currentRow = e.target as HTMLTableRowElement; + + if (e.key === "ArrowDown" || e.key === "j" || e.key === "J") { + e.preventDefault(); + const nextRow = currentRow.nextElementSibling as HTMLTableRowElement; + if (nextRow) nextRow.focus(); + } else if (e.key === "ArrowUp" || e.key === "k" || e.key === "K") { + e.preventDefault(); + const prevRow = currentRow.previousElementSibling as HTMLTableRowElement; + if (prevRow) prevRow.focus(); + } + } + }; + const cellPadding = density === "compact" ? "py-1.5" : "py-3"; const visibleColCount = Object.values(columns).filter(Boolean).length; @@ -164,7 +183,7 @@ export function EventFeedTable({ )} - + {isLoading ? Array.from({ length: 5 }).map(function (_, i) { return ; @@ -175,7 +194,9 @@ export function EventFeedTable({ return ( - {copied ? : } + + {copied ? "Copied" : ""} + + {copied ?