BOUNTY: AWS Comprehend PII redactor (issue #290)

JS/edgechains/arakoodev/package.json

@@ -1,79 +1,81 @@
 {
-    "name": "@arakoodev/edgechains.js",
-    "version": "0.1.23",
-    "main": "dist/index.js",
-    "files": [
-        "dist"
-    ],
-    "exports": {
-        "./ai": "./dist/ai/src/index.js",
-        "./vector-db": "./dist/vector-db/src/index.js",
-        "./document-loader": "./dist/document-loader/src/index.js",
-        "./splitter": "./dist/splitter/src/index.js",
-        "./arakooserver": "./dist/arakooserver/src/index.js",
-        "./db": "./dist/db/src/index.js",
-        "./scraper": "./dist/scraper/src/index.js",
-        "./sync-rpc": "./dist/sync-rpc/index.js"
-    },
-    "scripts": {
-        "build": "rm -rf dist && tsc -b",
-        "lint": "eslint --ignore-path .eslintignore --ext .js,.ts",
-        "format": "prettier --ignore-path .gitignore --write \"**/*.+(js|ts|json)\"",
-        "test": "vitest"
-    },
-    "dependencies": {
-        "@babel/core": "^7.24.4",
-        "@babel/preset-env": "^7.24.4",
-        "@hono/node-server": "^0.6.0",
-        "@lifeomic/attempt": "^3.1.0",
-        "@playwright/test": "^1.45.3",
-        "@supabase/supabase-js": "^2.42.3",
-        "axios": "^1.7.4",
-        "axios-retry": "^4.1.0",
-        "cheerio": "^1.0.0-rc.12",
-        "cors": "^2.8.5",
-        "document": "^0.4.7",
-        "dts-bundle-generator": "^9.3.1",
-        "esbuild": "^0.20.2",
-        "hono": "3.9",
-        "jest-environment-jsdom": "^29.7.0",
-        "jsdom": "^24.1.0",
-        "node-html-parser": "^6.1.13",
-        "pdf-parse": "^1.1.1",
-        "pg": "^8.11.5",
-        "playwright": "^1.45.1",
-        "prettier": "^3.2.5",
-        "regenerator-runtime": "^0.14.1",
-        "request": "^2.88.2",
-        "retell-client-js-sdk": "^2.0.4",
-        "retell-sdk": "^4.9.0",
-        "retry": "^0.13.1",
-        "ts-node": "^10.9.2",
-        "typeorm": "^0.3.20",
-        "vitest": "^2.0.3",
-        "youtube-transcript": "^1.2.1",
-        "zod": "^3.23.8",
-        "zod-to-json-schema": "^3.23.0"
-    },
-    "keywords": [],
-    "author": "",
-    "license": "ISC",
-    "devDependencies": {
-        "@babel/preset-typescript": "^7.24.1",
-        "@types/cors": "^2.8.17",
-        "@types/jest": "^29.5.12",
-        "@types/node": "^20.17.2",
-        "@types/pdf-parse": "^1.1.4",
-        "@types/ws": "^8.5.12",
-        "buffer": "^6.0.3",
-        "crypto-browserify": "^3.12.1",
-        "jest": "^29.7.0",
-        "process": "^0.11.10",
-        "stream-browserify": "^3.0.0",
-        "stream-http": "^3.2.0",
-        "ts-jest": "^29.1.2",
-        "ts-loader": "^9.5.1",
-        "typescript": "^5.6.3",
-        "util": "^0.12.5"
-    }
+  "name": "@arakoodev/edgechains.js",
+  "version": "0.1.23",
+  "main": "dist/index.js",
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    "./ai": "./dist/ai/src/index.js",
+    "./vector-db": "./dist/vector-db/src/index.js",
+    "./document-loader": "./dist/document-loader/src/index.js",
+    "./splitter": "./dist/splitter/src/index.js",
+    "./arakooserver": "./dist/arakooserver/src/index.js",
+    "./db": "./dist/db/src/index.js",
+    "./scraper": "./dist/scraper/src/index.js",
+    "./sync-rpc": "./dist/sync-rpc/index.js"
+  },
+  "scripts": {
+    "build": "rm -rf dist && tsc -b",
+    "lint": "eslint --ignore-path .eslintignore --ext .js,.ts",
+    "format": "prettier --ignore-path .gitignore --write \"**/*.+(js|ts|json)\"",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "@babel/core": "^7.24.4",
+    "@babel/preset-env": "^7.24.4",
+    "@hono/node-server": "^0.6.0",
+    "@lifeomic/attempt": "^3.1.0",
+    "@playwright/test": "^1.45.3",
+    "@supabase/supabase-js": "^2.42.3",
+    "axios": "^1.7.4",
+    "axios-retry": "^4.1.0",
+    "cheerio": "^1.0.0-rc.12",
+    "cors": "^2.8.5",
+    "document": "^0.4.7",
+    "dts-bundle-generator": "^9.3.1",
+    "esbuild": "^0.20.2",
+    "hono": "3.9",
+    "jest-environment-jsdom": "^29.7.0",
+    "jsdom": "^24.1.0",
+    "node-html-parser": "^6.1.13",
+    "pdf-parse": "^1.1.1",
+    "pg": "^8.11.5",
+    "playwright": "^1.45.1",
+    "prettier": "^3.2.5",
+    "regenerator-runtime": "^0.14.1",
+    "request": "^2.88.2",
+    "retell-client-js-sdk": "^2.0.4",
+    "retell-sdk": "^4.9.0",
+    "retry": "^0.13.1",
+    "ts-node": "^10.9.2",
+    "typeorm": "^0.3.20",
+    "vitest": "^2.0.3",
+    "youtube-transcript": "^1.2.1",
+    "zod": "^3.23.8",
+    "zod-to-json-schema": "^3.23.0",
+    "@aws-sdk/client-comprehend": "^3.600.0",
+    "rxjs": "^7.8.1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@babel/preset-typescript": "^7.24.1",
+    "@types/cors": "^2.8.17",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.17.2",
+    "@types/pdf-parse": "^1.1.4",
+    "@types/ws": "^8.5.12",
+    "buffer": "^6.0.3",
+    "crypto-browserify": "^3.12.1",
+    "jest": "^29.7.0",
+    "process": "^0.11.10",
+    "stream-browserify": "^3.0.0",
+    "stream-http": "^3.2.0",
+    "ts-jest": "^29.1.2",
+    "ts-loader": "^9.5.1",
+    "typescript": "^5.6.3",
+    "util": "^0.12.5"
+  }
 }

JS/edgechains/arakoodev/src/ai/src/index.ts

@@ -3,3 +3,4 @@ export { GeminiAI } from "./lib/gemini/gemini.js";
 export { LlamaAI } from "./lib/llama/llama.js";
 export { RetellAI } from "./lib/retell-ai/retell.js";
 export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js";
+export { ComprehendRedactor } from "./lib/comprehend-redactor/index.js";

JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/ComprehendRedactor.ts

@@ -0,0 +1,112 @@
+import { Observable } from "rxjs";
+
+export type PiiEntityType =
+  | "ADDRESS" | "AGE" | "AWS_ACCESS_KEY" | "AWS_SECRET_KEY" | "BANK_ACCOUNT_NUMBER"
+  | "BANK_ROUTING" | "CREDIT_DEBIT_CVV" | "CREDIT_DEBIT_EXPIRY" | "CREDIT_DEBIT_NUMBER"
+  | "DATE_TIME" | "DRIVER_ID" | "EMAIL" | "INTERNATIONAL_BANK_ACCOUNT_NUMBER"
+  | "IP_ADDRESS" | "LICENSE_PLATE" | "MAC_ADDRESS" | "NAME" | "PASSPORT_NUMBER"
+  | "PASSWORD" | "PHONE" | "PIN" | "SSN" | "URL" | "USERNAME" | "VEHICLE_IDENTIFICATION_NUMBER";
+
+export interface ComprehendRedactorConfig {
+  region?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+}
+
+export interface RedactResult {
+  original: string;
+  redacted: string;
+  piiEntities: Array<{ type: string; score: number }>;
+}
+
+function maskPii(text: string, piiEntities: Array<{ type: string; score: number; beginOffset: number; endOffset: number }>): string {
+  let result = text;
+  const sorted = [...piiEntities].sort((a, b) => b.beginOffset - a.beginOffset);
+  for (const entity of sorted) {
+    const mask = `[${entity.type}]`;
+    result = result.slice(0, entity.beginOffset) + mask + result.slice(entity.endOffset);
+  }
+  return result;
+}
+
+export class ComprehendRedactor {
+  private config: ComprehendRedactorConfig;
+
+  constructor(config: ComprehendRedactorConfig = {}) {
+    this.config = {
+      region: config.region || process.env.AWS_REGION || "us-east-1",
+      accessKeyId: config.accessKeyId || process.env.AWS_ACCESS_KEY_ID || "",
+      secretAccessKey: config.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY || "",
+    };
+  }
+
+  async redact(text: string, piiTypes?: PiiEntityType[]): Promise<RedactResult> {
+    const entities = await this.detectPii(text, piiTypes);
+    const redacted = maskPii(text, entities);
+    return {
+      original: text,
+      redacted,
+      piiEntities: entities.map(e => ({ type: e.type, score: e.score })),
+    };
+  }
+
+  redact$(text: string, piiTypes?: PiiEntityType[]): Observable<RedactResult> {
+    return new Observable(subscriber => {
+      this.redact(text, piiTypes)
+        .then(result => {
+          subscriber.next(result);
+          subscriber.complete();
+        })
+        .catch(err => subscriber.error(err));
+    });
+  }
+
+  mask$(source: Observable<string>, piiTypes?: PiiEntityType[]): Observable<string> {
+    return new Observable(subscriber => {
+      source.subscribe({
+        next: async (text) => {
+          try {
+            const result = await this.redact(text, piiTypes);
+            subscriber.next(result.redacted);
+          } catch (err) {
+            subscriber.error(err);
+          }
+        },
+        error: (err) => subscriber.error(err),
+        complete: () => subscriber.complete(),
+      });
+    });
+  }
+
+  private async detectPii(
+    text: string,
+    piiTypes?: PiiEntityType[],
+  ): Promise<Array<{ type: string; score: number; beginOffset: number; endOffset: number }>> {
+    const { ComprehendClient, DetectPiiEntitiesCommand } = await import("@aws-sdk/client-comprehend");
+
+    const client = new ComprehendClient({
+      region: this.config.region,
+      credentials: {
+        accessKeyId: this.config.accessKeyId,
+        secretAccessKey: this.config.secretAccessKey,
+      },
+    });
+
+    const command = new DetectPiiEntitiesCommand({
+      Text: text,
+      LanguageCode: "en",
+    });
+
+    const response = await client.send(command);
+    const entities = (response.Entities || [])
+      .filter(e => !piiTypes || piiTypes.includes(e.Type as PiiEntityType))
+      .map(e => ({
+        type: e.Type || "UNKNOWN",
+        score: e.Score || 0,
+        beginOffset: e.BeginOffset || 0,
+        endOffset: e.EndOffset || 0,
+      }));
+
+    return entities;
+  }
+}

JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/index.ts

@@ -0,0 +1,2 @@
+export { ComprehendRedactor } from "./ComprehendRedactor";
+export type { ComprehendRedactorConfig, RedactResult, PiiEntityType } from "./ComprehendRedactor";

JS/edgechains/arakoodev/src/ai/src/tests/comprehendRedactor.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi } from "vitest";
+import { ComprehendRedactor } from "../lib/comprehend-redactor/ComprehendRedactor";
+
+vi.mock("@aws-sdk/client-comprehend", () => {
+  const mockSend = vi.fn();
+  return {
+    ComprehendClient: vi.fn(() => ({
+      send: mockSend,
+    })),
+    DetectPiiEntitiesCommand: vi.fn(),
+    __mockSend: mockSend,
+  };
+});
+
+describe("ComprehendRedactor", () => {
+  it("should redact PII from text", async () => {
+    const { __mockSend } = await import("@aws-sdk/client-comprehend");
+
+    __mockSend.mockResolvedValueOnce({
+      Entities: [
+        { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 },
+        { Type: "PHONE", Score: 0.95, BeginOffset: 18, EndOffset: 30 },
+      ],
+    });
+
+    const redactor = new ComprehendRedactor({
+      accessKeyId: "test-key",
+      secretAccessKey: "test-secret",
+    });
+
+    const result = await redactor.redact("user@example.com 123-456-7890");
+
+    expect(result.redacted).toContain("[EMAIL]");
+    expect(result.redacted).toContain("[PHONE]");
+    expect(result.piiEntities).toHaveLength(2);
+    expect(result.piiEntities[0].type).toBe("EMAIL");
+    expect(result.piiEntities[1].type).toBe("PHONE");
+  });
+
+  it("should filter by PII types", async () => {
+    const { __mockSend } = await import("@aws-sdk/client-comprehend");
+
+    __mockSend.mockResolvedValueOnce({
+      Entities: [
+        { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 },
+        { Type: "PHONE", Score: 0.95, BeginOffset: 18, EndOffset: 30 },
+      ],
+    });
+
+    const redactor = new ComprehendRedactor({});
+    const result = await redactor.redact("user@example.com 123-456-7890", ["PHONE"]);
+
+    expect(result.redacted).toBe("user@example.com [PHONE]");
+    expect(result.piiEntities).toHaveLength(1);
+    expect(result.piiEntities[0].type).toBe("PHONE");
+  });
+
+  it("should emit redacted text via observable", async () => {
+    const { __mockSend } = await import("@aws-sdk/client-comprehend");
+
+    __mockSend.mockResolvedValueOnce({
+      Entities: [
+        { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 },
+      ],
+    });
+
+    const redactor = new ComprehendRedactor({});
+    const result = await new Promise<any>((resolve, reject) => {
+      redactor.redact$("user@example.com").subscribe({
+        next: resolve,
+        error: reject,
+      });
+    });
+
+    expect(result.redacted).toContain("[EMAIL]");
+  });
+
+  it("should chain with mask$ observable", async () => {
+    const { __mockSend } = await import("@aws-sdk/client-comprehend");
+    const { of } = await import("rxjs");
+
+    __mockSend.mockResolvedValueOnce({
+      Entities: [
+        { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 },
+      ],
+    });
+
+    const redactor = new ComprehendRedactor({});
+    const source$ = of("contact: user@example.com");
+
+    const result = await new Promise<string>((resolve, reject) => {
+      redactor.mask$(source$).subscribe({
+        next: resolve,
+        error: reject,
+      });
+    });
+
+    expect(result).toContain("[EMAIL]");
+    expect(result).not.toContain("user@example.com");
+  });
+});