From 97ce0bf923b19aa78f757ff425b8ba08bd7f0938 Mon Sep 17 00:00:00 2001 From: zhaoziyuan2024 Date: Thu, 9 Jul 2026 21:04:50 +1000 Subject: [PATCH 1/3] Add Qdrant vector database client --- JS/edgechains/arakoodev/package.json | 1 + .../arakoodev/src/vector-db/src/index.ts | 1 + .../src/vector-db/src/lib/qdrant/qdrant.ts | 388 ++++++++++++++++++ .../vector-db/src/tests/qdrant/qdrant.test.ts | 142 +++++++ 4 files changed, 532 insertions(+) create mode 100644 JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts create mode 100644 JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts diff --git a/JS/edgechains/arakoodev/package.json b/JS/edgechains/arakoodev/package.json index 0b0bd378..5d4bda79 100644 --- a/JS/edgechains/arakoodev/package.json +++ b/JS/edgechains/arakoodev/package.json @@ -33,6 +33,7 @@ "cheerio": "^1.0.0-rc.12", "cors": "^2.8.5", "document": "^0.4.7", + "dotenv": "^16.4.7", "dts-bundle-generator": "^9.3.1", "esbuild": "^0.20.2", "hono": "3.9", diff --git a/JS/edgechains/arakoodev/src/vector-db/src/index.ts b/JS/edgechains/arakoodev/src/vector-db/src/index.ts index 557104a1..e68d60f2 100644 --- a/JS/edgechains/arakoodev/src/vector-db/src/index.ts +++ b/JS/edgechains/arakoodev/src/vector-db/src/index.ts @@ -1 +1,2 @@ export { Supabase } from "./lib/supabase/supabase.js"; +export { Qdrant } from "./lib/qdrant/qdrant.js"; diff --git a/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts new file mode 100644 index 00000000..9a9f1d65 --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts @@ -0,0 +1,388 @@ +import axios, { AxiosInstance } from "axios"; +import retry from "retry"; +import { config } from "dotenv"; +config(); + +type QdrantPointId = number | string; +type QdrantVector = number[] | Record; +type QdrantDistance = "Cosine" | "Euclid" | "Dot" | "Manhattan"; + +interface QdrantClientArgs { + client?: AxiosInstance; +} + +interface QdrantCollectionArgs extends QdrantClientArgs { + collectionName: string; +} + +interface QdrantQueryArgs extends QdrantCollectionArgs { + consistency?: number | "majority" | "quorum" | "all"; + timeout?: number; +} + +interface CreateCollectionArgs extends QdrantCollectionArgs { + vectorSize?: number; + distance?: QdrantDistance; + vectors?: object; + shardNumber?: number; + replicationFactor?: number; + writeConsistencyFactor?: number; + onDiskPayload?: boolean; + hnswConfig?: object; + walConfig?: object; + optimizersConfig?: object; + quantizationConfig?: object; + sparseVectors?: object; + metadata?: Record; + timeout?: number; +} + +interface QdrantPoint { + id: QdrantPointId; + vector: QdrantVector; + payload?: Record; +} + +interface UpsertVectorDataArgs extends QdrantCollectionArgs { + points: QdrantPoint[]; + wait?: boolean; + ordering?: "weak" | "medium" | "strong"; + timeout?: number; +} + +interface InsertVectorDataArgs extends QdrantCollectionArgs { + id: QdrantPointId; + vector: QdrantVector; + payload?: Record; + wait?: boolean; + ordering?: "weak" | "medium" | "strong"; + timeout?: number; +} + +interface QueryVectorDataArgs extends QdrantQueryArgs { + query?: QdrantVector | QdrantPointId | object; + filter?: object; + params?: object; + limit?: number; + offset?: number; + using?: string; + withPayload?: boolean | string[] | object; + withVector?: boolean | string[]; + scoreThreshold?: number; + lookupFrom?: object; +} + +interface SearchVectorDataArgs extends QdrantQueryArgs { + vector: QdrantVector; + filter?: object; + params?: object; + limit?: number; + offset?: number; + withPayload?: boolean | string[] | object; + withVector?: boolean | string[]; + scoreThreshold?: number; +} + +interface GetDataByIdArgs extends QdrantQueryArgs { + id: QdrantPointId; +} + +interface DeleteByIdArgs extends QdrantCollectionArgs { + id?: QdrantPointId; + ids?: QdrantPointId[]; + wait?: boolean; + ordering?: "weak" | "medium" | "strong"; + timeout?: number; +} + +export class Qdrant { + QDRANT_URL: string; + QDRANT_API_KEY: string; + + constructor(QDRANT_URL?: string, QDRANT_API_KEY?: string) { + this.QDRANT_URL = (QDRANT_URL || process.env.QDRANT_URL || "").replace( + /\/+$/, + "", + ); + this.QDRANT_API_KEY = QDRANT_API_KEY || process.env.QDRANT_API_KEY || ""; + } + + createClient() { + return axios.create({ + baseURL: this.QDRANT_URL, + headers: { + "Content-Type": "application/json", + ...(this.QDRANT_API_KEY ? { "api-key": this.QDRANT_API_KEY } : {}), + }, + }); + } + + async createCollection({ + client = this.createClient(), + collectionName, + vectorSize, + distance = "Cosine", + vectors, + shardNumber, + replicationFactor, + writeConsistencyFactor, + onDiskPayload, + hnswConfig, + walConfig, + optimizersConfig, + quantizationConfig, + sparseVectors, + metadata, + timeout, + }: CreateCollectionArgs): Promise { + if (!vectors && !vectorSize) { + throw new Error( + "Qdrant collection creation requires vectorSize or vectors", + ); + } + + const body = this.removeUndefinedValues({ + vectors: vectors || { size: vectorSize, distance }, + shard_number: shardNumber, + replication_factor: replicationFactor, + write_consistency_factor: writeConsistencyFactor, + on_disk_payload: onDiskPayload, + hnsw_config: hnswConfig, + wal_config: walConfig, + optimizers_config: optimizersConfig, + quantization_config: quantizationConfig, + sparse_vectors: sparseVectors, + metadata, + }); + + return this.request(() => + client.put(this.collectionPath(collectionName), body, { + params: this.removeUndefinedValues({ timeout }), + }), + ); + } + + async getCollection({ + client = this.createClient(), + collectionName, + consistency, + timeout, + }: QdrantQueryArgs): Promise { + return this.request(() => + client.get(this.collectionPath(collectionName), { + params: this.removeUndefinedValues({ consistency, timeout }), + }), + ); + } + + async deleteCollection({ + client = this.createClient(), + collectionName, + timeout, + }: QdrantCollectionArgs & { timeout?: number }): Promise { + return this.request(() => + client.delete(this.collectionPath(collectionName), { + params: this.removeUndefinedValues({ timeout }), + }), + ); + } + + async upsertVectorData({ + client = this.createClient(), + collectionName, + points, + wait, + ordering, + timeout, + }: UpsertVectorDataArgs): Promise { + return this.request(() => + client.put( + `${this.collectionPath(collectionName)}/points`, + { points }, + { params: this.removeUndefinedValues({ wait, ordering, timeout }) }, + ), + ); + } + + async insertVectorData({ + client = this.createClient(), + collectionName, + id, + vector, + payload, + wait, + ordering, + timeout, + }: InsertVectorDataArgs): Promise { + return this.upsertVectorData({ + client, + collectionName, + points: [{ id, vector, payload }], + wait, + ordering, + timeout, + }); + } + + async queryVectorData({ + client = this.createClient(), + collectionName, + query, + filter, + params, + limit = 10, + offset, + using, + withPayload = true, + withVector = false, + scoreThreshold, + lookupFrom, + consistency, + timeout, + }: QueryVectorDataArgs): Promise { + const body = this.removeUndefinedValues({ + query, + filter, + params, + limit, + offset, + using, + with_payload: withPayload, + with_vector: withVector, + score_threshold: scoreThreshold, + lookup_from: lookupFrom, + }); + + return this.request(() => + client.post(`${this.collectionPath(collectionName)}/points/query`, body, { + params: this.removeUndefinedValues({ consistency, timeout }), + }), + ); + } + + async searchVectorData({ + client = this.createClient(), + collectionName, + vector, + filter, + params, + limit = 10, + offset, + withPayload = true, + withVector = false, + scoreThreshold, + consistency, + timeout, + }: SearchVectorDataArgs): Promise { + const body = this.removeUndefinedValues({ + vector, + filter, + params, + limit, + offset, + with_payload: withPayload, + with_vector: withVector, + score_threshold: scoreThreshold, + }); + + return this.request(() => + client.post( + `${this.collectionPath(collectionName)}/points/search`, + body, + { + params: this.removeUndefinedValues({ consistency, timeout }), + }, + ), + ); + } + + async getDataById({ + client = this.createClient(), + collectionName, + id, + consistency, + timeout, + }: GetDataByIdArgs): Promise { + return this.request(() => + client.get( + `${this.collectionPath(collectionName)}/points/${encodeURIComponent(id)}`, + { + params: this.removeUndefinedValues({ consistency, timeout }), + }, + ), + ); + } + + async deleteById({ + client = this.createClient(), + collectionName, + id, + ids, + wait, + ordering, + timeout, + }: DeleteByIdArgs): Promise { + const points = ids || (id !== undefined ? [id] : undefined); + if (!points?.length) { + throw new Error("Qdrant deleteById requires id or ids"); + } + + return this.request(() => + client.post( + `${this.collectionPath(collectionName)}/points/delete`, + { points }, + { params: this.removeUndefinedValues({ wait, ordering, timeout }) }, + ), + ); + } + + private async request(operation: () => Promise<{ data: T }>): Promise { + return new Promise((resolve, reject) => { + const retryOperation = retry.operation({ + retries: 5, + factor: 3, + minTimeout: 1 * 1000, + maxTimeout: 60 * 1000, + randomize: true, + }); + + retryOperation.attempt(async () => { + try { + const response = await operation(); + resolve(response.data); + } catch (error: any) { + if (retryOperation.retry(error)) { + return; + } + reject(this.normalizeError(error)); + } + }); + }); + } + + private collectionPath(collectionName: string): string { + return `/collections/${encodeURIComponent(collectionName)}`; + } + + private removeUndefinedValues>( + value: T, + ): Partial { + return Object.fromEntries( + Object.entries(value).filter( + ([, entryValue]) => entryValue !== undefined, + ), + ) as Partial; + } + + private normalizeError(error: any): Error { + const detail = + error?.response?.data?.status?.error || error?.response?.data?.message; + if (detail) { + return new Error(`Qdrant request failed: ${detail}`); + } + if (error instanceof Error) { + return error; + } + return new Error("Qdrant request failed"); + } +} diff --git a/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts new file mode 100644 index 00000000..9450906a --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts @@ -0,0 +1,142 @@ +import { Qdrant } from "../../lib/qdrant/qdrant"; + +const createMockClient = () => + ({ + put: jest.fn().mockResolvedValue({ data: { status: "ok", result: true } }), + post: jest.fn().mockResolvedValue({ data: { status: "ok", result: [] } }), + get: jest + .fn() + .mockResolvedValue({ data: { status: "ok", result: { id: 1 } } }), + delete: jest + .fn() + .mockResolvedValue({ data: { status: "ok", result: true } }), + }) as any; + +describe("Qdrant", () => { + it("creates a collection with vector settings", async () => { + const client = createMockClient(); + const qdrant = new Qdrant("http://localhost:6333", "test-key"); + + const result = await qdrant.createCollection({ + client, + collectionName: "documents", + vectorSize: 1536, + distance: "Cosine", + }); + + expect(result).toEqual({ status: "ok", result: true }); + expect(client.put).toHaveBeenCalledWith( + "/collections/documents", + { vectors: { size: 1536, distance: "Cosine" } }, + { params: {} }, + ); + }); + + it("upserts vector points through the Qdrant REST API", async () => { + const client = createMockClient(); + const qdrant = new Qdrant("http://localhost:6333", "test-key"); + + await qdrant.insertVectorData({ + client, + collectionName: "documents", + id: "doc-1", + vector: [0.1, 0.2, 0.3], + payload: { content: "hello qdrant" }, + wait: true, + }); + + expect(client.put).toHaveBeenCalledWith( + "/collections/documents/points", + { + points: [ + { + id: "doc-1", + vector: [0.1, 0.2, 0.3], + payload: { content: "hello qdrant" }, + }, + ], + }, + { params: { wait: true } }, + ); + }); + + it("queries points using the universal query endpoint", async () => { + const client = createMockClient(); + const qdrant = new Qdrant("http://localhost:6333", "test-key"); + + await qdrant.queryVectorData({ + client, + collectionName: "documents", + query: [0.1, 0.2, 0.3], + filter: { must: [{ key: "source", match: { value: "pdf" } }] }, + limit: 3, + withPayload: true, + }); + + expect(client.post).toHaveBeenCalledWith( + "/collections/documents/points/query", + { + query: [0.1, 0.2, 0.3], + filter: { must: [{ key: "source", match: { value: "pdf" } }] }, + limit: 3, + with_payload: true, + with_vector: false, + }, + { params: {} }, + ); + }); + + it("keeps a search helper for older Qdrant search integrations", async () => { + const client = createMockClient(); + const qdrant = new Qdrant("http://localhost:6333", "test-key"); + + await qdrant.searchVectorData({ + client, + collectionName: "documents", + vector: [0.1, 0.2, 0.3], + limit: 5, + scoreThreshold: 0.75, + }); + + expect(client.post).toHaveBeenCalledWith( + "/collections/documents/points/search", + { + vector: [0.1, 0.2, 0.3], + limit: 5, + with_payload: true, + with_vector: false, + score_threshold: 0.75, + }, + { params: {} }, + ); + }); + + it("retrieves and deletes points by id", async () => { + const client = createMockClient(); + const qdrant = new Qdrant("http://localhost:6333", "test-key"); + + await qdrant.getDataById({ + client, + collectionName: "documents", + id: "doc-1", + }); + await qdrant.deleteById({ + client, + collectionName: "documents", + ids: ["doc-1", "doc-2"], + wait: true, + }); + + expect(client.get).toHaveBeenCalledWith( + "/collections/documents/points/doc-1", + { + params: {}, + }, + ); + expect(client.post).toHaveBeenCalledWith( + "/collections/documents/points/delete", + { points: ["doc-1", "doc-2"] }, + { params: { wait: true } }, + ); + }); +}); From 0e0f5f771053892cb0a455a9787d7492a9e2ce11 Mon Sep 17 00:00:00 2001 From: zhaoziyuan2024 Date: Thu, 9 Jul 2026 22:10:32 +1000 Subject: [PATCH 2/3] Trigger CLA recheck From 5f3f3372e33eb68968754b1e2241797c893ed85c Mon Sep 17 00:00:00 2001 From: zhaoziyuan2024 Date: Thu, 9 Jul 2026 22:15:28 +1000 Subject: [PATCH 3/3] Refresh CLA status