Implement model gateway and security features in runtime-js

dist/gateway/anthropic.js

@@ -0,0 +1,84 @@
+/**
+ * Anthropic Provider Implementation
+ */
+export class AnthropicProvider {
+    constructor(apiKey, endpoint = 'https://api.anthropic.com', timeout = 30000) {
+        this.apiKey = apiKey;
+        this.endpoint = endpoint;
+        this.timeout = timeout;
+        this.name = 'anthropic';
+        this.type = 'llm';
+    }
+    async complete(request) {
+        const anthropicRequest = this.transformRequest(request);
+        const response = await fetch(`${this.endpoint}/v1/messages`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+                'anthropic-version': '2023-06-01'
+            },
+            body: JSON.stringify(anthropicRequest),
+            signal: AbortSignal.timeout(this.timeout)
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Anthropic API error: ${response.status} - ${error}`);
+        }
+        const data = await response.json();
+        return this.transformResponse(data);
+    }
+    async isAvailable() {
+        try {
+            // Anthropic doesn't have a models endpoint, so just check if we can make a minimal request
+            const response = await fetch(`${this.endpoint}/v1/messages`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-api-key': this.apiKey,
+                    'anthropic-version': '2023-06-01'
+                },
+                body: JSON.stringify({
+                    model: 'claude-3-haiku-20240307',
+                    messages: [{ role: 'user', content: 'test' }],
+                    max_tokens: 1
+                }),
+                signal: AbortSignal.timeout(5000)
+            });
+            return response.ok || response.status === 400; // 400 is ok, means auth worked
+        }
+        catch {
+            return false;
+        }
+    }
+    transformRequest(request) {
+        // Extract system message if present
+        const messages = request.messages.filter(m => m.role !== 'system');
+        const systemMessage = request.messages.find(m => m.role === 'system');
+        return {
+            model: request.model,
+            messages: messages.map(msg => ({
+                role: msg.role === 'assistant' ? 'assistant' : 'user',
+                content: msg.content
+            })),
+            system: systemMessage?.content,
+            temperature: request.temperature ?? 0.7,
+            max_tokens: request.max_tokens ?? 4096,
+            stop_sequences: request.stop,
+            stream: request.stream ?? false
+        };
+    }
+    transformResponse(data) {
+        const content = data.content?.[0];
+        return {
+            content: content?.text || '',
+            model: data.model,
+            usage: data.usage ? {
+                prompt_tokens: data.usage.input_tokens,
+                completion_tokens: data.usage.output_tokens,
+                total_tokens: data.usage.input_tokens + data.usage.output_tokens
+            } : undefined,
+            finish_reason: data.stop_reason
+        };
+    }
+}

dist/gateway/gateway.js

@@ -0,0 +1,216 @@
+/**
+ * Model Gateway - Provider orchestration with fallback and strategy
+ */
+import { ProviderRegistry, RateLimiter, DefaultCredentialResolver } from './provider.js';
+import { OpenAIProvider } from './openai.js';
+import { AnthropicProvider } from './anthropic.js';
+export class ModelGateway {
+    constructor(providers, credentialResolver) {
+        this.rateLimiters = new Map();
+        this.usageStats = new Map();
+        this.registry = new ProviderRegistry();
+        this.credentialResolver = credentialResolver || new DefaultCredentialResolver();
+        // Initialize providers
+        this.initializeProviders(providers);
+    }
+    async initializeProviders(providers) {
+        for (const providerConfig of providers) {
+            try {
+                const provider = await this.createProvider(providerConfig);
+                this.registry.register(providerConfig.id, provider);
+                // Setup rate limiting
+                if (providerConfig.limits) {
+                    this.rateLimiters.set(providerConfig.id, new RateLimiter(providerConfig.limits));
+                }
+                // Initialize usage stats
+                this.usageStats.set(providerConfig.id, {
+                    requests: 0,
+                    tokens: 0,
+                    errors: 0
+                });
+            }
+            catch (error) {
+                console.error(`Failed to initialize provider ${providerConfig.id}:`, error);
+            }
+        }
+    }
+    async createProvider(config) {
+        // Resolve credentials
+        let apiKey;
+        if (config.credentials) {
+            if ('type' in config.credentials) {
+                // CredentialReference
+                const ref = config.credentials;
+                if (ref.type === 'env') {
+                    apiKey = this.credentialResolver.getEnv(ref.ref);
+                }
+                else if (ref.type === 'secrets') {
+                    apiKey = await this.credentialResolver.getSecret(ref.ref);
+                }
+            }
+            else {
+                // CredentialBlock - resolve each key
+                for (const [key, value] of Object.entries(config.credentials)) {
+                    const credRef = value;
+                    if ('type' in credRef && 'ref' in credRef) {
+                        if (credRef.type === 'env') {
+                            apiKey = this.credentialResolver.getEnv(credRef.ref);
+                        }
+                        else if (credRef.type === 'secrets') {
+                            apiKey = await this.credentialResolver.getSecret(credRef.ref);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+        if (!apiKey) {
+            throw new Error(`No credentials found for provider ${config.id}`);
+        }
+        // Create provider instance based on type
+        const endpoint = config.config?.endpoint;
+        const timeout = config.config?.timeout;
+        switch (config.type) {
+            case 'llm':
+                // Detect provider type from id or name
+                const providerName = config.name.toLowerCase();
+                if (providerName.includes('openai') || config.id.includes('openai')) {
+                    return new OpenAIProvider(apiKey, endpoint, timeout);
+                }
+                else if (providerName.includes('anthropic') || config.id.includes('anthropic')) {
+                    return new AnthropicProvider(apiKey, endpoint, timeout);
+                }
+                else {
+                    throw new Error(`Unknown LLM provider: ${config.name}`);
+                }
+            default:
+                throw new Error(`Unsupported provider type: ${config.type}`);
+        }
+    }
+    /**
+     * Complete a request using the specified model configuration
+     */
+    async complete(modelConfig, messages, params) {
+        // Simple string model (backward compatibility)
+        if (typeof modelConfig === 'string') {
+            const [providerId, modelName] = modelConfig.split('/');
+            return this.completeWithProvider(providerId, modelName, messages, params);
+        }
+        // Advanced model config with fallback
+        const strategy = modelConfig.strategy || 'failover';
+        const providers = [modelConfig.primary, ...(modelConfig.fallback || [])];
+        return this.completeWithStrategy(strategy, providers, messages, params);
+    }
+    async completeWithStrategy(strategy, providers, messages, params) {
+        switch (strategy) {
+            case 'failover':
+                return this.failoverStrategy(providers, messages, params);
+            case 'cost_optimized':
+                return this.costOptimizedStrategy(providers, messages, params);
+            case 'latency_optimized':
+                return this.latencyOptimizedStrategy(providers, messages, params);
+            case 'round_robin':
+                return this.roundRobinStrategy(providers, messages, params);
+            default:
+                return this.failoverStrategy(providers, messages, params);
+        }
+    }
+    async failoverStrategy(providers, messages, params) {
+        let lastError;
+        for (const providerConfig of providers) {
+            try {
+                const providerId = this.extractProviderId(providerConfig.provider);
+                return await this.completeWithProvider(providerId, providerConfig.name, messages, { ...params, ...providerConfig.params });
+            }
+            catch (error) {
+                lastError = error;
+                console.warn(`Provider ${providerConfig.provider} failed, trying next...`);
+            }
+        }
+        throw new Error(`All providers failed. Last error: ${lastError?.message}`);
+    }
+    async costOptimizedStrategy(providers, messages, params) {
+        const sorted = [...providers].sort((a, b) => {
+            return this.estimateCost(a.name) - this.estimateCost(b.name);
+        });
+        return this.failoverStrategy(sorted, messages, params);
+    }
+    async latencyOptimizedStrategy(providers, messages, params) {
+        // TODO: Track latency stats and sort by historical performance
+        return this.failoverStrategy(providers, messages, params);
+    }
+    async roundRobinStrategy(providers, messages, params) {
+        const totalRequests = Array.from(this.usageStats.values())
+            .reduce((sum, stats) => sum + stats.requests, 0);
+        const index = totalRequests % providers.length;
+        const providerConfig = providers[index];
+        const providerId = this.extractProviderId(providerConfig.provider);
+        return this.completeWithProvider(providerId, providerConfig.name, messages, { ...params, ...providerConfig.params });
+    }
+    async completeWithProvider(providerId, modelName, messages, params) {
+        const provider = this.registry.get(providerId);
+        if (!provider) {
+            throw new Error(`Provider not found: ${providerId}`);
+        }
+        // Apply rate limiting
+        const limiter = this.rateLimiters.get(providerId);
+        if (limiter) {
+            await limiter.acquire();
+        }
+        // Track usage
+        const stats = this.usageStats.get(providerId);
+        stats.requests++;
+        try {
+            const request = {
+                messages,
+                model: modelName,
+                ...params
+            };
+            const response = await provider.complete(request);
+            // Track token usage
+            if (response.usage) {
+                stats.tokens += response.usage.total_tokens;
+            }
+            return response;
+        }
+        catch (error) {
+            stats.errors++;
+            throw error;
+        }
+    }
+    extractProviderId(providerRef) {
+        const parts = providerRef.split('.');
+        return parts[parts.length - 1];
+    }
+    estimateCost(modelName) {
+        const name = modelName.toLowerCase();
+        if (name.includes('gpt-4'))
+            return 100;
+        if (name.includes('gpt-3.5'))
+            return 10;
+        if (name.includes('claude-3-opus'))
+            return 100;
+        if (name.includes('claude-3-sonnet'))
+            return 50;
+        if (name.includes('claude-3-haiku'))
+            return 10;
+        return 50;
+    }
+    /**
+     * Get usage statistics
+     */
+    getUsageStats() {
+        return Object.fromEntries(this.usageStats);
+    }
+    /**
+     * Check provider availability
+     */
+    async checkProviders() {
+        const results = {};
+        for (const providerId of this.registry.list()) {
+            const provider = this.registry.get(providerId);
+            results[providerId] = await provider.isAvailable();
+        }
+        return results;
+    }
+}

dist/gateway/index.js

@@ -0,0 +1,8 @@
+/**
+ * Model Gateway Module
+ * Universal abstraction for LLM providers
+ */
+export * from './provider.js';
+export * from './openai.js';
+export * from './anthropic.js';
+export * from './gateway.js';

dist/gateway/openai.js

@@ -0,0 +1,70 @@
+/**
+ * OpenAI Provider Implementation
+ */
+export class OpenAIProvider {
+    constructor(apiKey, endpoint = 'https://api.openai.com/v1', timeout = 30000) {
+        this.apiKey = apiKey;
+        this.endpoint = endpoint;
+        this.timeout = timeout;
+        this.name = 'openai';
+        this.type = 'llm';
+    }
+    async complete(request) {
+        const openaiRequest = this.transformRequest(request);
+        const response = await fetch(`${this.endpoint}/chat/completions`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            body: JSON.stringify(openaiRequest),
+            signal: AbortSignal.timeout(this.timeout)
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+        }
+        const data = await response.json();
+        return this.transformResponse(data);
+    }
+    async isAvailable() {
+        try {
+            const response = await fetch(`${this.endpoint}/models`, {
+                headers: {
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                signal: AbortSignal.timeout(5000)
+            });
+            return response.ok;
+        }
+        catch {
+            return false;
+        }
+    }
+    transformRequest(request) {
+        return {
+            model: request.model,
+            messages: request.messages.map(msg => ({
+                role: msg.role,
+                content: msg.content
+            })),
+            temperature: request.temperature ?? 0.7,
+            max_tokens: request.max_tokens,
+            stop: request.stop,
+            stream: request.stream ?? false
+        };
+    }
+    transformResponse(data) {
+        const choice = data.choices?.[0];
+        return {
+            content: choice?.message?.content || '',
+            model: data.model,
+            usage: data.usage ? {
+                prompt_tokens: data.usage.prompt_tokens,
+                completion_tokens: data.usage.completion_tokens,
+                total_tokens: data.usage.total_tokens
+            } : undefined,
+            finish_reason: choice?.finish_reason
+        };
+    }
+}