Phase 4: endpoint and secret security

README.md

@@ -4,11 +4,13 @@ Relay is a reliable outbound webhook delivery platform built on Cloudflare Worke
 
 ## Project Status
 
-Relay has completed Phase 3: asynchronous webhook delivery and retry execution.
+Relay has completed Phase 4: endpoint and secret security.
 
-Relay now accepts authenticated events, creates delivery fanout atomically, publishes work through a durable outbox, consumes Cloudflare Queue messages, claims deliveries with leases, signs stable webhook requests, executes bounded outbound HTTP requests, records attempt evidence, classifies failures, honors `Retry-After`, schedules jittered retries through D1, recovers expired leases, and exhausts deliveries after the configured attempt limit.
+Relay accepts authenticated events, creates delivery fanout atomically, publishes work through a durable outbox, consumes Cloudflare Queue messages, claims deliveries with leases, signs stable webhook requests, records attempt evidence, classifies failures, schedules bounded retries, and recovers expired leases.
 
-Phase 4 endpoint verification, SSRF defenses, encrypted endpoint-specific signing secrets, secret rotation, hardened owner-session flows, and quota enforcement are not implemented yet.
+Phase 4 adds strict public-HTTPS endpoint URL policy, signed challenge-response endpoint verification, mandatory re-verification after URL changes, encrypted endpoint-specific signing secrets, bounded dual-signature secret rotation, delivery suppression for inactive endpoints, hardened signed owner sessions with CSRF protection, fail-closed per-key and global daily quotas, restrictive browser-origin policy, security headers, and generic server error responses.
+
+Phase 5 will connect the operational Console to production data and delivery controls.
 
 ## Delivery Guarantee
 
@@ -50,7 +52,7 @@ Relay provides at-least-once delivery. An event may be delivered more than once,
 
 ## Environment Variables
 
-Relay uses Cloudflare D1 and Queue bindings configured through `wrangler.jsonc`. During Phase 3, the delivery consumer resolves a temporary `DELIVERY_SIGNING_SECRET` Worker secret; Phase 4 will replace this with encrypted endpoint-specific signing secrets.
+Relay uses Cloudflare D1 and Queue bindings configured through `wrangler.jsonc`. Endpoint signing secrets are stored encrypted in D1 with AES-GCM and decrypted at delivery time using a versioned Worker-held master key.
 
 Document non-secret examples in `.env.example`. Store local Worker secrets in an ignored `.dev.vars` file. Never commit credentials, tokens, signing secrets, or production identifiers.
 

apps/console/src/delivery-endpoint-gate.d1.test.ts

@@ -0,0 +1,139 @@
+import { env } from 'cloudflare:workers'
+import { describe, expect, it, vi } from 'vitest'
+import { processDeliveryMessage } from '../worker/lib/delivery-processor.js'
+
+const now = '2026-08-07T03:30:00.000Z'
+const nowMs = Date.parse(now)
+
+describe('D1 delivery endpoint execution gate', () => {
+  it.each([
+    ['pending', null],
+    ['paused', null],
+    ['disabled', now],
+  ] as const)('cancels queued work when endpoint is %s', async (status, disabledAt) => {
+    const suffix = status
+
+    await env.DB.batch([
+      env.DB.prepare(
+        `INSERT INTO api_keys (
+               id, name, key_prefix, secret_hash,
+               status, created_at
+             )
+             VALUES (?, ?, ?, ?, 'active', ?)`,
+      ).bind(
+        `key_gate_${suffix}`,
+        `Gate ${suffix}`,
+        `rly_gate_${suffix}`,
+        (status === 'pending' ? 'a' : status === 'paused' ? 'b' : 'c').repeat(64),
+        now,
+      ),
+
+      env.DB.prepare(
+        `INSERT INTO endpoints (
+               id, name, url, status,
+               created_at, updated_at,
+               verified_at, disabled_at
+             )
+             VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+      ).bind(
+        `ep_gate_${suffix}`,
+        `Gate ${suffix}`,
+        `https://gate-${suffix}.example.test/webhook`,
+        status,
+        now,
+        now,
+        status === 'pending' ? null : now,
+        disabledAt,
+      ),
+
+      env.DB.prepare(
+        `INSERT INTO events (
+               id, api_key_id, idempotency_key,
+               event_type, payload_json,
+               payload_sha256, payload_bytes,
+               created_at
+             )
+             VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+      ).bind(
+        `evt_gate_${suffix}`,
+        `key_gate_${suffix}`,
+        `gate-${suffix}`,
+        'gate.test',
+        '{"safe":true}',
+        'b'.repeat(64),
+        13,
+        now,
+      ),
+
+      env.DB.prepare(
+        `INSERT INTO deliveries (
+               id, event_id, endpoint_id,
+               status, attempt_count,
+               next_attempt_at,
+               created_at, updated_at
+             )
+             VALUES (?, ?, ?, 'queued', 0, ?, ?, ?)`,
+      ).bind(`dlv_gate_${suffix}`, `evt_gate_${suffix}`, `ep_gate_${suffix}`, now, now, now),
+    ])
+
+    const fetcher = vi.fn()
+    const resolveSigningSecrets = vi.fn()
+
+    await expect(
+      processDeliveryMessage(
+        env.DB,
+        {
+          version: 1,
+          deliveryId: `dlv_gate_${suffix}`,
+          reason: 'initial',
+        },
+        {
+          resolveSigningSecrets,
+          fetcher: fetcher as typeof fetch,
+          nowMilliseconds: () => nowMs,
+          createLeaseToken: () => `lease_gate_${suffix}`,
+        },
+      ),
+    ).resolves.toEqual({
+      action: 'ack',
+      reason: 'cancelled',
+    })
+
+    expect(resolveSigningSecrets).not.toHaveBeenCalled()
+    expect(fetcher).not.toHaveBeenCalled()
+
+    const delivery = await env.DB.prepare(
+      `SELECT
+             status,
+             attempt_count,
+             lease_token,
+             last_error_class
+           FROM deliveries
+           WHERE id = ?`,
+    )
+      .bind(`dlv_gate_${suffix}`)
+      .first<{
+        status: string
+        attempt_count: number
+        lease_token: string | null
+        last_error_class: string | null
+      }>()
+
+    expect(delivery).toEqual({
+      status: 'cancelled',
+      attempt_count: 0,
+      lease_token: null,
+      last_error_class: 'endpoint_inactive',
+    })
+
+    const attempts = await env.DB.prepare(
+      `SELECT COUNT(*) AS count
+           FROM delivery_attempts
+           WHERE delivery_id = ?`,
+    )
+      .bind(`dlv_gate_${suffix}`)
+      .first<{ count: number }>()
+
+    expect(attempts?.count).toBe(0)
+  })
+})

apps/console/src/delivery-exhaustion.d1.test.ts

@@ -79,7 +79,7 @@ describe('D1 delivery exhaustion', () => {
           reason: 'retry',
         },
         {
-          resolveSigningSecret: async () => 'exhaust_secret',
+          resolveSigningSecrets: async () => ['exhaust_secret'],
           fetcher,
           nowMilliseconds: () => nowMs,
           random: () => 0,

apps/console/src/delivery-processor.d1.test.ts

@@ -102,7 +102,7 @@ describe('D1 delivery processor', () => {
     let attemptSequence = 0
 
     const dependencies = {
-      resolveSigningSecret: async () => 'endpoint_secret',
+      resolveSigningSecrets: async () => ['endpoint_secret'],
       fetcher,
       nowMilliseconds: () => nowMs,
       random: () => 0,

apps/console/src/delivery-resilience.d1.test.ts

@@ -78,7 +78,7 @@ function dependencies(fetcher: typeof fetch, suffix: string) {
   let attempt = 0
 
   return {
-    resolveSigningSecret: async () => 'test_secret',
+    resolveSigningSecrets: async () => ['test_secret'],
     fetcher,
     nowMilliseconds: () => createdMs,
     random: () => 0,

apps/console/src/delivery-retry-roundtrip.d1.test.ts

@@ -116,7 +116,7 @@ describe('D1 delivery retry roundtrip', () => {
     let attempt = 0
 
     const dependencies = {
-      resolveSigningSecret: async () => 'roundtrip_secret',
+      resolveSigningSecrets: async () => ['roundtrip_secret'],
       fetcher,
       nowMilliseconds: () => currentTime,
       random: () => 0,

apps/console/src/endpoint-persistence.d1.test.ts

@@ -1,6 +1,7 @@
 import { env } from 'cloudflare:workers'
 import { describe, expect, it } from 'vitest'
 import { createEndpoint, replaceEndpointSubscriptions } from '../worker/lib/endpoint-persistence.js'
+import { TEST_ENDPOINT_CRYPTO_DEPENDENCIES } from './test-endpoint-secret.js'
 
 describe('D1 endpoint persistence', () => {
   it('persists pending endpoints and atomically replaces subscriptions', async () => {
@@ -12,6 +13,7 @@ describe('D1 endpoint persistence', () => {
         eventTypes: ['order.created', 'order.cancelled', 'order.created'],
       },
       {
+        ...TEST_ENDPOINT_CRYPTO_DEPENDENCIES,
         now: () => '2026-08-05T12:15:00.000Z',
         createId: (prefix) => `${prefix}_endpoint_d1`,
       },
@@ -78,4 +80,34 @@ describe('D1 endpoint persistence', () => {
 
     expect(auditCount?.count).toBe(2)
   })
+
+  it('does not persist endpoints rejected by the URL policy', async () => {
+    const name = 'Blocked endpoint target'
+
+    await expect(
+      createEndpoint(
+        env.DB,
+        {
+          name,
+          url: 'http://127.0.0.1/webhook',
+          eventTypes: ['order.created'],
+        },
+        {
+          ...TEST_ENDPOINT_CRYPTO_DEPENDENCIES,
+          now: () => '2026-08-05T12:30:00.000Z',
+          createId: (prefix) => `${prefix}_blocked_d1`,
+        },
+      ),
+    ).rejects.toBeInstanceOf(TypeError)
+
+    const stored = await env.DB.prepare(
+      `SELECT COUNT(*) AS count
+         FROM endpoints
+         WHERE name = ?`,
+    )
+      .bind(name)
+      .first<{ count: number }>()
+
+    expect(stored?.count).toBe(0)
+  })
 })

apps/console/src/endpoint-persistence.test.ts

@@ -1,6 +1,10 @@
 import { describe, expect, it } from 'vitest'
 import type { RelayDatabase, RelayStatement } from '../worker/lib/database.js'
 import { createEndpoint, replaceEndpointSubscriptions } from '../worker/lib/endpoint-persistence.js'
+import {
+  TEST_ENDPOINT_CRYPTO_DEPENDENCIES,
+  TEST_ENDPOINT_SIGNING_SECRET,
+} from './test-endpoint-secret.js'
 
 class FakeStatement implements RelayStatement {
   readonly query: string
@@ -64,6 +68,8 @@ describe('endpoint persistence', () => {
         eventTypes: ['invoice.paid', 'invoice.created', 'invoice.paid'],
       },
       {
+        ...TEST_ENDPOINT_CRYPTO_DEPENDENCIES,
+        createSigningSecret: () => TEST_ENDPOINT_SIGNING_SECRET,
         now: () => '2026-08-05T12:00:00.000Z',
         createId: (prefix) => `${prefix}_created`,
       },
@@ -74,11 +80,12 @@ describe('endpoint persistence', () => {
       name: 'Billing webhook',
       url: 'https://hooks.example.test/billing',
       status: 'pending',
+      signingSecret: TEST_ENDPOINT_SIGNING_SECRET,
       eventTypes: ['invoice.created', 'invoice.paid'],
       createdAt: '2026-08-05T12:00:00.000Z',
     })
 
-    expect(database.batched).toHaveLength(4)
+    expect(database.batched).toHaveLength(5)
   })
 
   it('replaces subscriptions in one batch', async () => {
@@ -126,6 +133,11 @@ describe('endpoint persistence', () => {
       url: 'ftp://example.test/file',
       eventTypes: [],
     },
+    {
+      name: 'Endpoint',
+      url: 'http://example.test/webhook',
+      eventTypes: [],
+    },
     {
       name: 'Endpoint',
       url: 'https://user:secret@example.test',
@@ -137,6 +149,8 @@ describe('endpoint persistence', () => {
       eventTypes: [''],
     },
   ])('rejects invalid endpoint input %#', async (input) => {
-    await expect(createEndpoint(new FakeDatabase(), input)).rejects.toBeInstanceOf(TypeError)
+    await expect(
+      createEndpoint(new FakeDatabase(), input, TEST_ENDPOINT_CRYPTO_DEPENDENCIES),
+    ).rejects.toBeInstanceOf(TypeError)
   })
 })

apps/console/src/endpoint-secret-crypto.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from 'vitest'
+import {
+  createEndpointSigningSecret,
+  decryptEndpointSecret,
+  encryptEndpointSecret,
+} from '../worker/lib/endpoint-secret-crypto.js'
+
+const masterKeyV1 = btoa(
+  String.fromCharCode(...Array.from({ length: 32 }, (_, index) => index + 1)),
+)
+
+const keyring = {
+  v1: masterKeyV1,
+}
+
+describe('endpoint secret encryption', () => {
+  it('generates high-entropy endpoint signing secrets', () => {
+    const first = createEndpointSigningSecret()
+    const second = createEndpointSigningSecret()
+
+    expect(first).toMatch(/^rly_whsec_[0-9a-f]{64}$/)
+    expect(second).toMatch(/^rly_whsec_[0-9a-f]{64}$/)
+    expect(first).not.toBe(second)
+  })
+
+  it('round-trips a secret through AES-GCM', async () => {
+    const secret = createEndpointSigningSecret()
+
+    const encrypted = await encryptEndpointSecret(secret, 'ep_crypto_primary', 'v1', keyring)
+
+    expect(encrypted.keyVersion).toBe('v1')
+    expect(encrypted.ivBase64).not.toContain(secret)
+    expect(encrypted.ciphertextBase64).not.toContain(secret)
+
+    await expect(decryptEndpointSecret(encrypted, 'ep_crypto_primary', keyring)).resolves.toBe(
+      secret,
+    )
+  })
+
+  it('binds ciphertext to its endpoint ID', async () => {
+    const encrypted = await encryptEndpointSecret(
+      createEndpointSigningSecret(),
+      'ep_crypto_owner',
+      'v1',
+      keyring,
+    )
+
+    await expect(decryptEndpointSecret(encrypted, 'ep_crypto_other', keyring)).rejects.toThrow(
+      'Endpoint signing secret could not be decrypted.',
+    )
+  })
+
+  it('rejects decryption with the wrong master key', async () => {
+    const encrypted = await encryptEndpointSecret(
+      createEndpointSigningSecret(),
+      'ep_crypto_wrong_key',
+      'v1',
+      keyring,
+    )
+
+    const wrongKeyring = {
+      v1: btoa(String.fromCharCode(...Array.from({ length: 32 }, (_, index) => 255 - index))),
+    }
+
+    await expect(
+      decryptEndpointSecret(encrypted, 'ep_crypto_wrong_key', wrongKeyring),
+    ).rejects.toThrow('Endpoint signing secret could not be decrypted.')
+  })
+
+  it('rejects malformed master keys', async () => {
+    await expect(
+      encryptEndpointSecret(createEndpointSigningSecret(), 'ep_crypto_bad_key', 'v1', {
+        v1: btoa('too-short'),
+      }),
+    ).rejects.toThrow('Endpoint secret master key must contain exactly 32 bytes.')
+  })
+
+  it('rejects unavailable key versions', async () => {
+    await expect(
+      encryptEndpointSecret(createEndpointSigningSecret(), 'ep_crypto_missing', 'v2', keyring),
+    ).rejects.toThrow('Endpoint secret master key version v2 is unavailable.')
+  })
+})