diff --git a/apps/api/.env.example b/apps/api/.env.example index f2b813f1c1..8913358e40 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -48,6 +48,30 @@ OPENAI_API_KEY= ANTHROPIC_API_KEY= GROQ_API_KEY= +# Browserbase — the cloud browser that powers Browser Automations. +# REQUIRED for the browser-automation feature (connect, analyze, runs). +BROWSERBASE_API_KEY= +BROWSERBASE_PROJECT_ID= +# Optional model overrides (defaults are sensible; no need to set these). +# BROWSERBASE_CUA_MODEL: screenshot-based navigation agent. Defaults to +# "openai/gpt-5.6-terra" (needs OPENAI_API_KEY; falls back to Claude if it's +# missing). Set to "openai/gpt-5.6-sol" for max accuracy, or an "anthropic/…" +# computer-use model to use Claude instead. +# BROWSERBASE_STAGEHAND_MODEL: model behind page reading / pass-fail verdicts — +# keep this a general model (Claude), NOT a computer-use model. +BROWSERBASE_CUA_MODEL= +BROWSERBASE_STAGEHAND_MODEL= + +# 1Password service account token (starts with "ops_") for storing browser- +# automation logins in a Comp-managed vault. +# Enables the "Sign in for me" password path: Comp stores the login and signs in +# on its own (including generating 2FA codes) for the first sign-in and for +# scheduled runs. +# Optional — when unset, that path is unavailable and users sign in manually in +# the live browser each time (no unattended re-login). +# Create one at 1Password → Developer → Service Accounts (needs vault-create access). +OP_SERVICE_ACCOUNT_TOKEN= + # Inference.net Catalyst tracing (optional — no-op if CATALYST_OTLP_TOKEN is unset) CATALYST_OTLP_TOKEN= CATALYST_OTLP_ENDPOINT=https://telemetry.inference.net diff --git a/apps/api/package.json b/apps/api/package.json index 423679e678..2a23a80361 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -4,6 +4,7 @@ "version": "0.0.1", "author": "", "dependencies": { + "@1password/sdk": "0.4.0", "@ai-sdk/anthropic": "^3.0.75", "@ai-sdk/groq": "^3.0.38", "@ai-sdk/openai": "^3.0.62", diff --git a/apps/api/src/browserbase/browser-auth-profile.service.spec.ts b/apps/api/src/browserbase/browser-auth-profile.service.spec.ts index 64470d6be0..bc99697da9 100644 --- a/apps/api/src/browserbase/browser-auth-profile.service.spec.ts +++ b/apps/api/src/browserbase/browser-auth-profile.service.spec.ts @@ -1,3 +1,4 @@ +import { ForbiddenException } from '@nestjs/common'; import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; @@ -10,15 +11,20 @@ jest.mock('@db', () => ({ findFirst: jest.fn(), create: jest.fn(), update: jest.fn(), + delete: jest.fn(), deleteMany: jest.fn(), }, browserbaseContext: { findUnique: jest.fn(), + findFirst: jest.fn(), create: jest.fn(), update: jest.fn(), updateMany: jest.fn(), deleteMany: jest.fn(), }, + browserAutomation: { + findMany: jest.fn(), + }, }, })); @@ -37,6 +43,57 @@ describe('BrowserAuthProfileService', () => { service = new BrowserAuthProfileService(sessions); }); + describe('tenant guards (cross-org IDOR)', () => { + it('assertContextOwnedByOrg passes when a profile in the org owns the context', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue({ id: 'bap_1' }); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertContextOwnedByOrg({ organizationId: 'org_1', contextId: 'ctx_1' }), + ).resolves.toBeUndefined(); + }); + + it('assertContextOwnedByOrg passes when the org-level context row owns it', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue({ id: 'bbc_1' }); + await expect( + service.assertContextOwnedByOrg({ organizationId: 'org_1', contextId: 'ctx_1' }), + ).resolves.toBeUndefined(); + }); + + it('assertContextOwnedByOrg rejects a context that belongs to another org', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertContextOwnedByOrg({ organizationId: 'org_1', contextId: 'ctx_other' }), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('assertSessionOwnedByOrg returns the contextId when the org owns the session', async () => { + jest.spyOn(sessions, 'getSessionContextId').mockResolvedValue('ctx_1'); + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue({ id: 'bap_1' }); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertSessionOwnedByOrg({ organizationId: 'org_1', sessionId: 'sess_1' }), + ).resolves.toBe('ctx_1'); + }); + + it('assertSessionOwnedByOrg rejects a session whose context is another org', async () => { + jest.spyOn(sessions, 'getSessionContextId').mockResolvedValue('ctx_other'); + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertSessionOwnedByOrg({ organizationId: 'org_1', sessionId: 'sess_x' }), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('assertSessionOwnedByOrg rejects when the session context cannot be resolved', async () => { + jest.spyOn(sessions, 'getSessionContextId').mockResolvedValue(undefined); + await expect( + service.assertSessionOwnedByOrg({ organizationId: 'org_1', sessionId: 'gone' }), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + }); + it('normalizes hostname and login identity when creating a profile', async () => { (db.browserAuthProfile.findUnique as jest.Mock).mockResolvedValue(null); (db.browserbaseContext.findUnique as jest.Mock).mockResolvedValue(null); @@ -188,4 +245,110 @@ describe('BrowserAuthProfileService', () => { where: { organizationId: 'org_1', contextId: '__PENDING__' }, }); }); + + describe('listProfiles', () => { + it('attaches an automation count per connection by hostname', async () => { + (db.browserAuthProfile.findMany as jest.Mock).mockResolvedValue([ + { id: 'bap_gh', organizationId: 'org_1', hostname: 'github.com' }, + { id: 'bap_aws', organizationId: 'org_1', hostname: 'aws.amazon.com' }, + { id: 'bap_dd', organizationId: 'org_1', hostname: 'datadoghq.com' }, + ]); + (db.browserAutomation.findMany as jest.Mock).mockResolvedValue([ + { targetUrl: 'https://github.com/acme/repo' }, + { targetUrl: 'https://github.com/acme/other' }, + { targetUrl: 'https://aws.amazon.com/console' }, + { targetUrl: 'not-a-url' }, // malformed -> skipped, not fatal + ]); + + const result = await service.listProfiles('org_1'); + + expect(db.browserAutomation.findMany).toHaveBeenCalledWith({ + where: { task: { organizationId: 'org_1' } }, + select: { targetUrl: true }, + }); + expect(result.find((p) => p.id === 'bap_gh')?.automationCount).toBe(2); + expect(result.find((p) => p.id === 'bap_aws')?.automationCount).toBe(1); + expect(result.find((p) => p.id === 'bap_dd')?.automationCount).toBe(0); + }); + }); + + describe('updateProfile / deleteProfile', () => { + const existing = { + id: 'bap_1', + organizationId: 'org_1', + hostname: 'app.example.com', + displayName: 'Example', + status: 'verified', + }; + + beforeEach(() => { + (db.browserAuthProfile.update as jest.Mock).mockImplementation( + ({ data }) => ({ ...existing, ...data }), + ); + (db.browserAuthProfile.delete as jest.Mock).mockResolvedValue(existing); + }); + + it('updates only the display name (trimmed)', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + await service.updateProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + displayName: ' New name ', + }); + expect(db.browserAuthProfile.update).toHaveBeenCalledWith({ + where: { id: 'bap_1' }, + data: { displayName: 'New name' }, + }); + }); + + it('keeps the connection signed in when the URL stays on the same host', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + await service.updateProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + url: 'https://app.example.com/account/login', + }); + const data = (db.browserAuthProfile.update as jest.Mock).mock.calls[0][0] + .data; + expect(data.lastAuthCheckUrl).toBe( + 'https://app.example.com/account/login', + ); + expect(data.status).toBeUndefined(); + }); + + it('marks needs_reauth when the URL moves to a different host', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + await service.updateProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + url: 'https://other.com/login', + }); + const data = (db.browserAuthProfile.update as jest.Mock).mock.calls[0][0] + .data; + expect(data.status).toBe('needs_reauth'); + }); + + it('deletes the profile', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + const result = await service.deleteProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + expect(db.browserAuthProfile.delete).toHaveBeenCalledWith({ + where: { id: 'bap_1' }, + }); + expect(result.success).toBe(true); + }); + + it('throws when updating a missing profile', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.updateProfile({ + organizationId: 'org_1', + profileId: 'nope', + displayName: 'x', + }), + ).rejects.toThrow('not found'); + }); + }); }); diff --git a/apps/api/src/browserbase/browser-auth-profile.service.ts b/apps/api/src/browserbase/browser-auth-profile.service.ts index 0d18c60932..4b15a58acc 100644 --- a/apps/api/src/browserbase/browser-auth-profile.service.ts +++ b/apps/api/src/browserbase/browser-auth-profile.service.ts @@ -1,15 +1,19 @@ import { BadRequestException, + ForbiddenException, Injectable, NotFoundException, } from '@nestjs/common'; -import { db } from '@db'; +import { db, Prisma } from '@db'; import { defaultProfileDisplayName, normalizeHostnameFromUrl, normalizeLoginIdentity, } from './browserbase-url'; -import { BrowserbaseSessionService } from './browserbase-session.service'; +import { + BrowserbaseSessionService, + INTERACTIVE_VIEWPORT, +} from './browserbase-session.service'; import { BrowserAuthProfileContextService } from './browser-auth-profile-context.service'; import { BrowserbaseOrgContextService, @@ -41,10 +45,33 @@ export class BrowserAuthProfileService { ) {} async listProfiles(organizationId: string) { - return db.browserAuthProfile.findMany({ - where: { organizationId }, - orderBy: [{ hostname: 'asc' }, { updatedAt: 'desc' }], - }); + const [profiles, automations] = await Promise.all([ + db.browserAuthProfile.findMany({ + where: { organizationId }, + orderBy: [{ hostname: 'asc' }, { updatedAt: 'desc' }], + }), + db.browserAutomation.findMany({ + where: { task: { organizationId } }, + select: { targetUrl: true }, + }), + ]); + + // Automations bind to a connection by hostname (not a FK), so tally per host. + const countByHost = new Map(); + for (const { targetUrl } of automations) { + let host: string; + try { + host = normalizeHostnameFromUrl(targetUrl); + } catch { + continue; // skip malformed targetUrls rather than fail the whole list + } + countByHost.set(host, (countByHost.get(host) ?? 0) + 1); + } + + return profiles.map((profile) => ({ + ...profile, + automationCount: countByHost.get(profile.hostname) ?? 0, + })); } async getProfile({ @@ -163,7 +190,12 @@ export class BrowserAuthProfileService { throw new NotFoundException('Browser auth profile not found'); } const readyProfile = await this.profileContexts.ready(profile); - return this.sessions.createSessionWithContext(readyProfile.contextId); + // Human-facing session — use the smaller viewport so the sign-in page reads + // larger in the live view. + return this.sessions.createSessionWithContext( + readyProfile.contextId, + INTERACTIVE_VIEWPORT, + ); } async verifyProfileSession(input: { @@ -206,6 +238,70 @@ export class BrowserAuthProfileService { return { profile: updated, auth }; } + async markVerified(input: { organizationId: string; profileId: string }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + // Avoid a needless write when the profile is already verified. + if (profile.status === 'verified') return profile; + + return db.browserAuthProfile.update({ + where: { id: profile.id }, + data: { + status: 'verified', + lastVerifiedAt: new Date(), + blockedReason: null, + }, + }); + } + + async updateProfile(input: { + organizationId: string; + profileId: string; + displayName?: string; + url?: string; + }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + + const data: Prisma.BrowserAuthProfileUpdateInput = {}; + + const name = input.displayName?.trim(); + if (name) data.displayName = name; + + if (input.url !== undefined) { + data.lastAuthCheckUrl = input.url; + // A different hostname means the saved session no longer applies — the + // connection must be re-established. (Hostname is the connection identity, + // so we don't reassign it here.) + try { + if (normalizeHostnameFromUrl(input.url) !== profile.hostname) { + data.status = 'needs_reauth'; + data.blockedReason = 'Sign-in URL changed — reconnect required.'; + } + } catch { + // Ignore an unparseable URL — leave status untouched. + } + } + + return db.browserAuthProfile.update({ + where: { id: profile.id }, + data, + }); + } + + async deleteProfile(input: { organizationId: string; profileId: string }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + await db.browserAuthProfile.delete({ where: { id: profile.id } }); + return { success: true, profile }; + } + async markNeedsReauth(input: { organizationId: string; profileId: string; @@ -274,6 +370,57 @@ export class BrowserAuthProfileService { } } + /** + * Tenant guard: a Browserbase context belongs to an org only if a profile (or + * the legacy org-level context row) for that org points at it. Blocks acting on + * another org's context via a raw session/context endpoint (cross-tenant IDOR). + */ + async assertContextOwnedByOrg(input: { + organizationId: string; + contextId: string; + }): Promise { + const [profile, orgContext] = await Promise.all([ + db.browserAuthProfile.findFirst({ + where: { + organizationId: input.organizationId, + contextId: input.contextId, + }, + select: { id: true }, + }), + db.browserbaseContext.findFirst({ + where: { + organizationId: input.organizationId, + contextId: input.contextId, + }, + select: { id: true }, + }), + ]); + if (!profile && !orgContext) { + throw new ForbiddenException( + 'This browser context does not belong to your organization.', + ); + } + } + + /** + * Tenant guard for a live session: resolve the session's context and confirm + * that context belongs to the caller's org. Returns the resolved contextId. + */ + async assertSessionOwnedByOrg(input: { + organizationId: string; + sessionId: string; + }): Promise { + const contextId = await this.sessions.getSessionContextId(input.sessionId); + if (!contextId) { + throw new ForbiddenException('Could not verify the browser session.'); + } + await this.assertContextOwnedByOrg({ + organizationId: input.organizationId, + contextId, + }); + return contextId; + } + private async assertSessionMatchesProfile(input: { sessionId: string; profileContextId: string; diff --git a/apps/api/src/browserbase/browser-auth-profiles.controller.ts b/apps/api/src/browserbase/browser-auth-profiles.controller.ts index 10d2c2b347..f960e41731 100644 --- a/apps/api/src/browserbase/browser-auth-profiles.controller.ts +++ b/apps/api/src/browserbase/browser-auth-profiles.controller.ts @@ -1,8 +1,20 @@ -import { Body, Controller, Get, Param, Post, UseGuards } from '@nestjs/common'; +import { + BadRequestException, + Body, + Controller, + Delete, + Get, + Param, + Patch, + Post, + Query, + UseGuards, +} from '@nestjs/common'; import { ApiBody, ApiOperation, ApiParam, + ApiQuery, ApiResponse, ApiSecurity, ApiTags, @@ -11,6 +23,11 @@ import { OrganizationId } from '../auth/auth-context.decorator'; import { HybridAuthGuard } from '../auth/hybrid-auth.guard'; import { PermissionGuard } from '../auth/permission.guard'; import { RequirePermission } from '../auth/require-permission.decorator'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import { + BrowserMfaInstructionsService, + type MfaInstructions, +} from './browser-mfa-instructions.service'; import { BrowserbaseService } from './browserbase.service'; import { BrowserAuthProfileResponseDto, @@ -18,6 +35,11 @@ import { ResolveAuthProfileDto, ResolveAuthProfileResponseDto, SessionResponseDto, + SetAuthProfileTotpDto, + SignInAuthProfileDto, + SignInAuthProfileResponseDto, + StoreAuthProfileCredentialsDto, + UpdateAuthProfileDto, VerifyAuthProfileResponseDto, VerifyAuthProfileSessionDto, } from './dto/browserbase.dto'; @@ -27,7 +49,32 @@ import { @UseGuards(HybridAuthGuard, PermissionGuard) @ApiSecurity('apikey') export class BrowserAuthProfilesController { - constructor(private readonly browserbaseService: BrowserbaseService) {} + constructor( + private readonly browserbaseService: BrowserbaseService, + private readonly mfaInstructionsService: BrowserMfaInstructionsService, + private readonly credentialStorageService: BrowserCredentialStorageService, + ) {} + + @Get('mfa-instructions') + @RequirePermission('integration', 'read') + @ApiOperation({ + summary: 'Get authenticator (2FA) setup instructions for a vendor', + description: + 'Returns per-vendor, human-readable steps for finding the authenticator "setup key" (TOTP seed) so a user can enable unattended 2FA. Steps are AI-generated (no per-vendor hardcode), confidence-gated to a universal fallback, and cached per hostname.', + }) + @ApiQuery({ + name: 'host', + description: 'The vendor sign-in URL or hostname (e.g. github.com).', + }) + @ApiResponse({ status: 200 }) + async getMfaInstructions( + @Query('host') host?: string, + ): Promise { + if (!host?.trim()) { + throw new BadRequestException('host query parameter is required'); + } + return this.mfaInstructionsService.getInstructions(host.trim()); + } @Get('profiles') @RequirePermission('integration', 'read') @@ -112,6 +159,159 @@ export class BrowserAuthProfilesController { })) as VerifyAuthProfileResponseDto; } + @Post('profiles/:profileId/credentials') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Store browser auth profile credentials', + description: + 'Store the login (username, password, optional authenticator setup key) for a browser auth profile in the organization vault so scheduled and manual runs can sign in automatically when a session expires.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: StoreAuthProfileCredentialsDto }) + @ApiResponse({ status: 200, type: BrowserAuthProfileResponseDto }) + async storeProfileCredentials( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: StoreAuthProfileCredentialsDto, + ): Promise { + return (await this.browserbaseService.storeAuthProfileCredentials({ + organizationId, + profileId, + username: dto.username, + password: dto.password, + totpSeed: dto.totpSeed, + extraFields: dto.extraFields, + usernameLabel: dto.usernameLabel, + })) as BrowserAuthProfileResponseDto; + } + + @Get('profiles/:profileId/totp') + @RequirePermission('integration', 'read') + @ApiOperation({ + summary: 'Get automatic-2FA status for a connection', + description: + "Reports whether an authenticator setup key (TOTP seed) is stored for this connection, read live from the vault, so scheduled sign-ins can generate 2FA codes unattended.", + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 200 }) + async getProfileTotp( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise<{ configured: boolean }> { + return this.credentialStorageService.getProfileTotpStatus({ + organizationId, + profileId, + }); + } + + @Post('profiles/:profileId/totp') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Store an authenticator setup key for a connection', + description: + "Attach or replace the authenticator setup key (TOTP seed) on this connection's stored login, enabling unattended 2FA. Does not require re-entering the username or password.", + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: SetAuthProfileTotpDto }) + @ApiResponse({ status: 201 }) + async setProfileTotp( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: SetAuthProfileTotpDto, + ): Promise<{ configured: boolean }> { + return this.credentialStorageService.setProfileTotp({ + organizationId, + profileId, + totpSeed: dto.totpSeed, + }); + } + + @Delete('profiles/:profileId/totp') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Turn off automatic 2FA for a connection', + description: + 'Remove the stored authenticator setup key. Scheduled runs pause if the vendor then asks for a code.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 200 }) + async clearProfileTotp( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise<{ configured: boolean }> { + return this.credentialStorageService.clearProfileTotp({ + organizationId, + profileId, + }); + } + + @Post('profiles/:profileId/sign-in') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Automated sign-in for a browser auth profile', + description: + 'Starts a background run that signs in to the vendor using the credentials stored for this profile, so the user does not have to type them into the browser. Returns a run handle to subscribe to; if the automated sign-in cannot complete (CAPTCHA, email/SMS code, SSO), the connect flow falls back to a live browser.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: SignInAuthProfileDto }) + @ApiResponse({ status: 201, type: SignInAuthProfileResponseDto }) + async signInProfile( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: SignInAuthProfileDto, + ): Promise { + return this.browserbaseService.signInAuthProfile({ + organizationId, + profileId, + url: dto.url, + mode: dto.mode, + usernameLabel: dto.usernameLabel, + }); + } + + @Patch('profiles/:profileId') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Update a browser auth profile', + description: + 'Edit a connection’s display name and/or sign-in URL. Changing to a different hostname marks the connection as needing reconnection.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: UpdateAuthProfileDto }) + @ApiResponse({ status: 200, type: BrowserAuthProfileResponseDto }) + async updateProfile( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: UpdateAuthProfileDto, + ): Promise { + return (await this.browserbaseService.updateAuthProfile({ + organizationId, + profileId, + displayName: dto.displayName, + url: dto.url, + })) as BrowserAuthProfileResponseDto; + } + + @Delete('profiles/:profileId') + @RequirePermission('integration', 'delete') + @ApiOperation({ + summary: 'Remove a browser auth profile', + description: + 'Delete a connection and best-effort remove its stored login from the vault. Automations that relied on it stop running until reconnected.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 200 }) + async deleteProfile( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise<{ success: boolean }> { + const result = await this.browserbaseService.deleteAuthProfile({ + organizationId, + profileId, + }); + return { success: result.success }; + } + @Post('profiles/:profileId/needs-reauth') @RequirePermission('integration', 'update') @ApiOperation({ diff --git a/apps/api/src/browserbase/browser-automation-crud.service.ts b/apps/api/src/browserbase/browser-automation-crud.service.ts index d13d67afc0..28fa699fc1 100644 --- a/apps/api/src/browserbase/browser-automation-crud.service.ts +++ b/apps/api/src/browserbase/browser-automation-crud.service.ts @@ -8,6 +8,53 @@ const normalizeCriteria = (value: string | null | undefined): string | null => { return trimmed.length === 0 ? null : trimmed; }; +/** One step of a (possibly multi-vendor) automation. */ +export interface BrowserAutomationStepInput { + profileId?: string | null; + targetUrl: string; + instruction: string; + evaluationCriteria?: string | null; +} + +/** + * Every automation is stored as an ordered list of steps. When only the legacy + * single instruction is supplied, treat it as a one-step automation. + */ +function resolveSteps(data: { + targetUrl?: string; + instruction?: string; + evaluationCriteria?: string | null; + steps?: BrowserAutomationStepInput[]; +}): BrowserAutomationStepInput[] { + if (data.steps && data.steps.length > 0) return data.steps; + return [ + { + targetUrl: data.targetUrl ?? '', + instruction: data.instruction ?? '', + evaluationCriteria: data.evaluationCriteria, + }, + ]; +} + +const toStepCreate = (steps: BrowserAutomationStepInput[]) => + steps.map((step, index) => ({ + order: index, + profileId: step.profileId ?? null, + targetUrl: step.targetUrl, + instruction: step.instruction, + evaluationCriteria: normalizeCriteria(step.evaluationCriteria), + })); + +const STEP_INCLUDE = { steps: { orderBy: { order: 'asc' as const } } }; + +/** Per-step evidence for a run — one screenshot + verdict per step, in order. */ +const RUN_STEP_RUNS_INCLUDE = { + stepRuns: { + orderBy: { order: 'asc' as const }, + include: { step: { select: { targetUrl: true } } }, + }, +}; + @Injectable() export class BrowserAutomationCrudService { constructor( @@ -22,6 +69,7 @@ export class BrowserAutomationCrudService { targetUrl: string; instruction: string; evaluationCriteria?: string; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -30,19 +78,37 @@ export class BrowserAutomationCrudService { await this.requireTaskInOrg({ taskId: data.taskId, organizationId }); } + const steps = resolveSteps(data); + const first = steps[0]; + + // A task's browser evidence shares one cadence (set from the section + // header), so a new automation inherits the task's current schedule rather + // than silently defaulting to daily. + const scheduleFrequency = + data.scheduleFrequency ?? + ( + await db.browserAutomation.findFirst({ + where: { taskId: data.taskId }, + orderBy: { createdAt: 'asc' }, + select: { scheduleFrequency: true }, + }) + )?.scheduleFrequency; + return db.browserAutomation.create({ data: { taskId: data.taskId, name: data.name, description: data.description, - targetUrl: data.targetUrl, - instruction: data.instruction, - evaluationCriteria: normalizeCriteria(data.evaluationCriteria), + // Keep the legacy single fields in sync with the first step until the + // engine reads steps directly (then these columns get dropped). + targetUrl: first.targetUrl, + instruction: first.instruction, + evaluationCriteria: normalizeCriteria(first.evaluationCriteria), isEnabled: true, - ...(data.scheduleFrequency !== undefined - ? { scheduleFrequency: data.scheduleFrequency } - : {}), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + steps: { create: toStepCreate(steps) }, }, + include: STEP_INCLUDE, }); } @@ -51,7 +117,12 @@ export class BrowserAutomationCrudService { where: { id: automationId }, include: { task: { select: { organizationId: true } }, - runs: { orderBy: { createdAt: 'desc' }, take: 10 }, + runs: { + orderBy: { createdAt: 'desc' }, + take: 10, + include: RUN_STEP_RUNS_INCLUDE, + }, + ...STEP_INCLUDE, }, }); return this.hideCrossOrgAutomation({ automation, organizationId }); @@ -65,7 +136,12 @@ export class BrowserAutomationCrudService { return db.browserAutomation.findMany({ where: { taskId }, include: { - runs: { orderBy: { createdAt: 'desc' }, take: 1 }, + runs: { + orderBy: { createdAt: 'desc' }, + take: 1, + include: RUN_STEP_RUNS_INCLUDE, + }, + ...STEP_INCLUDE, }, orderBy: { createdAt: 'desc' }, }); @@ -80,6 +156,7 @@ export class BrowserAutomationCrudService { instruction?: string; evaluationCriteria?: string; isEnabled?: boolean; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -88,16 +165,58 @@ export class BrowserAutomationCrudService { await this.requireAutomationInOrg({ automationId, organizationId }); } - const { evaluationCriteria, scheduleFrequency, ...rest } = data; - return db.browserAutomation.update({ - where: { id: automationId }, - data: { - ...rest, - ...(evaluationCriteria !== undefined - ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } - : {}), - ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), - }, + const { evaluationCriteria, scheduleFrequency, steps, ...rest } = data; + + // Steps supplied → replace the whole sequence and sync the legacy columns + // from the first step (atomic so a run never sees a half-updated list). + if (steps && steps.length > 0) { + const first = steps[0]; + return db.$transaction(async (tx) => { + await tx.browserAutomationStep.deleteMany({ where: { automationId } }); + return tx.browserAutomation.update({ + where: { id: automationId }, + data: { + ...rest, + targetUrl: first.targetUrl, + instruction: first.instruction, + evaluationCriteria: normalizeCriteria(first.evaluationCriteria), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + steps: { create: toStepCreate(steps) }, + }, + include: STEP_INCLUDE, + }); + }); + } + + // Legacy single-field edit → update the automation and mirror it onto the + // first step so the two representations stay consistent. + const stepPatch = { + ...(rest.targetUrl !== undefined ? { targetUrl: rest.targetUrl } : {}), + ...(rest.instruction !== undefined ? { instruction: rest.instruction } : {}), + ...(evaluationCriteria !== undefined + ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } + : {}), + }; + + return db.$transaction(async (tx) => { + const updated = await tx.browserAutomation.update({ + where: { id: automationId }, + data: { + ...rest, + ...(evaluationCriteria !== undefined + ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } + : {}), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + }, + include: STEP_INCLUDE, + }); + if (Object.keys(stepPatch).length > 0) { + await tx.browserAutomationStep.updateMany({ + where: { automationId, order: 0 }, + data: stepPatch, + }); + } + return updated; }); } @@ -108,37 +227,79 @@ export class BrowserAutomationCrudService { return db.browserAutomation.delete({ where: { id: automationId } }); } + /** + * Set one cadence for every browser automation on a task. Browser evidence + * shares a task-level schedule (there's no need to run different vendors in + * one task on different days), so this bulk-updates them together. + */ + async setTaskSchedule( + taskId: string, + scheduleFrequency: TaskFrequency, + organizationId?: string, + ) { + if (organizationId) { + await this.requireTaskInOrg({ taskId, organizationId }); + } + const { count } = await db.browserAutomation.updateMany({ + where: { taskId }, + data: { scheduleFrequency }, + }); + return { success: true, scheduleFrequency, updated: count }; + } + + /** Presign a run's screenshots (full page + close-up) and each step's. */ + private async presignRun< + T extends { + screenshotUrl: string | null; + focusScreenshotUrl?: string | null; + stepRuns?: Array<{ + screenshotUrl: string | null; + focusScreenshotUrl?: string | null; + }>; + }, + >(run: T): Promise { + const presign = ( + key: string | null | undefined, + ): Promise => + key ? this.screenshots.getPresignedUrl({ key }) : Promise.resolve(key); + const [screenshotUrl, focusScreenshotUrl, stepRuns] = await Promise.all([ + presign(run.screenshotUrl), + presign(run.focusScreenshotUrl), + run.stepRuns + ? Promise.all( + run.stepRuns.map(async (stepRun) => ({ + ...stepRun, + screenshotUrl: await presign(stepRun.screenshotUrl), + focusScreenshotUrl: await presign(stepRun.focusScreenshotUrl), + })), + ) + : Promise.resolve(run.stepRuns), + ]); + return { ...run, screenshotUrl, focusScreenshotUrl, stepRuns } as T; + } + async getRunWithPresignedUrl(runId: string, organizationId?: string) { const run = await db.browserAutomationRun.findUnique({ where: { id: runId }, - include: { automation: { include: { task: true } } }, + include: { + automation: { include: { task: true } }, + ...RUN_STEP_RUNS_INCLUDE, + }, }); if (!run) return null; if (organizationId && run.automation.task.organizationId !== organizationId) { return null; } - if (!run.screenshotUrl) return run; - const screenshotUrl = await this.screenshots.getPresignedUrl({ - key: run.screenshotUrl, - }); - return { ...run, screenshotUrl }; + return this.presignRun(run); } async getAutomationsWithPresignedUrls(taskId: string, organizationId?: string) { const automations = await this.getBrowserAutomationsForTask(taskId, organizationId); return Promise.all( - automations.map(async (automation) => { - const runsWithUrls = await Promise.all( - automation.runs.map(async (run) => { - if (!run.screenshotUrl) return run; - const screenshotUrl = await this.screenshots.getPresignedUrl({ - key: run.screenshotUrl, - }); - return { ...run, screenshotUrl }; - }), - ); - return { ...automation, runs: runsWithUrls }; - }), + automations.map(async (automation) => ({ + ...automation, + runs: await Promise.all(automation.runs.map((run) => this.presignRun(run))), + })), ); } diff --git a/apps/api/src/browserbase/browser-automation-draft.service.spec.ts b/apps/api/src/browserbase/browser-automation-draft.service.spec.ts new file mode 100644 index 0000000000..ba9b29421e --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-draft.service.spec.ts @@ -0,0 +1,81 @@ +jest.mock('@db', () => ({ + db: { + task: { findFirst: jest.fn() }, + browserAutomationDraft: { + findMany: jest.fn(), + findFirst: jest.fn(), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }, + }, + Prisma: {}, +})); + +import { db } from '@db'; +import { BrowserAutomationDraftService } from './browser-automation-draft.service'; + +describe('BrowserAutomationDraftService', () => { + let service: BrowserAutomationDraftService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new BrowserAutomationDraftService(); + }); + + it('creates a draft after checking the task is in the org, storing plain JSON steps', async () => { + (db.task.findFirst as jest.Mock).mockResolvedValue({ id: 'tsk_1' }); + (db.browserAutomationDraft.create as jest.Mock).mockResolvedValue({ id: 'bad_1' }); + + await service.createDraft( + { + taskId: 'tsk_1', + name: 'GitHub 2FA', + steps: [{ profileId: 'p1', instruction: 'x', extra: undefined }], + }, + 'org_1', + ); + + expect(db.task.findFirst).toHaveBeenCalledWith({ + where: { id: 'tsk_1', organizationId: 'org_1' }, + select: { id: true }, + }); + const data = (db.browserAutomationDraft.create as jest.Mock).mock.calls[0][0].data; + expect(data.taskId).toBe('tsk_1'); + expect(data.name).toBe('GitHub 2FA'); + // toJson strips undefined -> plain JSON. + expect(data.steps).toEqual([{ profileId: 'p1', instruction: 'x' }]); + }); + + it('rejects creating a draft for a task in another org', async () => { + (db.task.findFirst as jest.Mock).mockResolvedValue(null); + + await expect( + service.createDraft({ taskId: 'tsk_x', steps: [] }, 'org_1'), + ).rejects.toThrow('Task not found'); + expect(db.browserAutomationDraft.create).not.toHaveBeenCalled(); + }); + + it('updates a draft only when it belongs to the org', async () => { + (db.browserAutomationDraft.findFirst as jest.Mock).mockResolvedValue({ id: 'bad_1' }); + (db.browserAutomationDraft.update as jest.Mock).mockResolvedValue({ id: 'bad_1' }); + + await service.updateDraft('bad_1', { steps: [{ instruction: 'y' }] }, 'org_1'); + + expect(db.browserAutomationDraft.findFirst).toHaveBeenCalledWith({ + where: { id: 'bad_1', task: { organizationId: 'org_1' } }, + select: { id: true }, + }); + expect(db.browserAutomationDraft.update).toHaveBeenCalledWith({ + where: { id: 'bad_1' }, + data: { steps: [{ instruction: 'y' }] }, + }); + }); + + it('rejects deleting a draft from another org', async () => { + (db.browserAutomationDraft.findFirst as jest.Mock).mockResolvedValue(null); + + await expect(service.deleteDraft('bad_x', 'org_1')).rejects.toThrow('Draft not found'); + expect(db.browserAutomationDraft.delete).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browser-automation-draft.service.ts b/apps/api/src/browserbase/browser-automation-draft.service.ts new file mode 100644 index 0000000000..e8f2c80121 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-draft.service.ts @@ -0,0 +1,80 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { db, Prisma } from '@db'; + +/** Strip class instances / undefined down to plain JSON for a Prisma Json column. */ +function toJson(value: unknown): Prisma.InputJsonValue { + return JSON.parse(JSON.stringify(value ?? null)); +} + +interface DraftInput { + name?: string | null; + steps: unknown; +} + +/** + * Server-side store for in-progress (unsaved) automations. Drafts are scoped to + * a task (and therefore an org); they hold the composer's raw step state as JSON + * and are deleted once the automation is saved for real. + */ +@Injectable() +export class BrowserAutomationDraftService { + private async requireTaskInOrg(taskId: string, organizationId: string) { + const task = await db.task.findFirst({ + where: { id: taskId, organizationId }, + select: { id: true }, + }); + if (!task) throw new NotFoundException('Task not found'); + } + + private async requireDraftInOrg(draftId: string, organizationId: string) { + const draft = await db.browserAutomationDraft.findFirst({ + where: { id: draftId, task: { organizationId } }, + select: { id: true }, + }); + if (!draft) throw new NotFoundException('Draft not found'); + } + + async listDraftsForTask(taskId: string, organizationId: string) { + await this.requireTaskInOrg(taskId, organizationId); + return db.browserAutomationDraft.findMany({ + where: { taskId }, + orderBy: { updatedAt: 'desc' }, + }); + } + + async createDraft( + input: DraftInput & { taskId: string; createdById?: string | null }, + organizationId: string, + ) { + await this.requireTaskInOrg(input.taskId, organizationId); + return db.browserAutomationDraft.create({ + data: { + taskId: input.taskId, + createdById: input.createdById ?? null, + name: input.name ?? null, + steps: toJson(input.steps), + }, + }); + } + + async updateDraft( + draftId: string, + input: { name?: string | null; steps?: unknown }, + organizationId: string, + ) { + await this.requireDraftInOrg(draftId, organizationId); + return db.browserAutomationDraft.update({ + where: { id: draftId }, + data: { + ...(input.name !== undefined ? { name: input.name } : {}), + ...(input.steps !== undefined ? { steps: toJson(input.steps) } : {}), + }, + }); + } + + async deleteDraft(draftId: string, organizationId: string) { + await this.requireDraftInOrg(draftId, organizationId); + await db.browserAutomationDraft.delete({ where: { id: draftId } }); + return { success: true }; + } +} diff --git a/apps/api/src/browserbase/browser-automation-execution.service.spec.ts b/apps/api/src/browserbase/browser-automation-execution.service.spec.ts index 67e0acfa31..ae50537ea7 100644 --- a/apps/api/src/browserbase/browser-automation-execution.service.spec.ts +++ b/apps/api/src/browserbase/browser-automation-execution.service.spec.ts @@ -9,6 +9,7 @@ jest.mock('@db', () => ({ $transaction: jest.fn(), browserAutomation: { findUnique: jest.fn() }, browserAutomationRun: { updateMany: jest.fn(), findUnique: jest.fn() }, + browserAutomationStepRun: { create: jest.fn(), update: jest.fn() }, }, Prisma: { TransactionIsolationLevel: { Serializable: 'Serializable' }, @@ -43,6 +44,9 @@ describe('BrowserAutomationExecutionService', () => { (db.browserAutomationRun.updateMany as jest.Mock).mockResolvedValue({ count: 1, }); + (db.browserAutomationStepRun.create as jest.Mock).mockResolvedValue({ + id: 'basr_1', + }); }); it('persists a failed terminal state when the runner throws', async () => { @@ -56,6 +60,7 @@ describe('BrowserAutomationExecutionService', () => { hostname: 'example.com', loginIdentity: '', displayName: 'example.com browser profile', + identifierLabel: null, contextId: 'ctx_1', status: 'verified', lastVerifiedAt: null, diff --git a/apps/api/src/browserbase/browser-automation-execution.service.ts b/apps/api/src/browserbase/browser-automation-execution.service.ts index 098082fd50..7bf3155b49 100644 --- a/apps/api/src/browserbase/browser-automation-execution.service.ts +++ b/apps/api/src/browserbase/browser-automation-execution.service.ts @@ -6,8 +6,15 @@ import { } from '@nestjs/common'; import { db } from '@db'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; -import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; import { failedBrowserEvidenceRunResult } from './browser-automation-run-result'; +import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; +import { stepsForRun } from './browser-automation-step-results'; +import { BrowserAutomationStepRunnerService } from './browser-automation-step-runner.service'; +import { + createEvidenceTimeline, + type BrowserRunLivePhase, + type EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; import { BrowserEvidenceRunnerService, type BrowserEvidenceRunResult, @@ -17,6 +24,7 @@ import { BrowserbaseSessionService } from './browserbase-session.service'; @Injectable() export class BrowserAutomationExecutionService { private readonly logger = new Logger(BrowserAutomationExecutionService.name); + private readonly stepRunner: BrowserAutomationStepRunnerService; constructor( private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), @@ -27,7 +35,15 @@ export class BrowserAutomationExecutionService { sessions, ), private readonly runs: BrowserAutomationRunStoreService = new BrowserAutomationRunStoreService(), - ) {} + ) { + // Built here (not injected) so it reuses these exact profile/runner/run + // instances — including the spied ones in unit tests. + this.stepRunner = new BrowserAutomationStepRunnerService( + this.profiles, + this.runner, + this.runs, + ); + } async startAutomationWithLiveView( automationId: string, @@ -57,7 +73,7 @@ export class BrowserAutomationExecutionService { startedAt: run.startedAt, result, }); - await this.applyProfileResult({ + await this.stepRunner.applyProfileResult({ organizationId, profileId: profile.id, result, @@ -71,6 +87,8 @@ export class BrowserAutomationExecutionService { runId: string, sessionId: string, organizationId: string, + /** Live activity timeline, surfaced to the Run live view via realtime. */ + onSteps?: (steps: EvidenceTimelineStep[]) => void, ) { const automation = await this.getRunnableAutomation({ automationId, @@ -86,6 +104,7 @@ export class BrowserAutomationExecutionService { targetUrl: automation.targetUrl, profileId: run.profileId ?? undefined, }); + const timeline = createEvidenceTimeline(onSteps); let result: BrowserEvidenceRunResult; try { result = await this.runner.executeEvidenceOnSession({ @@ -105,6 +124,7 @@ export class BrowserAutomationExecutionService { vaultExternalItemRef: profile.vaultExternalItemRef, vaultConnectionId: profile.vaultConnectionId, }, + onLog: (entry) => timeline.step(entry.message), beforeExecution: () => this.runs.assertRunIsStillActive({ runId, automationId }), }); @@ -114,8 +134,16 @@ export class BrowserAutomationExecutionService { result = failedBrowserEvidenceRunResult(error); } + timeline.finish( + result.success + ? 'done' + : result.status === 'blocked' || result.needsReauth + ? 'warn' + : 'fail', + ); + await this.runs.finishRun({ runId, startedAt: run.startedAt, result }); - await this.applyProfileResult({ + await this.stepRunner.applyProfileResult({ organizationId, profileId: profile.id, result, @@ -123,110 +151,78 @@ export class BrowserAutomationExecutionService { return this.toRunResponse({ runId, result }); } - async runBrowserAutomation(automationId: string, organizationId: string) { + /** + * The interactive "Run" — executes the FULL step sequence (every vendor), with + * step 0 running on the pre-opened live session (so it's watchable) and later + * vendors each in their own session. Streams a combined step timeline. This is + * what start-live + execute-live drives, so clicking Run no longer stops after + * the first vendor. + */ + async executeAutomationLive( + automationId: string, + runId: string, + sessionId: string, + organizationId: string, + onSteps?: (steps: EvidenceTimelineStep[]) => void, + onLiveView?: (url: string) => void, + onLivePhase?: (phase: BrowserRunLivePhase) => void, + ) { const automation = await this.getRunnableAutomation({ automationId, organizationId, }); - const profile = await this.profiles.resolveProfileForTarget({ + const run = await this.runs.getActiveRun({ runId, automationId }); + const steps = stepsForRun(automation); + const firstProfile = await this.stepRunner.resolveStepProfile({ organizationId, - targetUrl: automation.targetUrl, + step: steps[0], }); - const run = await this.runs.createRun({ + + const result = await this.stepRunner.runSteps({ + organizationId, + taskId: automation.taskId, automationId, - profileId: profile.id, + runId, + steps, + firstProfile, + firstSessionId: sessionId, + onSteps, + onLiveView, + onLivePhase, }); - if (profile.status !== 'verified') { - const result = this.profileBlockedResult(profile.status); - await this.runs.finishRun({ - runId: run.id, - startedAt: run.startedAt, - result, - }); - return this.toRunResponse({ runId: run.id, result }); - } + await this.runs.finishRun({ runId, startedAt: run.startedAt, result }); + return this.toRunResponse({ runId, result }); + } - let result: BrowserEvidenceRunResult; - try { - result = await this.runner.runEvidence({ - organizationId, - taskId: automation.taskId, - automationId, - runId: run.id, - targetUrl: automation.targetUrl, - instruction: automation.instruction, - evaluationCriteria: automation.evaluationCriteria, - profile: { - id: profile.id, - hostname: profile.hostname, - contextId: profile.contextId, - vaultProvider: profile.vaultProvider, - vaultExternalItemRef: profile.vaultExternalItemRef, - vaultConnectionId: profile.vaultConnectionId, - }, - }); - } catch (error) { - this.logger.error('Browser evidence runner failed', error); - result = failedBrowserEvidenceRunResult(error); - } - await this.runs.finishRun({ - runId: run.id, - startedAt: run.startedAt, - result, - }); - await this.applyProfileResult({ + async runBrowserAutomation(automationId: string, organizationId: string) { + const automation = await this.getRunnableAutomation({ + automationId, organizationId, - profileId: profile.id, - result, }); - return this.toRunResponse({ runId: run.id, result }); - } + const steps = stepsForRun(automation); - private async applyProfileResult(input: { - organizationId: string; - profileId: string; - result: BrowserEvidenceRunResult; - }) { - if (input.result.failureCode === 'needs_reauth') { - await this.profiles.markNeedsReauth({ - organizationId: input.organizationId, - profileId: input.profileId, - reason: input.result.blockedReason, - }); - } + // Attribute the run to the first step's connection. + const firstProfile = await this.stepRunner.resolveStepProfile({ + organizationId, + step: steps[0], + }); + const run = await this.runs.createRun({ + automationId, + profileId: firstProfile?.id, + }); - if ( - input.result.failureCode === 'captcha_blocked' || - input.result.failureCode === 'needs_user_action' - ) { - await this.profiles.markBlocked({ - organizationId: input.organizationId, - profileId: input.profileId, - reason: - input.result.blockedReason ?? - input.result.error ?? - 'User action is required before this automation can run.', - }); - } - } + const result = await this.stepRunner.runSteps({ + organizationId, + taskId: automation.taskId, + automationId, + runId: run.id, + steps, + firstProfile, + }); - private profileBlockedResult(status: string): BrowserEvidenceRunResult { - const needsUserAction = status === 'blocked'; - return { - success: false, - status: 'blocked', - error: needsUserAction - ? 'This browser profile is blocked. Resolve the blocked state before running automations.' - : 'This browser profile is not verified. Reconnect it before running automations.', - needsReauth: !needsUserAction, - failureCode: needsUserAction ? 'needs_user_action' : 'needs_reauth', - failureStage: 'auth', - blockedReason: needsUserAction - ? 'Browser profile is blocked.' - : 'Browser profile is not verified.', - logs: [], - }; + await this.runs.finishRun({ runId: run.id, startedAt: run.startedAt, result }); + return this.toRunResponse({ runId: run.id, result }); } private toRunResponse(input: { @@ -257,6 +253,7 @@ export class BrowserAutomationExecutionService { task: { select: { title: true, description: true, organizationId: true }, }, + steps: { orderBy: { order: 'asc' } }, }, }); if ( diff --git a/apps/api/src/browserbase/browser-automation-run-store.service.ts b/apps/api/src/browserbase/browser-automation-run-store.service.ts index ac02332299..124d5b4308 100644 --- a/apps/api/src/browserbase/browser-automation-run-store.service.ts +++ b/apps/api/src/browserbase/browser-automation-run-store.service.ts @@ -58,6 +58,7 @@ export class BrowserAutomationRunStoreService { ? Date.now() - input.startedAt.getTime() : 0, screenshotUrl: input.result.screenshotKey, + focusScreenshotUrl: input.result.focusScreenshotKey ?? null, evaluationStatus: input.result.evaluationStatus ?? null, evaluationReason: input.result.evaluationReason ?? null, error: input.result.error, @@ -74,6 +75,40 @@ export class BrowserAutomationRunStoreService { } } + async createStepRun(input: { + runId: string; + stepId: string | null; + order: number; + }) { + return db.browserAutomationStepRun.create({ + data: { + runId: input.runId, + stepId: input.stepId, + order: input.order, + status: 'running', + startedAt: new Date(), + }, + }); + } + + async finishStepRun(input: { + stepRunId: string; + result: BrowserEvidenceRunResult; + }): Promise { + await db.browserAutomationStepRun.update({ + where: { id: input.stepRunId }, + data: { + status: input.result.status, + completedAt: new Date(), + screenshotUrl: input.result.screenshotKey ?? null, + focusScreenshotUrl: input.result.focusScreenshotKey ?? null, + evaluationStatus: input.result.evaluationStatus ?? null, + evaluationReason: input.result.evaluationReason ?? null, + error: input.result.error ?? null, + }, + }); + } + async getActiveRun(input: { runId: string; automationId: string }) { const run = await db.browserAutomationRun.findUnique({ where: { id: input.runId }, diff --git a/apps/api/src/browserbase/browser-automation-step-results.ts b/apps/api/src/browserbase/browser-automation-step-results.ts new file mode 100644 index 0000000000..93d9540453 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-step-results.ts @@ -0,0 +1,129 @@ +import type { BrowserEvidenceRunResult } from './browser-evidence-runner.service'; + +/** A normalized step to execute (real step row, or a legacy inline instruction). */ +export type StepForRun = { + id: string | null; + order: number; + profileId: string | null; + targetUrl: string; + instruction: string; + evaluationCriteria: string | null; +}; + +/** Steps to run: the ordered step rows, or the inline instruction as one step. */ +export function stepsForRun(automation: { + targetUrl: string; + instruction: string; + evaluationCriteria: string | null; + steps?: Array<{ + id: string; + order: number; + profileId: string | null; + targetUrl: string; + instruction: string; + evaluationCriteria: string | null; + }>; +}): StepForRun[] { + if (automation.steps && automation.steps.length > 0) { + return [...automation.steps] + .sort((a, b) => a.order - b.order) + .map((step) => ({ + id: step.id, + order: step.order, + profileId: step.profileId, + targetUrl: step.targetUrl, + instruction: step.instruction, + evaluationCriteria: step.evaluationCriteria, + })); + } + return [ + { + id: null, + order: 0, + profileId: null, + targetUrl: automation.targetUrl, + instruction: automation.instruction, + evaluationCriteria: automation.evaluationCriteria, + }, + ]; +} + +export function profileMissingResult(): BrowserEvidenceRunResult { + return { + success: false, + status: 'blocked', + error: 'This step has no connected vendor login. Connect one, then run again.', + needsReauth: true, + failureCode: 'needs_reauth', + failureStage: 'auth', + blockedReason: 'No connection is bound to this step.', + logs: [], + }; +} + +export function profileBlockedResult(status: string): BrowserEvidenceRunResult { + const needsUserAction = status === 'blocked'; + return { + success: false, + status: 'blocked', + error: needsUserAction + ? 'This browser profile is blocked. Resolve the blocked state before running automations.' + : 'This browser profile is not verified. Reconnect it before running automations.', + needsReauth: !needsUserAction, + failureCode: needsUserAction ? 'needs_user_action' : 'needs_reauth', + failureStage: 'auth', + blockedReason: needsUserAction + ? 'Browser profile is blocked.' + : 'Browser profile is not verified.', + logs: [], + }; +} + +/** + * Combine per-step results into one run verdict: overall success only if every + * step ran; the check fails if any step's check fails; error metadata comes from + * the first step that failed technically. A single step passes through verbatim. + */ +export function rollUpStepResults( + results: BrowserEvidenceRunResult[], +): BrowserEvidenceRunResult { + if (results.length === 1) return results[0]; + + const firstProblem = results.find((result) => !result.success); + const failedCheck = results.find( + (result) => result.evaluationStatus === 'fail', + ); + const lastWithShot = [...results] + .reverse() + .find((result) => result.screenshotKey); + + const status: BrowserEvidenceRunResult['status'] = results.every( + (result) => result.success, + ) + ? 'completed' + : results.some((result) => result.status === 'failed') + ? 'failed' + : 'blocked'; + + return { + success: results.every((result) => result.success), + status, + screenshotKey: lastWithShot?.screenshotKey, + screenshotUrl: lastWithShot?.screenshotUrl, + finalUrl: results[results.length - 1]?.finalUrl, + evaluationStatus: failedCheck + ? 'fail' + : results.some((result) => result.evaluationStatus === 'pass') + ? 'pass' + : undefined, + evaluationReason: failedCheck?.evaluationReason, + error: firstProblem?.error, + needsReauth: results.some((result) => result.needsReauth), + failureCode: firstProblem?.failureCode, + failureStage: firstProblem?.failureStage, + blockedReason: firstProblem?.blockedReason, + logs: results.flatMap((result) => + Array.isArray(result.logs) ? result.logs : [], + ), + }; +} diff --git a/apps/api/src/browserbase/browser-automation-step-runner.service.ts b/apps/api/src/browserbase/browser-automation-step-runner.service.ts new file mode 100644 index 0000000000..daf8345f81 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-step-runner.service.ts @@ -0,0 +1,258 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { failedBrowserEvidenceRunResult } from './browser-automation-run-result'; +import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; +import { + profileBlockedResult, + profileMissingResult, + rollUpStepResults, + type StepForRun, +} from './browser-automation-step-results'; +import type { BrowserEvidenceLog } from './browser-evidence-execution'; +import { + createEvidenceTimeline, + type BrowserRunLivePhase, + type EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; +import { + BrowserEvidenceRunnerService, + type BrowserEvidenceRunResult, +} from './browser-evidence-runner.service'; +import { BrowserbaseSessionService } from './browserbase-session.service'; + +/** " · github.com" for a step's label, or "" when the URL can't be parsed. */ +function hostSuffix(targetUrl: string): string { + try { + return ` · ${new URL(targetUrl).hostname.replace(/^www\./, '')}`; + } catch { + return ''; + } +} + +type ResolvedProfile = Awaited< + ReturnType +>; + +/** + * Runs an automation's steps in order, each on its own connection's saved + * session, and rolls the per-step results into a single run verdict. Owns the + * per-step evidence bookkeeping (BrowserAutomationStepRun) and profile health + * updates so the execution service stays focused on run lifecycle. + */ +@Injectable() +export class BrowserAutomationStepRunnerService { + private readonly logger = new Logger(BrowserAutomationStepRunnerService.name); + + constructor( + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + new BrowserbaseSessionService(), + ), + private readonly runner: BrowserEvidenceRunnerService = new BrowserEvidenceRunnerService( + new BrowserbaseSessionService(), + ), + private readonly runs: BrowserAutomationRunStoreService = new BrowserAutomationRunStoreService(), + ) {} + + /** The connection a step runs on: its bound profile, or one matched by host. */ + async resolveStepProfile(input: { + organizationId: string; + step: StepForRun; + }): Promise { + try { + if (input.step.profileId) { + const bound = await this.profiles.getProfile({ + profileId: input.step.profileId, + organizationId: input.organizationId, + }); + if (bound) return bound; + } + return await this.profiles.resolveProfileForTarget({ + organizationId: input.organizationId, + targetUrl: input.step.targetUrl, + }); + } catch { + return null; + } + } + + async runSteps(input: { + organizationId: string; + taskId: string; + automationId: string; + runId: string; + steps: StepForRun[]; + firstProfile: ResolvedProfile; + /** When set, step 0 runs on this already-open (live) session, not a new one. */ + firstSessionId?: string; + /** Live activity timeline, streamed to the Run view via realtime. */ + onSteps?: (steps: EvidenceTimelineStep[]) => void; + /** Follow each vendor's live view as the run advances to it. */ + onLiveView?: (url: string) => void; + /** Live-view phase so the UI can cover the iframe between/after vendors. */ + onLivePhase?: (phase: BrowserRunLivePhase) => void; + }): Promise { + const multiStep = input.steps.length > 1; + const timeline = createEvidenceTimeline(input.onSteps); + const results: BrowserEvidenceRunResult[] = []; + for (let index = 0; index < input.steps.length; index += 1) { + const profile = + index === 0 + ? input.firstProfile + : await this.resolveStepProfile({ + organizationId: input.organizationId, + step: input.steps[index], + }); + // Mark each vendor boundary so the combined timeline reads GH → AWS → … + if (multiStep) { + timeline.step(`Step ${index + 1}${hostSuffix(input.steps[index].targetUrl)}`); + } + results.push( + await this.runStep({ + organizationId: input.organizationId, + taskId: input.taskId, + automationId: input.automationId, + runId: input.runId, + step: input.steps[index], + index, + profile, + // Step 0 runs on the pre-opened live session (so it's watchable); + // later vendors each get their own session inside runEvidence. + sessionId: index === 0 ? input.firstSessionId : undefined, + onLog: (entry) => timeline.step(entry.message), + onLiveView: input.onLiveView, + // When this vendor's session is torn down, tell the UI whether another + // vendor is coming ("switching") or the run is wrapping up ("finishing"). + onSessionClosing: () => + input.onLivePhase?.( + index === input.steps.length - 1 ? 'finishing' : 'switching', + ), + }), + ); + } + const rolled = rollUpStepResults(results); + timeline.finish( + rolled.success ? 'done' : rolled.status === 'blocked' ? 'warn' : 'fail', + ); + return rolled; + } + + private async runStep(input: { + organizationId: string; + taskId: string; + automationId: string; + runId: string; + step: StepForRun; + index: number; + profile: ResolvedProfile; + /** Run this step on an existing (live) session instead of a fresh one. */ + sessionId?: string; + /** Per-stage progress, streamed into the run timeline. */ + onLog?: (log: BrowserEvidenceLog) => void; + /** This step's live view once its session opens (fresh-session steps only). */ + onLiveView?: (url: string) => void; + /** Fired before this step's fresh session is closed (fresh-session steps only). */ + onSessionClosing?: () => void; + }): Promise { + const stepRun = await this.runs.createStepRun({ + runId: input.runId, + stepId: input.step.id, + order: input.index, + }); + const { profile } = input; + + let result: BrowserEvidenceRunResult; + if (!profile) { + result = profileMissingResult(); + } else { + const canAutoRelogin = + profile.status === 'needs_reauth' && Boolean(profile.vaultProvider); + if (profile.status !== 'verified' && !canAutoRelogin) { + result = profileBlockedResult(profile.status); + } else { + try { + const runInput = { + organizationId: input.organizationId, + taskId: input.taskId, + automationId: input.automationId, + // Unique per step so screenshots don't collide under one run id. + runId: stepRun.id, + targetUrl: input.step.targetUrl, + instruction: input.step.instruction, + evaluationCriteria: input.step.evaluationCriteria, + profile: { + id: profile.id, + hostname: profile.hostname, + contextId: profile.contextId, + vaultProvider: profile.vaultProvider, + vaultExternalItemRef: profile.vaultExternalItemRef, + vaultConnectionId: profile.vaultConnectionId, + }, + onLog: input.onLog, + }; + result = input.sessionId + ? await this.runner.executeEvidenceOnSession({ + ...runInput, + sessionId: input.sessionId, + }) + : await this.runner.runEvidence({ + ...runInput, + onSession: input.onLiveView + ? (info) => input.onLiveView?.(info.liveViewUrl) + : undefined, + onSessionClosing: input.onSessionClosing, + }); + } catch (error) { + this.logger.error('Browser evidence runner failed', error); + result = failedBrowserEvidenceRunResult(error); + } + } + } + + await this.runs.finishStepRun({ stepRunId: stepRun.id, result }); + if (profile) { + await this.applyProfileResult({ + organizationId: input.organizationId, + profileId: profile.id, + result, + }); + } + return result; + } + + /** Reflect a run's outcome onto the connection's health (verified/reauth/blocked). */ + async applyProfileResult(input: { + organizationId: string; + profileId: string; + result: BrowserEvidenceRunResult; + }) { + if (input.result.success) { + await this.profiles.markVerified({ + organizationId: input.organizationId, + profileId: input.profileId, + }); + return; + } + + if (input.result.failureCode === 'needs_reauth') { + await this.profiles.markNeedsReauth({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: input.result.blockedReason, + }); + } + + if ( + input.result.failureCode === 'captcha_blocked' || + input.result.failureCode === 'needs_user_action' + ) { + await this.profiles.markBlocked({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: + input.result.blockedReason ?? + input.result.error ?? + 'User action is required before this automation can run.', + }); + } + } +} diff --git a/apps/api/src/browserbase/browser-credential-login.spec.ts b/apps/api/src/browserbase/browser-credential-login.spec.ts new file mode 100644 index 0000000000..97da611df8 --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-login.spec.ts @@ -0,0 +1,187 @@ +import { + performCredentialLogin, + reloginWithStoredCredentials, +} from './browser-credential-login'; +import type { BrowserCredentialVaultAdapter } from './credential-vault'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { BrowserEvidenceSessionInput } from './browser-evidence-runner.service'; + +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const makeStagehand = () => ({ act: jest.fn().mockResolvedValue(undefined) }); +const makePage = () => ({ goto: jest.fn().mockResolvedValue(undefined) }); +const makeSessions = (page: ReturnType) => ({ + ensureActivePage: jest.fn().mockResolvedValue(page), +}); + +const baseInput: BrowserEvidenceSessionInput = { + organizationId: 'org_1', + automationId: 'bau_1', + runId: 'bar_1', + targetUrl: 'https://vendor.example.com/app', + instruction: 'capture evidence', + profile: { + id: 'bap_1', + hostname: 'vendor.example.com', + contextId: 'ctx_1', + vaultProvider: '1password', + vaultExternalItemRef: 'op://v/i', + vaultConnectionId: 'v', + }, + sessionId: 'sess_1', +}; + +describe('performCredentialLogin', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + it('fills each field in its own action with secrets passed via variables', async () => { + const stagehand = makeStagehand(); + const password = 'sup3r-s3cret-passphrase'; + + const promise = performCredentialLogin({ + stagehand: stagehand as unknown as Stagehand, + credentials: { username: 'alice', password, totpCode: '424242' }, + log: jest.fn(), + }); + await jest.runAllTimersAsync(); + await promise; + + // act() does one thing per call, so username, password and the code are + // each their own step (a single combined instruction would only fill one). + const calls = stagehand.act.mock.calls as [string, unknown?][]; + const findCall = (needle: string) => + calls.find(([instruction]) => instruction.includes(needle)); + + expect(findCall('%username%')?.[1]).toEqual({ + variables: { username: 'alice' }, + }); + expect(findCall('%password%')?.[1]).toEqual({ variables: { password } }); + expect(findCall('%code%')?.[1]).toEqual({ variables: { code: '424242' } }); + + // No raw secret value may appear in any instruction sent to the LLM. + for (const [instruction] of calls) { + expect(instruction).not.toContain(password); + expect(instruction).not.toContain('alice'); + expect(instruction).not.toContain('424242'); + } + }); +}); + +describe('reloginWithStoredCredentials', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + const runRelogin = async (args: { + stagehand: ReturnType; + sessions: ReturnType; + vault: BrowserCredentialVaultAdapter; + verifyLoggedIn: jest.Mock; + }) => { + const promise = reloginWithStoredCredentials({ + stagehand: args.stagehand as unknown as Stagehand, + sessions: args.sessions as unknown as BrowserbaseSessionService, + vault: args.vault, + input: baseInput, + verifyLoggedIn: args.verifyLoggedIn, + log: jest.fn(), + }); + await jest.runAllTimersAsync(); + return promise; + }; + + it('does not attempt a login when no credentials are stored', async () => { + const stagehand = makeStagehand(); + const page = makePage(); + const vault: BrowserCredentialVaultAdapter = { + resolveCredentialReference: jest.fn().mockResolvedValue(null), + }; + + const result = await runRelogin({ + stagehand, + sessions: makeSessions(page), + vault, + verifyLoggedIn: jest.fn().mockResolvedValue(false), + }); + + expect(result.isLoggedIn).toBe(false); + expect(result.reason).toMatch(/no stored credentials/i); + expect(stagehand.act).not.toHaveBeenCalled(); + }); + + it('signs in and verifies on the landing page without forcing a return to the URL', async () => { + const stagehand = makeStagehand(); + const page = makePage(); + const vault: BrowserCredentialVaultAdapter = { + resolveCredentialReference: jest + .fn() + .mockResolvedValue({ username: 'alice', password: 'pw' }), + }; + + const result = await runRelogin({ + stagehand, + sessions: makeSessions(page), + vault, + verifyLoggedIn: jest.fn().mockResolvedValue(true), + }); + + expect(result.isLoggedIn).toBe(true); + expect(stagehand.act).toHaveBeenCalled(); + // We stay on the post-login landing page and verify there — navigating back + // to the entered URL can return to a login page and read as a failed sign-in. + expect(page.goto).not.toHaveBeenCalled(); + }); + + it('retries once with a freshly resolved TOTP code', async () => { + const stagehand = makeStagehand(); + const page = makePage(); + const resolveCredentialReference = jest + .fn() + .mockResolvedValueOnce({ + username: 'alice', + password: 'pw', + totpCode: '111111', + }) + .mockResolvedValueOnce({ + username: 'alice', + password: 'pw', + totpCode: '222222', + }); + const vault: BrowserCredentialVaultAdapter = { resolveCredentialReference }; + const verifyLoggedIn = jest + .fn() + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true); + + const result = await runRelogin({ + stagehand, + sessions: makeSessions(page), + vault, + verifyLoggedIn, + }); + + expect(result.isLoggedIn).toBe(true); + // Resolving twice proves the second attempt used a freshly generated code. + expect(resolveCredentialReference).toHaveBeenCalledTimes(2); + }); + + it('gives up (user action required) when sign-in never authenticates', async () => { + const stagehand = makeStagehand(); + const page = makePage(); + const vault: BrowserCredentialVaultAdapter = { + resolveCredentialReference: jest + .fn() + .mockResolvedValue({ username: 'alice', password: 'pw' }), + }; + + const result = await runRelogin({ + stagehand, + sessions: makeSessions(page), + vault, + verifyLoggedIn: jest.fn().mockResolvedValue(false), + }); + + expect(result.isLoggedIn).toBe(false); + expect(result.reason).toMatch(/user action/i); + }); +}); diff --git a/apps/api/src/browserbase/browser-credential-login.ts b/apps/api/src/browserbase/browser-credential-login.ts new file mode 100644 index 0000000000..f0470c40e3 --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-login.ts @@ -0,0 +1,312 @@ +import { z } from 'zod'; +import type { + BrowserCredentialVaultAdapter, + RuntimeCredentialMaterial, +} from './credential-vault'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; + +type Stagehand = import('@browserbasehq/stagehand').Stagehand; +type ActivePage = Awaited< + ReturnType +>; + +/** + * The minimum a credential sign-in needs: which profile's stored login to + * resolve, and where to land afterward. Both the scheduler (evidence runs) and + * the connect flow's first sign-in satisfy this shape, so neither is coupled to + * the other's input type. + */ +export interface CredentialLoginTarget { + profile: { + id: string; + vaultProvider?: string | null; + vaultExternalItemRef?: string | null; + vaultConnectionId?: string | null; + }; + targetUrl: string; +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Drives an automated sign-in using stored credentials. Secret values are passed + * through Stagehand's `variables` substitution, so they are injected into the page + * at execution time and are never sent to the model as prompt text. + */ +export async function performCredentialLogin({ + stagehand, + credentials, + log, + usernameLabel, +}: { + stagehand: Stagehand; + credentials: RuntimeCredentialMaterial; + log: (message: string) => void; + /** The vendor's own label for the identifier field (e.g. "IAM username"), so + * the streamed step matches what's on screen. Falls back to "username". */ + usernameLabel?: string; +}): Promise { + // Stagehand's act() performs ONE action per call, so each field and the submit + // are separate steps — a single "enter username and password and submit" + // instruction would only fill the first field. Secrets go through `variables` + // substitution, so only placeholders (not the values) ever reach the model. + + // Site-specific fields (workspace, subdomain, …) usually precede the password. + const extraFields = credentials.extraFields ?? []; + for (const field of extraFields) { + log(`Entering ${field.label}.`); + await stagehand.act( + `Enter %value% into the "${field.label}" field on the sign-in form.`, + { variables: { value: field.value } }, + ); + await delay(1000); + } + + if (credentials.username) { + // Show the vendor's real field name in the step; the fill stays semantic so + // Stagehand still finds the field even if the label is approximate. + log(`Entering ${usernameLabel?.trim() || 'username'}.`); + await stagehand.act( + 'Enter %username% into the email or username field on the sign-in form.', + { variables: { username: credentials.username } }, + ); + await delay(1000); + // Two-step logins (Microsoft, Google, Okta) reveal the password only after a + // Next/Continue; single-page forms (e.g. GitHub) already show it. + await stagehand.act( + 'If no password field is visible yet, click the "Next" or "Continue" button to proceed. If a password field is already visible, do nothing.', + ); + await delay(1000); + } + + if (credentials.password) { + log('Entering password.'); + await stagehand.act( + 'Enter %password% into the password field on the sign-in form.', + { variables: { password: credentials.password } }, + ); + await delay(1000); + } + + if (credentials.username || credentials.password) { + log('Submitting the sign-in form.'); + await stagehand.act( + 'Click the "Sign in" or "Log in" button to submit the sign-in form.', + ); + await delay(2000); + } + + if (credentials.totpCode) { + log('Entering one-time passcode.'); + await stagehand.act( + 'If a one-time passcode, two-factor, or verification code field is shown, enter %code% into it. If no such field is present, do nothing.', + { variables: { code: credentials.totpCode } }, + ); + await delay(1500); + await stagehand.act( + 'If there is a button to submit or verify the code, click it. Otherwise do nothing.', + ); + await delay(2000); + } +} + +export interface CredentialReloginResult { + isLoggedIn: boolean; + page: ActivePage; + reason?: string; +} + +/** + * Attempts to recover an expired session using credentials stored for the profile. + * Returns `isLoggedIn: false` (with a reason) when no credentials exist or the + * automated sign-in couldn't complete — e.g. a login step needs a human (SMS, + * email code, push approval), which is left to the existing re-auth fallback. + */ +export async function reloginWithStoredCredentials({ + stagehand, + sessions, + vault, + input, + verifyLoggedIn, + log, +}: { + stagehand: Stagehand; + sessions: BrowserbaseSessionService; + vault: BrowserCredentialVaultAdapter; + input: CredentialLoginTarget; + verifyLoggedIn: () => Promise; + log: (message: string) => void; +}): Promise { + const resolveCredentials = () => + vault.resolveCredentialReference({ + profileId: input.profile.id, + provider: input.profile.vaultProvider, + externalItemRef: input.profile.vaultExternalItemRef, + connectionId: input.profile.vaultConnectionId, + }); + + const credentials = await resolveCredentials(); + let page = await sessions.ensureActivePage(stagehand); + + if (!credentials) { + return { + isLoggedIn: false, + page, + reason: 'Session expired and no stored credentials are available.', + }; + } + + page = await runLoginAttempt({ + stagehand, + sessions, + credentials, + log, + }); + if (await verifyLoggedIn()) return { isLoggedIn: true, page }; + + // Retry once with a freshly resolved TOTP code, which guards against the ~30s + // rotation boundary landing between resolve and submit. + if (credentials.totpCode) { + const fresh = await resolveCredentials(); + if (fresh?.totpCode) { + page = await runLoginAttempt({ + stagehand, + sessions, + credentials: fresh, + log, + }); + if (await verifyLoggedIn()) return { isLoggedIn: true, page }; + } + } + + return { + isLoggedIn: false, + page, + reason: 'Automated sign-in did not complete; user action may be required.', + }; +} + +async function runLoginAttempt({ + stagehand, + sessions, + credentials, + log, +}: { + stagehand: Stagehand; + sessions: BrowserbaseSessionService; + credentials: RuntimeCredentialMaterial; + log: (message: string) => void; +}): Promise { + await performCredentialLogin({ stagehand, credentials, log }); + // Stay on the page the site lands on after login (typically an app/dashboard) + // and verify there. Navigating back to the entered URL can return to a login + // page — some apps always show login at the root — which reads as a failed + // sign-in even though it succeeded. The evidence instruction navigates from + // wherever we land. + await delay(1500); + return sessions.ensureActivePage(stagehand); +} + +export type SignInOutcome = + | 'logged_in' + | 'invalid_credentials' + | 'needs_2fa' + | 'challenge' + | 'unknown'; + +/** + * Reads the current page after a sign-in attempt and classifies the outcome, so + * the connect flow can tell the user what happened and route them correctly. + * Never throws — an unreadable page degrades to 'unknown'. + */ +export async function classifyLoginOutcome( + stagehand: Stagehand, +): Promise { + try { + // Give the model where the browser actually is, so it can judge for itself + // whether we're on the real app or still on a sign-in / identity-provider + // page (it knows hosts like signin.aws.amazon.com or login.microsoftonline.com + // without us hardcoding URL patterns). A hint only — falls back to content. + let currentUrl = ''; + try { + const pages = stagehand.context?.pages?.() ?? []; + currentUrl = pages[pages.length - 1]?.url() ?? ''; + } catch { + // URL unavailable — classify from page content alone. + } + const { state } = await stagehand.extract( + (currentUrl ? `The browser is currently at this URL: ${currentUrl}\n\n` : '') + + 'Classify this page after a sign-in attempt, using BOTH the page content AND the URL. ' + + 'Return exactly one value: ' + + '"logged_in" — there is clear evidence the user is signed in to the actual application ' + + '(a dashboard, an account/avatar menu, or a "Sign out" control) AND the browser is on the ' + + 'application itself. If the URL is a sign-in, login, SSO, or identity-provider page, or the ' + + 'page is blank, loading, redirecting, or still shows a sign-in form or a "Sign in" / ' + + '"Log in" button, it is NOT logged_in; ' + + '"invalid_credentials" — it shows an incorrect username/email/password error; ' + + '"needs_2fa" — it asks for a two-factor, one-time, authenticator, or verification code; ' + + '"challenge" — it shows a CAPTCHA, a "verify it\'s you", a device-approval, or an email/SMS link step; ' + + '"unknown" — none of the above clearly apply.', + z.object({ + state: z.enum([ + 'logged_in', + 'invalid_credentials', + 'needs_2fa', + 'challenge', + 'unknown', + ]), + }), + ); + return state; + } catch { + return 'unknown'; + } +} + +/** + * The connect flow's first sign-in: fill the stored credentials and classify + * where we land. Unlike reloginWithStoredCredentials (used by the scheduler), + * this does NOT navigate back to the target afterward — the post-submit page is + * exactly what we need to classify (an error, a 2FA prompt, a challenge, or a + * signed-in dashboard). + */ +export async function signInAndClassify({ + stagehand, + vault, + input, + log, + usernameLabel, +}: { + stagehand: Stagehand; + vault: BrowserCredentialVaultAdapter; + input: CredentialLoginTarget; + log: (message: string) => void; + /** Vendor's identifier-field label, for a truthful streamed step. */ + usernameLabel?: string; +}): Promise<{ outcome: SignInOutcome }> { + const resolveCredentials = () => + vault.resolveCredentialReference({ + profileId: input.profile.id, + provider: input.profile.vaultProvider, + externalItemRef: input.profile.vaultExternalItemRef, + connectionId: input.profile.vaultConnectionId, + }); + + const credentials = await resolveCredentials(); + if (!credentials) return { outcome: 'unknown' }; + + await performCredentialLogin({ stagehand, credentials, log, usernameLabel }); + let outcome = await classifyLoginOutcome(stagehand); + + // A 2FA prompt still showing with a stored seed can mean the code landed on + // the ~30s rotation boundary — retry once with a freshly generated code. + if (outcome === 'needs_2fa' && credentials.totpCode) { + const fresh = await resolveCredentials(); + if (fresh?.totpCode) { + await performCredentialLogin({ stagehand, credentials: fresh, log, usernameLabel }); + outcome = await classifyLoginOutcome(stagehand); + } + } + + return { outcome }; +} diff --git a/apps/api/src/browserbase/browser-credential-signin.service.spec.ts b/apps/api/src/browserbase/browser-credential-signin.service.spec.ts new file mode 100644 index 0000000000..115006ac40 --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-signin.service.spec.ts @@ -0,0 +1,177 @@ +import { NotFoundException } from '@nestjs/common'; +import { BrowserCredentialSigninService } from './browser-credential-signin.service'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; + +// Avoid instantiating the real Prisma client at import time (the service pulls +// in the profile service, which imports @db). Collaborators are mocked below. +jest.mock('@db', () => ({ db: {} })); + +jest.mock('./browser-credential-vault.factory', () => ({ + resolveBrowserCredentialVaultAdapter: jest.fn(), +})); + +const mockedResolveVault = resolveBrowserCredentialVaultAdapter as jest.Mock; + +const profile = { + id: 'prof_1', + organizationId: 'org_1', + hostname: 'app.example.com', + contextId: 'ctx_1', + vaultProvider: '1password', + vaultExternalItemRef: 'op://vault/item', + vaultConnectionId: 'vault', +}; + +// `extract` classifies the page — return { state: } per call. +function makeSessions(extract: jest.Mock, act: jest.Mock) { + const page = { + goto: jest.fn().mockResolvedValue(undefined), + url: jest.fn().mockReturnValue('https://app.example.com/home'), + }; + return { + createStagehand: jest.fn().mockResolvedValue({ extract, act }), + ensureActivePage: jest.fn().mockResolvedValue(page), + safeCloseStagehand: jest.fn().mockResolvedValue(undefined), + closeSession: jest.fn().mockResolvedValue(undefined), + }; +} + +function makeProfiles(found: typeof profile | null) { + return { + getProfile: jest.fn().mockResolvedValue(found), + markVerified: jest.fn().mockResolvedValue(found), + markNeedsReauth: jest.fn().mockResolvedValue(found), + }; +} + +const input = { + organizationId: 'org_1', + profileId: 'prof_1', + url: 'https://app.example.com', + sessionId: 'sess_1', +}; + +const withCredentials = (creds: Record | null) => + mockedResolveVault.mockReturnValue({ + resolveCredentialReference: jest.fn().mockResolvedValue(creds), + }); + +describe('BrowserCredentialSigninService', () => { + beforeEach(() => { + jest.useFakeTimers(); + mockedResolveVault.mockReset(); + }); + afterEach(() => jest.useRealTimers()); + + const run = async ( + sessions: ReturnType, + profiles: ReturnType, + ) => { + const service = new BrowserCredentialSigninService( + sessions as unknown as BrowserbaseSessionService, + profiles as unknown as BrowserAuthProfileService, + ); + const promise = service.signInWithStoredCredentials(input); + await jest.runAllTimersAsync(); + return promise; + }; + + it('connects to the given session and never closes it', async () => { + const extract = jest.fn().mockResolvedValue({ state: 'logged_in' }); + const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined)); + const profiles = makeProfiles(profile); + withCredentials({ username: 'u', password: 'p' }); + + await run(sessions, profiles); + + expect(sessions.createStagehand).toHaveBeenCalledWith('sess_1'); + // Release our automation handle, but leave the session open for the user. + expect(sessions.safeCloseStagehand).toHaveBeenCalledTimes(1); + expect(sessions.closeSession).not.toHaveBeenCalled(); + }); + + it('marks verified without re-entering credentials when already signed in', async () => { + const extract = jest.fn().mockResolvedValue({ state: 'logged_in' }); + const act = jest.fn().mockResolvedValue(undefined); + const sessions = makeSessions(extract, act); + const profiles = makeProfiles(profile); + withCredentials({ username: 'u', password: 'p' }); + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(true); + expect(profiles.markVerified).toHaveBeenCalledTimes(1); + expect(act).not.toHaveBeenCalled(); // already in — no navigation or fill + }); + + it('signs in with stored credentials and marks verified', async () => { + const extract = jest + .fn() + .mockResolvedValueOnce({ state: 'unknown' }) + .mockResolvedValue({ state: 'logged_in' }); + const act = jest.fn().mockResolvedValue(undefined); + const sessions = makeSessions(extract, act); + const profiles = makeProfiles(profile); + withCredentials({ username: 'user@x.com', password: 'secret' }); + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(true); + // The authenticated landing page is returned so runs can target it directly. + expect(result.homeUrl).toBe('https://app.example.com/home'); + expect(act).toHaveBeenCalled(); // navigate to sign-in + fill + expect(profiles.markVerified).toHaveBeenCalledTimes(1); + expect(profiles.markNeedsReauth).not.toHaveBeenCalled(); + }); + + it('reports invalid credentials as a failure and marks needs-reauth', async () => { + const extract = jest + .fn() + .mockResolvedValueOnce({ state: 'unknown' }) + .mockResolvedValue({ state: 'invalid_credentials' }); + const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined)); + const profiles = makeProfiles(profile); + withCredentials({ username: 'user@x.com', password: 'wrong' }); + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(false); + expect(result.failure).toBe('invalid_credentials'); + expect(profiles.markNeedsReauth).toHaveBeenCalledTimes(1); + expect(profiles.markVerified).not.toHaveBeenCalled(); + // Session stays open so the user can take over. + expect(sessions.closeSession).not.toHaveBeenCalled(); + }); + + it('reports needs_2fa when a two-factor prompt blocks an account with no seed', async () => { + const extract = jest + .fn() + .mockResolvedValueOnce({ state: 'unknown' }) + .mockResolvedValue({ state: 'needs_2fa' }); + const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined)); + const profiles = makeProfiles(profile); + withCredentials({ username: 'user@x.com', password: 'secret' }); // no totpCode + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(false); + expect(result.failure).toBe('needs_2fa'); + expect(profiles.markNeedsReauth).toHaveBeenCalledTimes(1); + }); + + it('throws when the profile does not exist', async () => { + const sessions = makeSessions(jest.fn(), jest.fn()); + const profiles = makeProfiles(null); + const service = new BrowserCredentialSigninService( + sessions as unknown as BrowserbaseSessionService, + profiles as unknown as BrowserAuthProfileService, + ); + + await expect( + service.signInWithStoredCredentials(input), + ).rejects.toBeInstanceOf(NotFoundException); + expect(sessions.createStagehand).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browser-credential-signin.service.ts b/apps/api/src/browserbase/browser-credential-signin.service.ts new file mode 100644 index 0000000000..2306bac154 --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-signin.service.ts @@ -0,0 +1,228 @@ +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { + classifyLoginOutcome, + signInAndClassify, +} from './browser-credential-login'; +import { navigateToSignIn } from './browser-login-navigation'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; + +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export type AutoSignInFailure = + | 'invalid_credentials' + | 'needs_2fa' + | 'challenge' + // SSO: the AI opened the identity provider; the user finishes the login there. + | 'sso_handoff' + | 'unknown'; + +export interface SignInStep { + /** Step label. */ + l: string; + /** Clock timestamp, e.g. "06:02:14". */ + t: string; + state: 'done' | 'active' | 'pending' | 'warn' | 'fail'; +} + +const clock = () => + new Date().toLocaleTimeString('en-US', { + hour12: false, + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + +export interface AutoSignInResult { + isLoggedIn: boolean; + /** Why the automated sign-in couldn't complete (set only when not signed in). */ + failure?: AutoSignInFailure; + /** + * The page the app landed on after a successful login — an authenticated URL to + * target on future runs, so they reuse the session instead of re-logging in + * (e.g. sites whose root always shows a login form). + */ + homeUrl?: string; +} + +const FAILURE_REASON: Record = { + invalid_credentials: 'The stored username or password was not accepted.', + needs_2fa: 'The account requires a two-factor code to sign in.', + challenge: 'The site asked for a human verification step.', + sso_handoff: 'Finish signing in with your identity provider.', + unknown: 'Automated sign-in could not complete.', +}; + +/** + * Performs the connect flow's first sign-in on a session the caller already + * created and is showing to the user as a live view. The user watches the + * auto-fill happen; if it can't finish (wrong password, 2FA, a human + * challenge), we leave the session open on the exact page it stopped on so the + * user can take over. We only close our own Stagehand handle — never the + * session, which the caller owns. + */ +@Injectable() +export class BrowserCredentialSigninService { + private readonly logger = new Logger(BrowserCredentialSigninService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + sessions, + ), + ) {} + + /** + * Emit the live-view URL of the tab the AI is currently on, so the connect + * flow's iframe follows it across new tabs. Best-effort — a failure just + * leaves the live view on its current tab. + */ + private async streamActiveLiveView( + stagehand: Stagehand, + sessionId: string, + onLiveView?: (url: string) => void, + ): Promise { + if (!onLiveView) return; + try { + const pages = stagehand.context?.pages?.() ?? []; + const activeUrl = pages[pages.length - 1]?.url(); + const url = await this.sessions.getActivePageLiveViewUrl(sessionId, activeUrl); + if (url) onLiveView(url); + } catch { + // Best-effort — keep the current live view. + } + } + + async signInWithStoredCredentials(input: { + organizationId: string; + profileId: string; + url: string; + sessionId: string; + /** 'password' fills stored credentials; 'sso' drives to the identity provider. */ + mode?: 'password' | 'sso'; + /** The vendor's detected identifier-field label (e.g. "IAM username"), so the + * streamed step shows the real field name instead of a generic "username". */ + usernameLabel?: string; + /** Live activity timeline, surfaced to the connect flow's activity panel. */ + onSteps?: (steps: SignInStep[]) => void; + /** Follow the AI's tab: emit the live-view URL of the page it's actually on + * (a vendor may open sign-in in a new tab) so the UI iframe can follow it. */ + onLiveView?: (url: string) => void; + }): Promise { + const mode = input.mode ?? 'password'; + const steps: SignInStep[] = []; + const emit = () => input.onSteps?.(steps.map((s) => ({ ...s }))); + // Each call advances the timeline: the prior active step becomes done and a + // new active step is appended. + const step = (label: string) => { + const last = steps[steps.length - 1]; + if (last?.state === 'active') last.state = 'done'; + steps.push({ l: label, t: clock(), state: 'active' }); + this.logger.log(`[sign-in] ${label}`); + emit(); + }; + const finish = (state: SignInStep['state']) => { + const last = steps[steps.length - 1]; + if (last) last.state = state; + emit(); + }; + + const profile = await this.profiles.getProfile({ + profileId: input.profileId, + organizationId: input.organizationId, + }); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + + let stagehand: Stagehand | null = null; + try { + stagehand = await this.sessions.createStagehand(input.sessionId); + const activeStagehand = stagehand; + const page = await this.sessions.ensureActivePage(activeStagehand); + step('Opening the sign-in page'); + await page.goto(input.url, { + waitUntil: 'domcontentloaded', + timeoutMs: 30000, + }); + await delay(1500); + + // The persisted context may already carry a valid session — no need to + // re-enter credentials if we're already in. The classifier is URL-aware, so + // it won't call a mid-redirect / sign-in page "logged in" (the AWS false + // positive) — it decides from the page AND where the browser actually is. + step('Checking if you’re already signed in'); + if ((await classifyLoginOutcome(activeStagehand)) === 'logged_in') { + await this.profiles.markVerified(input); + finish('done'); + return { isLoggedIn: true, homeUrl: page.url() }; + } + + // Get onto the actual sign-in form first — the entered URL may be a + // homepage or dashboard rather than the login page. + step('Finding the sign-in form'); + await navigateToSignIn(activeStagehand); + // The sign-in may have opened in a new tab — point the live view at + // wherever the AI actually is, so the user watches (and can complete a + // 2FA take-over on) the right page. + await this.streamActiveLiveView(activeStagehand, input.sessionId, input.onLiveView); + + // SSO: we can't hold the customer's identity-provider credentials, so the + // AI only clicks through to the provider, then hands the live browser to + // the user to finish there (their IdP login + MFA). + if (mode === 'sso') { + step('Opening your single sign-on provider'); + try { + await activeStagehand.act( + "Click the option to sign in with single sign-on (SSO) or an identity provider — buttons labeled like 'Sign in with SSO', 'Continue with Google/Microsoft/Okta', 'Use SSO', or a company login. Do not type any username or password.", + ); + } catch { + // Best effort — if we can't find the button, the user can click it in + // the live browser during take-over. + } + await delay(2500); + // The identity provider often opens in a new tab — follow it. + await this.streamActiveLiveView(activeStagehand, input.sessionId, input.onLiveView); + step('Finish signing in with your provider'); + finish('warn'); + return { isLoggedIn: false, failure: 'sso_handoff' }; + } + + const vault = resolveBrowserCredentialVaultAdapter(); + const { outcome } = await signInAndClassify({ + stagehand: activeStagehand, + vault, + input: { profile, targetUrl: input.url }, + log: step, + // On reconnect the caller doesn't pass a label (no capture form), so fall + // back to the one detected + stored at connect time. + usernameLabel: input.usernameLabel ?? profile.identifierLabel ?? undefined, + }); + step('Checking whether that worked'); + + if (outcome === 'logged_in') { + await this.profiles.markVerified(input); + finish('done'); + // Re-read the active page: signing in usually navigates to an app/home page. + const landed = await this.sessions.ensureActivePage(activeStagehand); + return { isLoggedIn: true, homeUrl: landed.url() }; + } + + // Narrowed to the failure states now that logged_in is handled. + await this.profiles.markNeedsReauth({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: FAILURE_REASON[outcome], + }); + finish('warn'); + return { isLoggedIn: false, failure: outcome }; + } finally { + // Release our automation handle but leave the session open — the caller + // shows it to the user (to watch, or to take over) and closes it later. + if (stagehand) await this.sessions.safeCloseStagehand(stagehand); + } + } +} diff --git a/apps/api/src/browserbase/browser-credential-storage.service.spec.ts b/apps/api/src/browserbase/browser-credential-storage.service.spec.ts new file mode 100644 index 0000000000..91cf2b021c --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-storage.service.spec.ts @@ -0,0 +1,201 @@ +import { BadRequestException } from '@nestjs/common'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import * as opClient from './onepassword-client'; +import { TOTP_FIELD_TITLE, buildItemReference } from './onepassword-credential-item'; + +jest.mock('@db', () => ({ + db: { browserAuthProfile: { findFirst: jest.fn(), update: jest.fn() } }, +})); +jest.mock('./onepassword-client'); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { db } = require('@db'); +const findFirst = db.browserAuthProfile.findFirst as jest.Mock; +const update = db.browserAuthProfile.update as jest.Mock; + +const mockConfigured = opClient.isOnePasswordConfigured as jest.Mock; +const mockGetClient = opClient.getOnePasswordClient as jest.Mock; +const mockLoadModule = opClient.loadOnePasswordModule as jest.Mock; + +const REF = buildItemReference('vault-1', 'item-1'); + +interface Field { + id: string; + title: string; + fieldType: string; + value: string; +} + +const field = (title: string, value = 'x'): Field => ({ + id: title, + title, + fieldType: 'Text', + value, +}); + +describe('BrowserCredentialStorageService — TOTP', () => { + let service: BrowserCredentialStorageService; + let itemsGet: jest.Mock; + let itemsPut: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + service = new BrowserCredentialStorageService(); + mockConfigured.mockReturnValue(true); + mockLoadModule.mockResolvedValue({ ItemFieldType: { Totp: 'Totp' } }); + itemsGet = jest.fn(); + itemsPut = jest.fn().mockResolvedValue(undefined); + mockGetClient.mockResolvedValue({ items: { get: itemsGet, put: itemsPut } }); + }); + + const profile = (overrides: Record = {}) => ({ + id: 'bap_1', + organizationId: 'org_1', + vaultExternalItemRef: REF, + ...overrides, + }); + + describe('getProfileTotpStatus', () => { + it('is configured when the item has a non-empty TOTP field', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ + fields: [field('username'), field(TOTP_FIELD_TITLE, 'SEED')], + }); + + const result = await service.getProfileTotpStatus({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: true }); + }); + + it('is not configured when there is no TOTP field', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ fields: [field('username'), field('password')] }); + + const result = await service.getProfileTotpStatus({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + }); + + it('is not configured when the connection has no stored login', async () => { + findFirst.mockResolvedValue(profile({ vaultExternalItemRef: null })); + + const result = await service.getProfileTotpStatus({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + expect(itemsGet).not.toHaveBeenCalled(); + }); + }); + + describe('setProfileTotp', () => { + it('replaces any existing TOTP field and writes the item back', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ + fields: [field('username'), field(TOTP_FIELD_TITLE, 'OLD')], + }); + + const result = await service.setProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + totpSeed: ' NEW SEED ', + }); + + expect(result).toEqual({ configured: true }); + const written = itemsPut.mock.calls[0][0]; + const totpFields = written.fields.filter( + (f: Field) => f.title === TOTP_FIELD_TITLE, + ); + expect(totpFields).toHaveLength(1); + expect(totpFields[0]).toMatchObject({ fieldType: 'Totp', value: 'NEW SEED' }); + }); + + it('rejects when the connection has no stored login', async () => { + findFirst.mockResolvedValue(profile({ vaultExternalItemRef: null })); + + await expect( + service.setProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + totpSeed: 'SEED', + }), + ).rejects.toBeInstanceOf(BadRequestException); + expect(itemsPut).not.toHaveBeenCalled(); + }); + }); + + describe('storeProfileCredentials — identifier label', () => { + it('persists the detected identifier label (trimmed) on the profile', async () => { + findFirst.mockResolvedValue({ + id: 'bap_1', + organizationId: 'org_1', + displayName: 'GitHub', + hostname: 'github.com', + }); + update.mockResolvedValue({}); + mockGetClient.mockResolvedValue({ + vaults: { + list: jest.fn().mockResolvedValue([]), + create: jest.fn().mockResolvedValue({ id: 'vault-1' }), + }, + items: { create: jest.fn().mockResolvedValue({ id: 'item-1' }) }, + }); + mockLoadModule.mockResolvedValue({ + ItemCategory: { Login: 'Login' }, + ItemFieldType: { Text: 'Text', Concealed: 'Concealed', Totp: 'Totp' }, + }); + + await service.storeProfileCredentials({ + organizationId: 'org_1', + profileId: 'bap_1', + username: 'ci-bot', + password: 'secret', + usernameLabel: ' IAM username ', + }); + + expect(update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ identifierLabel: 'IAM username' }), + }), + ); + }); + }); + + describe('clearProfileTotp', () => { + it('drops the TOTP field and writes the item back', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ + fields: [field('username'), field(TOTP_FIELD_TITLE, 'SEED')], + }); + + const result = await service.clearProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + const written = itemsPut.mock.calls[0][0]; + expect(written.fields.some((f: Field) => f.title === TOTP_FIELD_TITLE)).toBe(false); + }); + + it('is a no-op when there is no TOTP field', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ fields: [field('username')] }); + + const result = await service.clearProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + expect(itemsPut).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/api/src/browserbase/browser-credential-storage.service.ts b/apps/api/src/browserbase/browser-credential-storage.service.ts new file mode 100644 index 0000000000..da85ba7c46 --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-storage.service.ts @@ -0,0 +1,314 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, + ServiceUnavailableException, +} from '@nestjs/common'; +import { db } from '@db'; +import type { ItemField } from '@1password/sdk'; +import { + getOnePasswordClient, + isOnePasswordConfigured, + loadOnePasswordModule, + type OnePasswordClient, +} from './onepassword-client'; +import { + ONEPASSWORD_PROVIDER, + PASSWORD_FIELD_TITLE, + TOTP_FIELD_TITLE, + USERNAME_FIELD_TITLE, + buildItemReference, + buildOrgVaultTitle, + parseItemReference, +} from './onepassword-credential-item'; + +export interface StoreProfileCredentialsInput { + organizationId: string; + profileId: string; + username: string; + password: string; + totpSeed?: string; + /** Extra site-specific fields (e.g. workspace, subdomain). */ + extraFields?: { label: string; value: string }[]; + /** The vendor's own label for the identifier field (e.g. "IAM username"). */ + usernameLabel?: string; +} + +/** + * Writes a browser auth profile's login (username, password, optional TOTP seed) + * into a per-organization 1Password vault and points the profile at the created + * item. Comp never persists the raw secrets itself — only the `op://` reference. + */ +@Injectable() +export class BrowserCredentialStorageService { + private readonly logger = new Logger(BrowserCredentialStorageService.name); + + async storeProfileCredentials(input: StoreProfileCredentialsInput) { + if (!isOnePasswordConfigured()) { + throw new ServiceUnavailableException( + 'Credential storage is not configured for this environment.', + ); + } + + const profile = await db.browserAuthProfile.findFirst({ + where: { id: input.profileId, organizationId: input.organizationId }, + }); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + + const client = await getOnePasswordClient(); + const vaultId = await this.ensureOrgVault({ + client, + organizationId: input.organizationId, + }); + + const fields = await this.buildLoginFields({ + username: input.username, + password: input.password, + totpSeed: input.totpSeed, + extraFields: input.extraFields, + }); + + const { ItemCategory } = await loadOnePasswordModule(); + const item = await client.items.create({ + category: ItemCategory.Login, + vaultId, + title: `${profile.displayName} (${profile.hostname})`, + fields, + }); + + const itemRef = buildItemReference(vaultId, item.id); + this.logger.log( + `Stored browser credentials for profile ${profile.id} in 1Password.`, + ); + + return db.browserAuthProfile.update({ + where: { id: profile.id }, + data: { + vaultProvider: ONEPASSWORD_PROVIDER, + vaultExternalItemRef: itemRef, + vaultConnectionId: vaultId, + // Persist the detected identifier label so reconnects and scheduled + // sign-ins show the real field name (undefined → Prisma leaves it as-is). + identifierLabel: input.usernameLabel?.trim() || undefined, + }, + }); + } + + /** + * Whether an authenticator setup key (TOTP seed) is stored for this connection + * — the source of truth for the "Automatic 2FA" status, read live from the + * vault so no DB flag can drift. Returns false (never throws) when storage + * isn't configured or the item can't be read. + */ + async getProfileTotpStatus(input: { + organizationId: string; + profileId: string; + }): Promise<{ configured: boolean }> { + if (!isOnePasswordConfigured()) return { configured: false }; + + const profile = await this.findProfile(input); + const ref = profile.vaultExternalItemRef; + if (!ref) return { configured: false }; + const { vaultId, itemId } = parseItemReference(ref); + if (!vaultId || !itemId) return { configured: false }; + + try { + const client = await getOnePasswordClient(); + const item = await client.items.get(vaultId, itemId); + const configured = item.fields.some( + (field) => field.title === TOTP_FIELD_TITLE && field.value.trim().length > 0, + ); + return { configured }; + } catch (error) { + this.logger.warn('Failed to read TOTP status from 1Password', { + error: error instanceof Error ? error.message : String(error), + }); + return { configured: false }; + } + } + + /** + * Attach or replace the authenticator setup key on a connection's existing + * login item — WITHOUT re-collecting the username/password. Requires the + * connection to already have a stored login. + */ + async setProfileTotp(input: { + organizationId: string; + profileId: string; + totpSeed: string; + }): Promise<{ configured: boolean }> { + if (!isOnePasswordConfigured()) { + throw new ServiceUnavailableException( + 'Credential storage is not configured for this environment.', + ); + } + const seed = input.totpSeed.trim(); + if (!seed) { + throw new BadRequestException('An authenticator setup key is required.'); + } + + const profile = await this.findProfile(input); + const ref = profile.vaultExternalItemRef; + const parsed = ref ? parseItemReference(ref) : { vaultId: '', itemId: '' }; + if (!parsed.vaultId || !parsed.itemId) { + throw new BadRequestException( + 'Store a login for this connection before adding an authenticator key.', + ); + } + + const client = await getOnePasswordClient(); + const { ItemFieldType } = await loadOnePasswordModule(); + const item = await client.items.get(parsed.vaultId, parsed.itemId); + item.fields = [ + ...item.fields.filter((field) => field.title !== TOTP_FIELD_TITLE), + { + id: TOTP_FIELD_TITLE, + title: TOTP_FIELD_TITLE, + fieldType: ItemFieldType.Totp, + value: seed, + }, + ]; + await client.items.put(item); + this.logger.log(`Set authenticator key for profile ${profile.id}.`); + return { configured: true }; + } + + /** + * Remove the stored authenticator setup key from a connection's login item + * ("turn off" automatic 2FA). Idempotent — a no-op when nothing is stored. + */ + async clearProfileTotp(input: { + organizationId: string; + profileId: string; + }): Promise<{ configured: boolean }> { + if (!isOnePasswordConfigured()) return { configured: false }; + + const profile = await this.findProfile(input); + const ref = profile.vaultExternalItemRef; + if (!ref) return { configured: false }; + const { vaultId, itemId } = parseItemReference(ref); + if (!vaultId || !itemId) return { configured: false }; + + const client = await getOnePasswordClient(); + const item = await client.items.get(vaultId, itemId); + const remaining = item.fields.filter( + (field) => field.title !== TOTP_FIELD_TITLE, + ); + if (remaining.length !== item.fields.length) { + item.fields = remaining; + await client.items.put(item); + this.logger.log(`Removed authenticator key for profile ${profile.id}.`); + } + return { configured: false }; + } + + private async findProfile(input: { + organizationId: string; + profileId: string; + }) { + const profile = await db.browserAuthProfile.findFirst({ + where: { id: input.profileId, organizationId: input.organizationId }, + }); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + return profile; + } + + /** + * Best-effort removal of a profile's stored login from 1Password when its + * connection is deleted, so we don't leave orphaned secrets in the vault. + * Never throws — a failed cleanup shouldn't block removing the connection. + */ + async deleteProfileCredentialItem(profile: { + vaultExternalItemRef?: string | null; + }): Promise { + if (!isOnePasswordConfigured() || !profile.vaultExternalItemRef) return; + const { vaultId, itemId } = parseItemReference(profile.vaultExternalItemRef); + if (!vaultId || !itemId) return; + try { + const client = await getOnePasswordClient(); + await client.items.delete(vaultId, itemId); + this.logger.log(`Deleted 1Password item for a removed connection.`); + } catch (error) { + this.logger.warn('Failed to delete 1Password item on connection removal', { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + private async ensureOrgVault({ + client, + organizationId, + }: { + client: OnePasswordClient; + organizationId: string; + }): Promise { + const title = buildOrgVaultTitle(organizationId); + const existing = await client.vaults.list(); + const match = existing.find((vault) => vault.title === title); + if (match) return match.id; + + const created = await client.vaults.create({ + title, + description: `Browser automation logins for organization ${organizationId}.`, + }); + this.logger.log( + `Created 1Password vault for organization ${organizationId}.`, + ); + return created.id; + } + + private async buildLoginFields({ + username, + password, + totpSeed, + extraFields, + }: { + username: string; + password: string; + totpSeed?: string; + extraFields?: { label: string; value: string }[]; + }): Promise { + const { ItemFieldType } = await loadOnePasswordModule(); + const fields: ItemField[] = [ + { + id: USERNAME_FIELD_TITLE, + title: USERNAME_FIELD_TITLE, + fieldType: ItemFieldType.Text, + value: username, + }, + { + id: PASSWORD_FIELD_TITLE, + title: PASSWORD_FIELD_TITLE, + fieldType: ItemFieldType.Concealed, + value: password, + }, + ]; + + if (totpSeed?.trim()) { + fields.push({ + id: TOTP_FIELD_TITLE, + title: TOTP_FIELD_TITLE, + fieldType: ItemFieldType.Totp, + value: totpSeed.trim(), + }); + } + + for (const field of extraFields ?? []) { + const label = field.label.trim(); + if (!label || !field.value.trim()) continue; + fields.push({ + id: label, + title: label, + fieldType: ItemFieldType.Text, + value: field.value, + }); + } + + return fields; + } +} diff --git a/apps/api/src/browserbase/browser-credential-vault.factory.ts b/apps/api/src/browserbase/browser-credential-vault.factory.ts new file mode 100644 index 0000000000..01a4795a69 --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-vault.factory.ts @@ -0,0 +1,18 @@ +import { + type BrowserCredentialVaultAdapter, + NoopBrowserCredentialVaultAdapter, +} from './credential-vault'; +import { isOnePasswordConfigured } from './onepassword-client'; +import { OnePasswordCredentialVaultAdapter } from './onepassword-credential-vault.adapter'; + +/** + * Picks the credential vault adapter for the current runtime: the 1Password + * adapter when a service account token is configured, otherwise the Noop adapter + * (which resolves nothing, preserving today's human re-auth behavior). + */ +export function resolveBrowserCredentialVaultAdapter(): BrowserCredentialVaultAdapter { + if (isOnePasswordConfigured()) { + return new OnePasswordCredentialVaultAdapter(); + } + return new NoopBrowserCredentialVaultAdapter(); +} diff --git a/apps/api/src/browserbase/browser-evidence-evaluation.ts b/apps/api/src/browserbase/browser-evidence-evaluation.ts index bd378205ba..3ac2bfba6f 100644 --- a/apps/api/src/browserbase/browser-evidence-evaluation.ts +++ b/apps/api/src/browserbase/browser-evidence-evaluation.ts @@ -16,10 +16,13 @@ export interface BrowserEvidenceEvaluator { export async function evaluateIfNeeded({ stagehand, criteria, + instruction, logs, }: { stagehand: BrowserEvidenceEvaluator; criteria?: string | null; + /** What the automation was capturing — anchors the check to the intended target. */ + instruction?: string | null; logs: BrowserEvidenceLog[]; }): Promise<{ success: boolean; @@ -43,16 +46,22 @@ export async function evaluateIfNeeded({ pass: z.boolean(), reason: z.string(), }); + const target = instruction?.trim(); const evaluation = await stagehand.extract( [ 'You are an auditor reviewing the current page after an automation has finished navigating.', - 'Decide whether the page clearly satisfies this criteria.', - 'Only return pass=true if the evidence is unambiguously present and visible.', - 'If it is ambiguous, missing, or contradicted, return pass=false.', - 'Always provide a short reason (max 220 characters).', + target + ? `The automation was asked to: "${target}". Judge the criteria about THAT specific target — ignore unrelated items that merely happen to appear on the page (e.g. a matching value belonging to a different item does not count).` + : '', + 'Decide whether the page clearly satisfies this criteria for the intended target.', + 'Only return pass=true if the evidence is unambiguously present and visible for that target.', + 'If it is ambiguous, missing, applies only to a different item, or is contradicted, return pass=false.', + 'Always provide a short reason (max 220 characters) that names the specific value and WHERE on the page it appears, so a reviewer can find it (e.g. "commit count 8,142, in the repository header").', '', `Criteria: ${normalizedCriteria}`, - ].join('\n'), + ] + .filter(Boolean) + .join('\n'), evalSchema, ); diff --git a/apps/api/src/browserbase/browser-evidence-execution.ts b/apps/api/src/browserbase/browser-evidence-execution.ts index 85cc9c6560..a46558148b 100644 --- a/apps/api/src/browserbase/browser-evidence-execution.ts +++ b/apps/api/src/browserbase/browser-evidence-execution.ts @@ -1,4 +1,5 @@ import { Logger } from '@nestjs/common'; +import sharp from 'sharp'; import { z } from 'zod'; import { renderOverlay } from './screenshot-overlay'; import type { BrowserbaseSessionService } from './browserbase-session.service'; @@ -14,10 +15,116 @@ import { classifyBrowserAutomationError, } from './browser-automation-errors'; import type { BrowserEvidenceSessionInput } from './browser-evidence-runner.service'; +import type { BrowserCredentialVaultAdapter } from './credential-vault'; +import { reloginWithStoredCredentials } from './browser-credential-login'; type Stagehand = import('@browserbasehq/stagehand').Stagehand; -const STAGEHAND_CUA_MODEL = 'anthropic/claude-sonnet-4-6'; +// Screenshot-based navigation model. GPT-5.6 Terra is the balanced tier of +// OpenAI's newest family — strong browser-automation accuracy at roughly half +// Sol's cost — a good fit for a multi-step agent. Configurable via env for A/B +// tests (e.g. openai/gpt-5.6-sol for max accuracy), with no per-site tuning. +const DEFAULT_CUA_MODEL = 'openai/gpt-5.6-terra'; +// Claude fallback used when the primary model is unavailable (missing OpenAI key, +// preview access, rate limits, upstream errors). Must be a computer-use-capable +// model Stagehand supports — claude-sonnet-5 is NOT one; the proven Claude CUA +// options are claude-opus-4-8 / claude-sonnet-4-6 / claude-haiku-4-5. +const FALLBACK_CUA_MODEL = 'anthropic/claude-sonnet-4-6'; +// How many screenshot→action steps the agent may take. Generous so it can +// recover from a wrong turn on a complex site rather than giving up. +const CUA_MAX_STEPS = 30; + +// A `type` (not `interface`) so it stays assignable to Stagehand's model config, +// which intersects with Record. +type CuaModel = { + modelName: string; + apiKey?: string; +}; + +function resolveCuaModel(logger: Logger): CuaModel { + const requested = process.env.BROWSERBASE_CUA_MODEL || DEFAULT_CUA_MODEL; + if (requested.startsWith('openai/') && !process.env.OPENAI_API_KEY) { + logger.warn( + `OPENAI_API_KEY not set; falling back from ${requested} to ${FALLBACK_CUA_MODEL} for navigation.`, + ); + return { + modelName: FALLBACK_CUA_MODEL, + apiKey: process.env.ANTHROPIC_API_KEY, + }; + } + return { + modelName: requested, + apiKey: requested.startsWith('openai/') + ? process.env.OPENAI_API_KEY + : process.env.ANTHROPIC_API_KEY, + }; +} + +function claudeFallbackModel(): CuaModel { + return { modelName: FALLBACK_CUA_MODEL, apiKey: process.env.ANTHROPIC_API_KEY }; +} + +async function runCuaNavigation({ + stagehand, + instruction, + model, +}: { + stagehand: Stagehand; + instruction: string; + model: CuaModel; +}): Promise { + await stagehand + .agent({ mode: 'cua', model }) + .execute({ instruction, maxSteps: CUA_MAX_STEPS }); +} + +/** + * The agent's `.execute()` is a single opaque call, but it opens/switches tabs + * mid-run (e.g. a sign-in or console tab). Poll the newest tab's URL and, when it + * changes, push its live-view URL so a watched run follows the active tab instead + * of showing the stale initial one. Best-effort: never lets live-view following + * disturb the run. Returns a stop function. + */ +function startLiveViewFollower({ + stagehand, + sessions, + sessionId, + onLiveView, +}: { + stagehand: Stagehand; + sessions: BrowserbaseSessionService; + sessionId: string; + onLiveView: (url: string) => void; +}): () => void { + let lastUrl = ''; + let stopped = false; + let inFlight = false; + + const tick = async () => { + if (stopped || inFlight) return; + inFlight = true; + try { + const pages = stagehand.context?.pages?.() ?? []; + const url = pages.at(-1)?.url?.() ?? ''; + if (url && url !== lastUrl) { + lastUrl = url; + const liveUrl = await sessions.getActivePageLiveViewUrl(sessionId, url); + if (liveUrl && !stopped) onLiveView(liveUrl); + } + } catch { + // Best-effort — following the tab must never affect the run. + } finally { + inFlight = false; + } + }; + + const interval = setInterval(() => void tick(), 2500); + return () => { + stopped = true; + clearInterval(interval); + }; +} + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export interface BrowserEvidenceLog { @@ -29,6 +136,8 @@ export interface BrowserEvidenceLog { export interface BrowserEvidenceExecutionResult { success: boolean; screenshot?: string; + /** A close-up: the agent's final viewport (where it left the evidence). */ + focusScreenshot?: string; finalUrl?: string; evaluationStatus?: 'pass' | 'fail'; evaluationReason?: string; @@ -44,14 +153,28 @@ export async function executeBrowserEvidence({ input, sessions, logger, + vault, + onLog, + onLiveView, }: { input: BrowserEvidenceSessionInput; sessions: BrowserbaseSessionService; logger: Logger; + vault: BrowserCredentialVaultAdapter; + /** Called as each stage begins, so a live test run can stream its progress. */ + onLog?: (log: BrowserEvidenceLog) => void; + /** Called with the active tab's live-view URL so a watched run follows tabs. */ + onLiveView?: (url: string) => void; }): Promise { const logs: BrowserEvidenceLog[] = []; const log = (stage: string, message: string) => { - logs.push({ timestamp: new Date().toISOString(), stage, message }); + const entry: BrowserEvidenceLog = { + timestamp: new Date().toISOString(), + stage, + message, + }; + logs.push(entry); + onLog?.(entry); }; let stagehand: Stagehand | null = null; let currentStage: BrowserAutomationFailureStage = 'session'; @@ -59,6 +182,9 @@ export async function executeBrowserEvidence({ try { log('session', 'Initializing Stagehand session.'); stagehand = await sessions.createStagehand(input.sessionId); + // Stable non-null handle for use inside async closures below, where the + // `let stagehand` binding would otherwise widen back to `Stagehand | null`. + const activeStagehand = stagehand; const initialPage = await sessions.ensureActivePage(stagehand); let page = initialPage; @@ -70,30 +196,95 @@ export async function executeBrowserEvidence({ }); await delay(1000); + // Point a watched run's live view at the current tab up front, so it's + // correct before the agent starts opening/switching tabs. + if (onLiveView) { + const initialLiveView = await sessions + .getActivePageLiveViewUrl(input.sessionId, page.url()) + .catch(() => null); + if (initialLiveView) onLiveView(initialLiveView); + } + currentStage = 'auth'; const authCheck = await checkAuth(stagehand); if (!authCheck.isLoggedIn) { - const classified = classifyBrowserAutomationError( - new Error('Session expired. User is not logged in.'), + log( 'auth', + 'Not signed in on this page — signing in with stored credentials.', ); - log('auth', classified.userFacing); - return toExecutionFailure({ classified, logs }); + const relogin = await reloginWithStoredCredentials({ + stagehand: activeStagehand, + sessions, + vault, + input, + verifyLoggedIn: async () => + (await checkAuth(activeStagehand)).isLoggedIn, + log: (message) => log('auth', message), + }); + page = relogin.page ?? page; + + if (!relogin.isLoggedIn) { + // Classify our own known outcome directly rather than relying on string + // matching: auto sign-in couldn't establish a session, so the profile + // needs a human to reconnect (e.g. SMS/email/push/SSO login). + const classified: ClassifiedBrowserAutomationError = { + code: 'needs_reauth', + stage: 'auth', + userFacing: + relogin.reason ?? + 'Authentication is no longer valid. Reconnect this browser profile.', + needsReauth: true, + blockedReason: 'Automated sign-in could not establish a session.', + }; + log('auth', classified.userFacing); + return toExecutionFailure({ classified, logs }); + } + log('auth', 'Re-authenticated with stored credentials.'); } currentStage = 'action'; log('action', 'Running navigation instruction.'); - const instruction = `${input.instruction}. After completing all navigation steps, stop and wait.`; - await stagehand - .agent({ - cua: true, - model: { - modelName: STAGEHAND_CUA_MODEL, - apiKey: process.env.ANTHROPIC_API_KEY, - }, - }) - .execute({ instruction, maxSteps: 20 }); + // Find its own way (no exact directions needed), self-correct a wrong turn, + // and read what's already on screen instead of over-navigating. + const instruction = `${input.instruction}. Work out the path yourself — you don't need exact directions. If the instruction names a specific item or page, open it so the final screenshot clearly shows just that item, not a long list of many. Before finishing, verify the page matches what was asked and correct it if you opened the wrong thing. Don't take unnecessary detours. When the right thing is clearly shown, stop and wait.`; + const primaryModel = resolveCuaModel(logger); + // Follow the agent across tabs while it works (test/watched runs only). + const stopFollowing = onLiveView + ? startLiveViewFollower({ + stagehand: activeStagehand, + sessions, + sessionId: input.sessionId, + onLiveView, + }) + : () => {}; + try { + try { + await runCuaNavigation({ + stagehand: activeStagehand, + instruction, + model: primaryModel, + }); + } catch (navError) { + // The navigation model can be unavailable at runtime (preview access, rate + // limits, upstream errors). Fall back to Claude once — unless we were + // already on it — rather than failing the whole run. + if (primaryModel.modelName === FALLBACK_CUA_MODEL) throw navError; + logger.warn( + `Navigation with ${primaryModel.modelName} failed; retrying with ${FALLBACK_CUA_MODEL}. ${ + navError instanceof Error ? navError.message : String(navError) + }`, + ); + log('action', 'Primary navigation model unavailable — retrying with a backup model.'); + await runCuaNavigation({ + stagehand: activeStagehand, + instruction, + model: claudeFallbackModel(), + }); + } + } finally { + stopFollowing(); + } await delay(2000); page = await resolveEvidencePage({ @@ -105,7 +296,18 @@ export async function executeBrowserEvidence({ currentStage = 'screenshot'; log('screenshot', 'Capturing screenshot.'); + // Full page, not the viewport: a short page would otherwise leave a tall + // band of empty viewport below the content (a wide gap above our overlay), + // and the full page is more complete evidence. const rawScreenshot = await page.screenshot({ + type: 'jpeg', + quality: 80, + fullPage: true, + }); + // The current viewport — the agent stops with the evidence on screen, so this + // is a naturally focused close-up (one screen, no scroll noise) to sit + // alongside the full-page shot as evidence. + const rawFocus = await page.screenshot({ type: 'jpeg', quality: 80, fullPage: false, @@ -118,11 +320,33 @@ export async function executeBrowserEvidence({ instruction: input.instruction, finalUrl, }); + // Only keep the close-up when it actually differs — i.e. the page is taller + // than one screen. If the page fits in the viewport, the two shots are the + // same, so we'd just show a duplicate. + let focusScreenshot: string | undefined; + try { + const [fullMeta, focusMeta] = await Promise.all([ + sharp(rawScreenshot).metadata(), + sharp(rawFocus).metadata(), + ]); + if ((fullMeta.height ?? 0) > (focusMeta.height ?? 0) * 1.15) { + focusScreenshot = await renderScreenshot({ + logger, + logs, + rawScreenshot: rawFocus, + instruction: input.instruction, + finalUrl, + }); + } + } catch { + // If we can't measure, skip the close-up rather than risk a duplicate. + } currentStage = 'evaluation'; await bringEvidencePageToFront(page); const evaluation = await evaluateIfNeeded({ stagehand, criteria: input.evaluationCriteria, + instruction: input.instruction, logs, }); @@ -130,6 +354,7 @@ export async function executeBrowserEvidence({ return { success: false, screenshot, + focusScreenshot, finalUrl, evaluationStatus: evaluation.evaluationStatus, evaluationReason: evaluation.evaluationReason, @@ -143,6 +368,7 @@ export async function executeBrowserEvidence({ return { success: true, screenshot, + focusScreenshot, finalUrl, evaluationStatus: evaluation.evaluationStatus, evaluationReason: evaluation.evaluationReason, @@ -199,8 +425,12 @@ async function checkAuth( stagehand: Stagehand, ): Promise<{ isLoggedIn: boolean }> { const loginSchema = z.object({ isLoggedIn: z.boolean() }); + // Detect the sign-in state by the presence of a login prompt rather than by + // an avatar/profile menu: many apps have no obvious "logged-in" marker, so + // requiring one produced false negatives (and needless re-logins). Treat the + // page as logged in unless a sign-in prompt is clearly the ask. return stagehand.extract( - 'Check if the user is logged in to this website. Look for a user avatar, profile menu, account dropdown, or login/sign-in buttons. Return true if logged in, false if you see login buttons or a login form.', + 'Is this page asking the user to sign in? Look for a visible login prompt: a password field, username/email + password fields, an SSO/"Continue with" screen, or a page whose main call to action is Sign in / Log in. Return isLoggedIn=false ONLY if such a sign-in prompt is clearly present. For any ordinary application or content page with no sign-in prompt, return isLoggedIn=true.', loginSchema, ); } diff --git a/apps/api/src/browserbase/browser-evidence-page.spec.ts b/apps/api/src/browserbase/browser-evidence-page.spec.ts index df155bfeaa..4692ae1322 100644 --- a/apps/api/src/browserbase/browser-evidence-page.spec.ts +++ b/apps/api/src/browserbase/browser-evidence-page.spec.ts @@ -43,6 +43,41 @@ describe('selectEvidencePage', () => { expect(selected).toBe(newTab); }); + it('follows a new tab the agent opened on a different host (AWS console vs sign-in)', () => { + const initialPage = page({ id: 'home', url: 'https://aws.amazon.com/' }); + const iamTab = page({ + id: 'iam', + url: 'https://console.aws.amazon.com/iam/home#/users', + }); + + const selected = selectEvidencePage({ + pages: [initialPage, iamTab], + initialPage, + // Entered host matches neither tab — the screenshot must still land on the + // page the agent navigated to, not the stale homepage. + targetUrl: 'https://us-east-2.signin.aws.amazon.com/', + isClosed, + }); + + expect(selected).toBe(iamTab); + }); + + it('keeps the initial page when the agent navigated within it (no new tab)', () => { + const initialPage = page({ + id: 'initial', + url: 'https://console.aws.amazon.com/iam/home', + }); + + const selected = selectEvidencePage({ + pages: [initialPage], + initialPage, + targetUrl: 'https://us-east-2.signin.aws.amazon.com/', + isClosed, + }); + + expect(selected).toBe(initialPage); + }); + it('returns null when every page is closed', () => { const initialPage = page({ closed: true, diff --git a/apps/api/src/browserbase/browser-evidence-page.ts b/apps/api/src/browserbase/browser-evidence-page.ts index ee12e64d9f..28869f14d8 100644 --- a/apps/api/src/browserbase/browser-evidence-page.ts +++ b/apps/api/src/browserbase/browser-evidence-page.ts @@ -50,15 +50,22 @@ export function selectEvidencePage({ const openMatchesTarget = (page: Page) => targetHostname !== null && safeHostname(page.url()) === targetHostname; - const newestMatchingNewPage = [...openPages] - .reverse() - .find((page) => page !== initialPage && openMatchesTarget(page)); + const reversed = [...openPages].reverse(); + + // A new tab that matches the entered host (a popup straight to the target). + const newestMatchingNewPage = reversed.find( + (page) => page !== initialPage && openMatchesTarget(page), + ); if (newestMatchingNewPage) return newestMatchingNewPage; - if (!isClosed(initialPage)) return initialPage; + // The instruction often navigates to a DIFFERENT host than the entered URL + // (e.g. AWS console vs the sign-in host), commonly in a new tab. Prefer the + // newest non-initial page over the (now stale) initial one, so the screenshot + // matches where the agent actually ended up — not the starting/homepage tab. + const newestNonInitial = reversed.find((page) => page !== initialPage); + if (newestNonInitial) return newestNonInitial; - const newestMatchingPage = [...openPages].reverse().find(openMatchesTarget); - if (newestMatchingPage) return newestMatchingPage; + if (!isClosed(initialPage)) return initialPage; return openPages.at(-1) ?? null; } diff --git a/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts b/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts index 1f615889d4..699476c5ef 100644 --- a/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts +++ b/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts @@ -98,4 +98,45 @@ describe('BrowserEvidenceRunnerService', () => { const call = jest.mocked(executeBrowserEvidence).mock.calls[0]; expect(call?.[0].input).not.toHaveProperty('credentialMaterial'); }); + + it('signals the imminent teardown before closing a fresh session', async () => { + const sessions = new BrowserbaseSessionService(); + jest.spyOn(sessions, 'createSessionWithContext').mockResolvedValue({ + sessionId: 'sess_new', + liveViewUrl: 'https://live.example/sess_new', + }); + const closeSpy = jest + .spyOn(sessions, 'closeSession') + .mockResolvedValue(undefined); + + jest.mocked(executeBrowserEvidence).mockResolvedValue({ + success: true, + finalUrl: 'https://example.com/final', + logs: [], + }); + + const service = new BrowserEvidenceRunnerService( + sessions, + new BrowserbaseScreenshotService(), + ); + + const onSessionClosing = jest.fn(); + await service.runEvidence({ + organizationId: 'org_1', + automationId: 'bau_1', + runId: 'bar_1', + targetUrl: 'https://example.com', + instruction: 'collect evidence', + profile: { id: 'bap_1', hostname: 'example.com', contextId: 'ctx_1' }, + onSessionClosing, + }); + + expect(onSessionClosing).toHaveBeenCalledTimes(1); + expect(closeSpy).toHaveBeenCalledWith('sess_new'); + // Must fire BEFORE the close, so the UI can cover the iframe before the live + // session's socket drops (the whole point of the transition overlay). + expect(onSessionClosing.mock.invocationCallOrder[0]).toBeLessThan( + closeSpy.mock.invocationCallOrder[0], + ); + }); }); diff --git a/apps/api/src/browserbase/browser-evidence-runner.service.ts b/apps/api/src/browserbase/browser-evidence-runner.service.ts index 0e07b19582..ff317e29cc 100644 --- a/apps/api/src/browserbase/browser-evidence-runner.service.ts +++ b/apps/api/src/browserbase/browser-evidence-runner.service.ts @@ -1,7 +1,12 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger } from '@nestjs/common'; import { Prisma } from '@db'; import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; +import { + BROWSER_CREDENTIAL_VAULT_ADAPTER, + type BrowserCredentialVaultAdapter, +} from './credential-vault'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; import { type BrowserAutomationFailureCode, type BrowserAutomationFailureStage, @@ -30,6 +35,17 @@ export interface BrowserEvidenceRunnerInput { vaultConnectionId?: string | null; }; beforeExecution?: () => Promise; + /** Live per-stage progress callback (used to stream a test run's activity). */ + onLog?: (log: BrowserEvidenceLog) => void; + /** Fired as the agent switches tabs, so a watched run's live view follows it. */ + onLiveView?: (url: string) => void; + /** Fired once this step's live session opens, so the Run view can follow it. */ + onSession?: (info: { sessionId: string; liveViewUrl: string }) => void; + /** + * Fired right before this step's session is torn down, so the Run view can + * cover the (about-to-disconnect) live iframe with a transition state. + */ + onSessionClosing?: () => void; } export interface BrowserEvidenceSessionInput extends BrowserEvidenceRunnerInput { @@ -41,6 +57,9 @@ export interface BrowserEvidenceRunResult { status: 'completed' | 'failed' | 'blocked'; screenshotKey?: string; screenshotUrl?: string; + /** A focused close-up (the agent's final viewport) shown beside the full page. */ + focusScreenshotKey?: string; + focusScreenshotUrl?: string; finalUrl?: string; evaluationStatus?: 'pass' | 'fail'; evaluationReason?: string; @@ -68,6 +87,8 @@ export class BrowserEvidenceRunnerService { constructor( private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), private readonly screenshots: BrowserbaseScreenshotService = new BrowserbaseScreenshotService(), + @Inject(BROWSER_CREDENTIAL_VAULT_ADAPTER) + private readonly vault: BrowserCredentialVaultAdapter = resolveBrowserCredentialVaultAdapter(), ) {} async runEvidence( @@ -77,9 +98,10 @@ export class BrowserEvidenceRunnerService { profileId: input.profile.id, hostname: input.profile.hostname, run: async () => { - const { sessionId } = await this.sessions.createSessionWithContext( - input.profile.contextId, - ); + const { sessionId, liveViewUrl } = + await this.sessions.createSessionWithContext(input.profile.contextId); + // Surface this step's live view so a watched run can follow each vendor. + input.onSession?.({ sessionId, liveViewUrl }); try { return await this.executeEvidenceOnSessionUnlocked({ @@ -87,6 +109,9 @@ export class BrowserEvidenceRunnerService { sessionId, }); } finally { + // Signal the imminent teardown before we actually close, so the UI can + // cover the live view before Browserbase's iframe shows "disconnected". + input.onSessionClosing?.(); await this.closeSession(sessionId); } }, @@ -112,9 +137,16 @@ export class BrowserEvidenceRunnerService { input, sessions: this.sessions, logger: this.logger, + vault: this.vault, + onLog: input.onLog, + onLiveView: input.onLiveView, }); - let uploaded: { screenshotKey: string; screenshotUrl: string } | null = - null; + let uploaded: { + screenshotKey?: string; + screenshotUrl?: string; + focusScreenshotKey?: string; + focusScreenshotUrl?: string; + } | null = null; try { uploaded = await this.uploadCapturedScreenshot({ input, execution }); } catch (err) { @@ -138,6 +170,8 @@ export class BrowserEvidenceRunnerService { status: this.blockedStatusForCode(execution.failureCode), screenshotKey: uploaded?.screenshotKey, screenshotUrl: uploaded?.screenshotUrl, + focusScreenshotKey: uploaded?.focusScreenshotKey, + focusScreenshotUrl: uploaded?.focusScreenshotUrl, finalUrl: execution.finalUrl, evaluationStatus: execution.evaluationStatus, evaluationReason: execution.evaluationReason, @@ -155,6 +189,8 @@ export class BrowserEvidenceRunnerService { status: 'completed', screenshotKey: uploaded?.screenshotKey, screenshotUrl: uploaded?.screenshotUrl, + focusScreenshotKey: uploaded?.focusScreenshotKey, + focusScreenshotUrl: uploaded?.focusScreenshotUrl, finalUrl: execution.finalUrl, evaluationStatus: execution.evaluationStatus, evaluationReason: execution.evaluationReason, @@ -162,26 +198,50 @@ export class BrowserEvidenceRunnerService { }; } + private async uploadOne( + input: BrowserEvidenceRunnerInput, + base64: string, + variant?: string, + ): Promise<{ key: string; url: string }> { + // A variant keys to a distinct object (…/runId-focus.jpg) so it doesn't + // overwrite the full-page shot. + const key = await this.screenshots.uploadScreenshot({ + organizationId: input.organizationId, + automationId: input.automationId, + runId: variant ? `${input.runId}-${variant}` : input.runId, + base64Screenshot: base64, + }); + const url = await this.screenshots.getPresignedUrl({ key }); + return { key, url }; + } + private async uploadCapturedScreenshot({ input, execution, }: { input: BrowserEvidenceRunnerInput; execution: BrowserEvidenceExecutionResult; - }): Promise<{ screenshotKey: string; screenshotUrl: string } | null> { + }): Promise<{ + screenshotKey?: string; + screenshotUrl?: string; + focusScreenshotKey?: string; + focusScreenshotUrl?: string; + } | null> { if (!execution.screenshot) return null; - const screenshotKey = await this.screenshots.uploadScreenshot({ - organizationId: input.organizationId, - automationId: input.automationId, - runId: input.runId, - base64Screenshot: execution.screenshot, - }); - const screenshotUrl = await this.screenshots.getPresignedUrl({ - key: screenshotKey, - }); + const [full, focus] = await Promise.all([ + this.uploadOne(input, execution.screenshot), + execution.focusScreenshot + ? this.uploadOne(input, execution.focusScreenshot, 'focus') + : Promise.resolve(null), + ]); - return { screenshotKey, screenshotUrl }; + return { + screenshotKey: full.key, + screenshotUrl: full.url, + focusScreenshotKey: focus?.key, + focusScreenshotUrl: focus?.url, + }; } private async closeSession(sessionId: string): Promise { diff --git a/apps/api/src/browserbase/browser-evidence-step-timeline.ts b/apps/api/src/browserbase/browser-evidence-step-timeline.ts new file mode 100644 index 0000000000..a33415807c --- /dev/null +++ b/apps/api/src/browserbase/browser-evidence-step-timeline.ts @@ -0,0 +1,60 @@ +/** + * A live activity timeline for an evidence run — the same "steps" the composer's + * Test panel and the connect/sign-in flow show. Each engine log advances the + * timeline (the prior active step becomes done, a new active step is appended), + * and the whole snapshot is emitted so a Trigger.dev task can publish it to + * realtime metadata for the UI to render. + */ +export interface EvidenceTimelineStep { + /** Step label. */ + l: string; + /** Clock timestamp, e.g. "06:02:14". */ + t: string; + state: 'done' | 'active' | 'pending' | 'warn' | 'fail'; +} + +/** + * What the Run view's live iframe should reflect. Between vendors and at the end + * of a run a browser session is torn down, and the (now-dead) Browserbase live + * view briefly shows its own "disconnected" notice. We stream this phase so the + * UI can cover the iframe with a calm transition state instead: + * • running — a live session is up; show the iframe + * • switching — the current vendor is done; the next vendor's session is opening + * • finishing — the last vendor is done; the run is wrapping up + */ +export type BrowserRunLivePhase = 'running' | 'switching' | 'finishing'; + +const clock = () => + new Date().toLocaleTimeString('en-US', { + hour12: false, + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + +export interface EvidenceTimeline { + /** Advance the timeline with a new active step (marks the prior one done). */ + step: (label: string) => void; + /** Set the final state of the last step (done/warn/fail). */ + finish: (state: EvidenceTimelineStep['state']) => void; +} + +export function createEvidenceTimeline( + onSteps?: (steps: EvidenceTimelineStep[]) => void, +): EvidenceTimeline { + const steps: EvidenceTimelineStep[] = []; + const emit = () => onSteps?.(steps.map((s) => ({ ...s }))); + return { + step(label: string) { + const last = steps[steps.length - 1]; + if (last?.state === 'active') last.state = 'done'; + steps.push({ l: label, t: clock(), state: 'active' }); + emit(); + }, + finish(state: EvidenceTimelineStep['state']) { + const last = steps[steps.length - 1]; + if (last) last.state = state; + emit(); + }, + }; +} diff --git a/apps/api/src/browserbase/browser-instruction-test.service.spec.ts b/apps/api/src/browserbase/browser-instruction-test.service.spec.ts new file mode 100644 index 0000000000..079411b429 --- /dev/null +++ b/apps/api/src/browserbase/browser-instruction-test.service.spec.ts @@ -0,0 +1,160 @@ +import { BrowserInstructionTestService } from './browser-instruction-test.service'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { BrowserAuthProfileService } from './browser-auth-profile.service'; +import type { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import type { + BrowserEvidenceRunResult, + BrowserEvidenceSessionInput, +} from './browser-evidence-runner.service'; + +// The service imports the profile/runner services which pull in @db; mock it so +// the real Prisma client isn't instantiated at import time. +jest.mock('@db', () => ({ db: {} })); + +const profile = { + id: 'prof_1', + hostname: 'app.example.com', + contextId: 'ctx_1', + vaultProvider: '1password', + vaultExternalItemRef: 'op://vault/item', + vaultConnectionId: 'vault', +}; + +function makeProfiles() { + return { + resolveProfileForTarget: jest.fn().mockResolvedValue(profile), + }; +} + +/** A runner whose `executeEvidenceOnSession` streams a couple of stages, then returns `result`. */ +function makeRunner( + result: Partial & { success: boolean; status: BrowserEvidenceRunResult['status'] }, +) { + return { + executeEvidenceOnSession: jest.fn( + async (input: BrowserEvidenceSessionInput) => { + input.onLog?.({ timestamp: 't1', stage: 'navigation', message: 'Opening the page' }); + input.onLog?.({ timestamp: 't2', stage: 'action', message: 'Running instruction' }); + return { logs: [], ...result } as BrowserEvidenceRunResult; + }, + ), + }; +} + +const baseInput = { + organizationId: 'org_1', + taskId: 'task_1', + profileId: 'prof_1', + targetUrl: 'https://app.example.com/settings', + instruction: 'Screenshot the MFA policy', + evaluationCriteria: 'MFA is enforced', + sessionId: 'sess_1', +}; + +function build(runner: ReturnType, profiles = makeProfiles()) { + const service = new BrowserInstructionTestService( + {} as unknown as BrowserbaseSessionService, + profiles as unknown as BrowserAuthProfileService, + runner as unknown as BrowserEvidenceRunnerService, + ); + return { service, profiles }; +} + +describe('BrowserInstructionTestService', () => { + it('runs the ad-hoc instruction through the evidence runner with synthetic ids', async () => { + const runner = makeRunner({ success: true, status: 'completed', screenshotUrl: 'https://s3/x.png' }); + const { service, profiles } = build(runner); + + await service.testInstructionOnSession(baseInput); + + expect(profiles.resolveProfileForTarget).toHaveBeenCalledWith({ + organizationId: 'org_1', + targetUrl: 'https://app.example.com/settings', + profileId: 'prof_1', + }); + const call = runner.executeEvidenceOnSession.mock.calls[0][0]; + expect(call.instruction).toBe('Screenshot the MFA policy'); + expect(call.evaluationCriteria).toBe('MFA is enforced'); + expect(call.sessionId).toBe('sess_1'); + expect(call.automationId).toContain('test-'); + expect(call.runId).toContain('test-'); + expect(call.profile.contextId).toBe('ctx_1'); + }); + + it('streams accumulating steps and marks the last one done on success', async () => { + const runner = makeRunner({ success: true, status: 'completed', screenshotUrl: 'https://s3/x.png' }); + const { service } = build(runner); + const frames: { l: string; state: string }[][] = []; + + const result = await service.testInstructionOnSession({ + ...baseInput, + onSteps: (steps) => frames.push(steps.map((s) => ({ l: s.l, state: s.state }))), + }); + + // First stage active, second stage flips the first to done and appends active. + expect(frames[0]).toEqual([{ l: 'Opening the page', state: 'active' }]); + expect(frames[1]).toEqual([ + { l: 'Opening the page', state: 'done' }, + { l: 'Running instruction', state: 'active' }, + ]); + // Final frame: last step resolved to done. + const last = frames[frames.length - 1]; + expect(last[last.length - 1]).toEqual({ l: 'Running instruction', state: 'done' }); + expect(result.success).toBe(true); + expect(result.screenshotUrl).toBe('https://s3/x.png'); + }); + + it('maps the runner verdict into the test result', async () => { + const runner = makeRunner({ + success: true, + status: 'completed', + evaluationStatus: 'fail', + evaluationReason: 'MFA optional', + screenshotUrl: 'https://s3/x.png', + }); + const { service } = build(runner); + + const result = await service.testInstructionOnSession(baseInput); + + expect(result.evaluationStatus).toBe('fail'); + expect(result.evaluationReason).toBe('MFA optional'); + }); + + it('marks the last step failed when the run fails', async () => { + const runner = makeRunner({ + success: false, + status: 'failed', + error: 'stuck', + failureCode: 'timeout', + }); + const { service } = build(runner); + let final: { state: string }[] = []; + + const result = await service.testInstructionOnSession({ + ...baseInput, + onSteps: (steps) => (final = steps.map((s) => ({ state: s.state }))), + }); + + expect(final[final.length - 1].state).toBe('fail'); + expect(result.success).toBe(false); + expect(result.error).toBe('stuck'); + }); + + it('marks the last step as a warning when the connection needs reauth', async () => { + const runner = makeRunner({ + success: false, + status: 'blocked', + needsReauth: true, + error: 'session expired', + }); + const { service } = build(runner); + let final: { state: string }[] = []; + + await service.testInstructionOnSession({ + ...baseInput, + onSteps: (steps) => (final = steps.map((s) => ({ state: s.state }))), + }); + + expect(final[final.length - 1].state).toBe('warn'); + }); +}); diff --git a/apps/api/src/browserbase/browser-instruction-test.service.ts b/apps/api/src/browserbase/browser-instruction-test.service.ts new file mode 100644 index 0000000000..78abe7b5fa --- /dev/null +++ b/apps/api/src/browserbase/browser-instruction-test.service.ts @@ -0,0 +1,116 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import type { BrowserEvidenceLog } from './browser-evidence-execution'; +import { + createEvidenceTimeline, + type EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; + +/** A single line in the live test-run activity timeline (mirrors the sign-in flow). */ +export type InstructionTestStep = EvidenceTimelineStep; + +export interface InstructionTestResult { + success: boolean; + screenshotUrl?: string; + /** A focused close-up (the agent's final viewport) shown beside the full page. */ + focusScreenshotUrl?: string; + finalUrl?: string; + evaluationStatus?: 'pass' | 'fail'; + evaluationReason?: string; + error?: string; + needsReauth?: boolean; + failureCode?: string; + blockedReason?: string; +} + +/** + * Runs an instruction the user hasn't saved yet against the connection's live + * session, so they can watch it work before committing it to the schedule. + * + * It reuses the exact evidence runner scheduled runs use, so a passing test + * means the saved instruction will behave the same unattended. Nothing is + * persisted — no automation, no run record — this only proves the instruction + * out. The caller owns the session (it shows the live view); we never close it. + */ +@Injectable() +export class BrowserInstructionTestService { + private readonly logger = new Logger(BrowserInstructionTestService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + sessions, + ), + private readonly runner: BrowserEvidenceRunnerService = new BrowserEvidenceRunnerService( + sessions, + ), + ) {} + + async testInstructionOnSession(input: { + organizationId: string; + taskId?: string; + profileId?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + sessionId: string; + /** Live activity timeline, surfaced to the composer's test panel. */ + onSteps?: (steps: InstructionTestStep[]) => void; + /** Active-tab live-view URL, so the test panel follows the agent across tabs. */ + onLiveView?: (url: string) => void; + }): Promise { + const timeline = createEvidenceTimeline(input.onSteps); + + const profile = await this.profiles.resolveProfileForTarget({ + organizationId: input.organizationId, + targetUrl: input.targetUrl, + profileId: input.profileId, + }); + + const result = await this.runner.executeEvidenceOnSession({ + organizationId: input.organizationId, + taskId: input.taskId, + // Synthetic ids: nothing is persisted, but the runner uses them for the + // screenshot key path and its own logging. + automationId: `test-${profile.id}`, + runId: `test-${input.sessionId}`, + sessionId: input.sessionId, + targetUrl: input.targetUrl, + instruction: input.instruction, + evaluationCriteria: input.evaluationCriteria, + profile: { + id: profile.id, + hostname: profile.hostname, + contextId: profile.contextId, + vaultProvider: profile.vaultProvider, + vaultExternalItemRef: profile.vaultExternalItemRef, + vaultConnectionId: profile.vaultConnectionId, + }, + onLog: (entry: BrowserEvidenceLog) => timeline.step(entry.message), + onLiveView: input.onLiveView, + }); + + timeline.finish( + result.success + ? 'done' + : result.status === 'blocked' || result.needsReauth + ? 'warn' + : 'fail', + ); + + return { + success: result.success, + screenshotUrl: result.screenshotUrl, + focusScreenshotUrl: result.focusScreenshotUrl, + finalUrl: result.finalUrl, + evaluationStatus: result.evaluationStatus, + evaluationReason: result.evaluationReason, + error: result.error, + needsReauth: result.needsReauth, + failureCode: result.failureCode, + blockedReason: result.blockedReason, + }; + } +} diff --git a/apps/api/src/browserbase/browser-login-analysis.spec.ts b/apps/api/src/browserbase/browser-login-analysis.spec.ts new file mode 100644 index 0000000000..ff31f7c4a1 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analysis.spec.ts @@ -0,0 +1,81 @@ +import { + analyzeDetectedLogin, + manualLoginAnalysis, + type LoginDetection, +} from './browser-login-analysis'; + +const base: LoginDetection = { + reachable: true, + hasPasswordField: false, + identifierType: 'unknown', + ssoProviders: [], + hasPasskey: false, + extraFields: [], +}; + +describe('analyzeDetectedLogin', () => { + it('recommends "ready" when a password field is present', () => { + const result = analyzeDetectedLogin({ + ...base, + hasPasswordField: true, + identifierType: 'email', + }); + expect(result.recommendation.category).toBe('ready'); + expect(result.detectedMethods).toContain('password'); + }); + + it('recommends check-ins for SSO-only sites', () => { + const result = analyzeDetectedLogin({ ...base, ssoProviders: ['Google'] }); + expect(result.recommendation.category).toBe('works_with_checkins'); + expect(result.detectedMethods).toContain('sso'); + }); + + it('recommends check-ins for passkey-only sites', () => { + const result = analyzeDetectedLogin({ ...base, hasPasskey: true }); + expect(result.recommendation.category).toBe('works_with_checkins'); + expect(result.detectedMethods).toContain('passkey'); + }); + + it('prefers the password path (ready) when both password and SSO exist', () => { + const result = analyzeDetectedLogin({ + ...base, + hasPasswordField: true, + ssoProviders: ['Google'], + }); + expect(result.recommendation.category).toBe('ready'); + expect(result.detectedMethods).toEqual( + expect.arrayContaining(['password', 'sso']), + ); + }); + + it('falls back to manual when nothing is detected', () => { + expect(analyzeDetectedLogin(base).recommendation.category).toBe('manual'); + }); + + it('falls back to manual when the page is unreachable, even with a password field', () => { + const result = analyzeDetectedLogin({ + ...base, + reachable: false, + hasPasswordField: true, + }); + expect(result.recommendation.category).toBe('manual'); + }); + + it('passes extra fields through unchanged', () => { + const result = analyzeDetectedLogin({ + ...base, + hasPasswordField: true, + extraFields: [{ label: 'Workspace URL' }], + }); + expect(result.extraFields).toEqual([{ label: 'Workspace URL' }]); + }); +}); + +describe('manualLoginAnalysis', () => { + it('is an unreachable, manual recommendation', () => { + const result = manualLoginAnalysis(); + expect(result.reachable).toBe(false); + expect(result.recommendation.category).toBe('manual'); + expect(result.detectedMethods).toEqual([]); + }); +}); diff --git a/apps/api/src/browserbase/browser-login-analysis.ts b/apps/api/src/browserbase/browser-login-analysis.ts new file mode 100644 index 0000000000..fb8039b569 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analysis.ts @@ -0,0 +1,115 @@ +import { z } from 'zod'; + +/** + * What we can reliably read from a vendor's *first* sign-in page. Note: the + * specific 2FA method (authenticator vs SMS vs push) usually isn't visible until + * after the identifier/password step, so it is resolved during live sign-in — not + * here. This detection is best-effort and always degrades to a manual fallback. + */ +export const loginDetectionSchema = z.object({ + reachable: z + .boolean() + .describe('Whether this looks like a real sign-in page we could read'), + hasPasswordField: z + .boolean() + .describe('Whether a password input is present on the page'), + identifierType: z + .enum(['email', 'username', 'either', 'unknown']) + .describe('What the first login field accepts'), + ssoProviders: z + .array(z.string()) + .describe( + 'Third-party sign-in buttons offered, e.g. Google, Microsoft, SSO', + ), + hasPasskey: z + .boolean() + .describe('Whether passkey / security-key sign-in is offered'), + extraFields: z + .array(z.object({ label: z.string() })) + .describe( + 'Other required fields before the password, e.g. company, workspace, subdomain', + ), +}); + +export type LoginDetection = z.infer; + +export type LoginMethod = 'password' | 'sso' | 'passkey'; + +export type LoginRecommendationCategory = + 'ready' | 'works_with_checkins' | 'manual'; + +export interface LoginAnalysis { + reachable: boolean; + detectedMethods: LoginMethod[]; + identifierType: LoginDetection['identifierType']; + extraFields: { label: string }[]; + recommendation: { + category: LoginRecommendationCategory; + headline: string; + detail: string; + }; +} + +const READY = { + category: 'ready' as const, + headline: "You're set.", + detail: + 'Sign in once — we capture what the scheduler needs to sign in on its own from then on.', +}; + +const CHECKINS = { + category: 'works_with_checkins' as const, + headline: 'This will work — with an occasional check-in.', + detail: + 'Runs reuse your session; when it expires we email you to sign in again. Runs pause, never fail silently.', +}; + +const MANUAL = { + category: 'manual' as const, + headline: "We couldn't detect a sign-in form here.", + detail: + "Make sure the URL is your login page (not the homepage) — or just sign in below and we'll take it from there.", +}; + +/** + * Turns raw page detection into a recommendation for the connect flow. Pure and + * deterministic so it can be unit-tested without a browser. + */ +export function analyzeDetectedLogin(detection: LoginDetection): LoginAnalysis { + const detectedMethods: LoginMethod[] = []; + if (detection.hasPasswordField) detectedMethods.push('password'); + if (detection.ssoProviders.length > 0) detectedMethods.push('sso'); + if (detection.hasPasskey) detectedMethods.push('passkey'); + + let recommendation: LoginAnalysis['recommendation'] = MANUAL; + if (detection.reachable && detection.hasPasswordField) { + // A password form means we can capture credentials + an authenticator key + // and run fully unattended (the 2FA specifics surface during sign-in). + recommendation = READY; + } else if ( + detection.reachable && + (detection.ssoProviders.length > 0 || detection.hasPasskey) + ) { + // SSO / passkey can't be replayed unattended — session reuse + human re-auth. + recommendation = CHECKINS; + } + + return { + reachable: detection.reachable, + detectedMethods, + identifierType: detection.identifierType, + extraFields: detection.extraFields, + recommendation, + }; +} + +export function manualLoginAnalysis(): LoginAnalysis { + return analyzeDetectedLogin({ + reachable: false, + hasPasswordField: false, + identifierType: 'unknown', + ssoProviders: [], + hasPasskey: false, + extraFields: [], + }); +} diff --git a/apps/api/src/browserbase/browser-login-analyzer.service.spec.ts b/apps/api/src/browserbase/browser-login-analyzer.service.spec.ts new file mode 100644 index 0000000000..b836ac80af --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analyzer.service.spec.ts @@ -0,0 +1,90 @@ +import { BrowserLoginAnalyzerService } from './browser-login-analyzer.service'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { LoginDetection } from './browser-login-analysis'; + +function makeSessions(extract: jest.Mock) { + const page = { goto: jest.fn().mockResolvedValue(undefined) }; + return { + createBrowserbaseContext: jest.fn().mockResolvedValue('ctx_1'), + createSessionWithContext: jest + .fn() + .mockResolvedValue({ sessionId: 'sess_1', liveViewUrl: '' }), + createStagehand: jest + .fn() + .mockResolvedValue({ + extract, + act: jest.fn().mockResolvedValue(undefined), + }), + ensureActivePage: jest.fn().mockResolvedValue(page), + safeCloseStagehand: jest.fn().mockResolvedValue(undefined), + closeSession: jest.fn().mockResolvedValue(undefined), + }; +} + +const passwordDetection: LoginDetection = { + reachable: true, + hasPasswordField: true, + identifierType: 'email', + ssoProviders: [], + hasPasskey: false, + extraFields: [], +}; + +describe('BrowserLoginAnalyzerService', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + const run = async ( + sessions: ReturnType, + url: string, + ) => { + const service = new BrowserLoginAnalyzerService( + sessions as unknown as BrowserbaseSessionService, + ); + const promise = service.analyzeLogin(url); + await jest.runAllTimersAsync(); + return promise; + }; + + it('returns a ready recommendation and cleans up the session', async () => { + const extract = jest.fn().mockResolvedValue(passwordDetection); + const sessions = makeSessions(extract); + + const result = await run( + sessions, + 'https://app.datadoghq.com/account/login', + ); + + expect(result.recommendation.category).toBe('ready'); + expect(sessions.safeCloseStagehand).toHaveBeenCalledTimes(1); + expect(sessions.closeSession).toHaveBeenCalledWith('sess_1'); + }); + + it('falls back to manual when the page cannot be read, and still cleans up', async () => { + const extract = jest.fn().mockRejectedValue(new Error('page unreadable')); + const sessions = makeSessions(extract); + + const result = await run(sessions, 'https://weird.example.com'); + + expect(result.reachable).toBe(false); + expect(result.recommendation.category).toBe('manual'); + expect(sessions.closeSession).toHaveBeenCalledWith('sess_1'); + }); + + it('falls back to manual (no crash) when Browserbase is unavailable', async () => { + const sessions = makeSessions(jest.fn()); + sessions.createBrowserbaseContext.mockRejectedValue( + new Error('BROWSERBASE_API_KEY is missing'), + ); + + const result = await run( + sessions, + 'https://app.datadoghq.com/account/login', + ); + + expect(result.reachable).toBe(false); + expect(result.recommendation.category).toBe('manual'); + // No session was created, so nothing to close. + expect(sessions.closeSession).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browser-login-analyzer.service.ts b/apps/api/src/browserbase/browser-login-analyzer.service.ts new file mode 100644 index 0000000000..96b9f87f04 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analyzer.service.ts @@ -0,0 +1,81 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { + analyzeDetectedLogin, + loginDetectionSchema, + manualLoginAnalysis, + type LoginAnalysis, +} from './browser-login-analysis'; +import { navigateToSignIn } from './browser-login-navigation'; + +const EXTRACT_PROMPT = + 'Look at this sign-in page. Report: is it a real login page we can read; is a ' + + 'password field present; does the first login field accept an email, a username, ' + + 'or either; which third-party sign-in buttons are offered (e.g. Google, Microsoft, ' + + 'Okta, SSO); is passkey / security-key sign-in offered; and list any other fields ' + + 'required before the password (e.g. company, workspace, subdomain).'; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Opens a vendor's sign-in page in a throwaway cloud browser and detects which + * login methods it supports, so the connect flow can recommend the most reliable + * path. Reads a public page only — no credentials involved. Always degrades to a + * manual fallback if the page can't be read. + */ +@Injectable() +export class BrowserLoginAnalyzerService { + private readonly logger = new Logger(BrowserLoginAnalyzerService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + ) {} + + async analyzeLogin(url: string): Promise { + let sessionId: string | null = null; + let stagehand: Awaited< + ReturnType + > | null = null; + try { + // Session/context creation is inside the try so that an unavailable or + // unconfigured Browserbase (e.g. missing BROWSERBASE_API_KEY) degrades to + // the manual fallback instead of surfacing a 500 to the user. + const contextId = await this.sessions.createBrowserbaseContext(); + const session = await this.sessions.createSessionWithContext(contextId); + sessionId = session.sessionId; + + stagehand = await this.sessions.createStagehand(sessionId); + const page = await this.sessions.ensureActivePage(stagehand); + await page.goto(url, { waitUntil: 'domcontentloaded', timeoutMs: 30000 }); + await delay(1500); + + // Navigate to the actual sign-in page so the customer can paste any URL + // (a homepage, a dashboard) rather than the exact login page. + await navigateToSignIn(stagehand); + + const detection = await stagehand.extract( + EXTRACT_PROMPT, + loginDetectionSchema, + ); + return analyzeDetectedLogin(detection); + } catch (err) { + // A page we can't read (or a Browserbase hiccup) isn't an error the user + // needs to see — fall back to manual entry so the connect flow continues. + // Log the full error + stack so a misconfiguration is diagnosable and not + // silently hidden behind the manual fallback. + this.logger.warn('Login analysis failed; falling back to manual entry', { + url, + error: err instanceof Error ? err.message : String(err), + stack: err instanceof Error ? err.stack : undefined, + }); + return manualLoginAnalysis(); + } finally { + if (stagehand) await this.sessions.safeCloseStagehand(stagehand); + if (sessionId) { + await this.sessions + .closeSession(sessionId) + .catch(() => undefined /* best-effort cleanup */); + } + } + } +} diff --git a/apps/api/src/browserbase/browser-login-navigation.ts b/apps/api/src/browserbase/browser-login-navigation.ts new file mode 100644 index 0000000000..982c18adc5 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-navigation.ts @@ -0,0 +1,50 @@ +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Current URL of the active page — a hint for settling; '' when unavailable. */ +function currentUrl(stagehand: Stagehand): string { + try { + const pages = stagehand.context?.pages?.() ?? []; + return pages[pages.length - 1]?.url() ?? ''; + } catch { + return ''; + } +} + +/** + * Wait until the URL stops changing, so a redirect chain (e.g. AWS bouncing from + * the homepage through signin.aws.amazon.com) finishes before the caller reads or + * fills the page. Bounded so a page that never settles can't hang the run. + */ +async function waitForUrlToSettle(stagehand: Stagehand): Promise { + let previous = currentUrl(stagehand); + for (let i = 0; i < 8; i += 1) { + await delay(700); + const now = currentUrl(stagehand); + if (now === previous) return; + previous = now; + } +} + +/** + * Best-effort navigation to the sign-in form. If the current page is a homepage + * or dashboard, the agent opens the "Sign in" link so the login form is on + * screen. Never throws — callers proceed on whatever page they end up on. Shared + * by login analysis and the automated sign-in so both start from the real form + * rather than, say, a marketing homepage. + */ +export async function navigateToSignIn(stagehand: Stagehand): Promise { + try { + await stagehand.act( + 'If this page is not already a login or sign-in page, find and open the ' + + '"Sign in", "Sign in to Console", or "Log in" link so the sign-in form is ' + + 'visible. If a sign-in form is already shown, do nothing.', + ); + // A sign-in link often kicks off a redirect chain; don't read/fill until it + // settles, or we'd act on the old (e.g. marketing homepage) page. + await waitForUrlToSettle(stagehand); + } catch { + // Ignore — detection / sign-in proceeds on the current page. + } +} diff --git a/apps/api/src/browserbase/browser-mfa-instructions.service.spec.ts b/apps/api/src/browserbase/browser-mfa-instructions.service.spec.ts new file mode 100644 index 0000000000..99bc4fcab9 --- /dev/null +++ b/apps/api/src/browserbase/browser-mfa-instructions.service.spec.ts @@ -0,0 +1,147 @@ +import { generateObject } from 'ai'; +import { BrowserMfaInstructionsService } from './browser-mfa-instructions.service'; + +jest.mock('ai', () => ({ generateObject: jest.fn() })); +jest.mock('@ai-sdk/anthropic', () => ({ anthropic: () => 'mock-model' })); + +const mockGenerate = generateObject as jest.MockedFunction; + +// Minimal shape the service reads from generateObject's result. +const asResult = (object: { steps: string[]; confident: boolean }) => + ({ object }) as unknown as Awaited>; + +const firecrawlOk = (markdown: string) => + ({ + ok: true, + json: async () => ({ + success: true, + data: { web: [{ url: 'https://docs.vendor.com/2fa', title: '2FA', markdown }] }, + }), + }) as unknown as Response; + +describe('BrowserMfaInstructionsService', () => { + let service: BrowserMfaInstructionsService; + let mockFetch: jest.Mock; + const originalKey = process.env.FIRECRAWL_API_KEY; + const originalFetch = global.fetch; + + beforeEach(() => { + jest.clearAllMocks(); + // Ungrounded by default (no web-search call) so the base cases are + // deterministic; grounding tests opt in by setting the key + mocking fetch. + delete process.env.FIRECRAWL_API_KEY; + mockFetch = jest.fn(); + global.fetch = mockFetch as unknown as typeof fetch; + service = new BrowserMfaInstructionsService(); + }); + + afterEach(() => { + process.env.FIRECRAWL_API_KEY = originalKey; + global.fetch = originalFetch; + }); + + it('returns generated steps when the model is confident', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Security', 'Add authenticator'], confident: true }), + ); + + const result = await service.getInstructions('https://github.com/login'); + + expect(result.hostname).toBe('github.com'); + expect(result.source).toBe('generated'); + expect(result.confident).toBe(true); + expect(result.grounded).toBe(false); + expect(result.steps).toEqual(['Open Security', 'Add authenticator']); + expect(result.tips.length).toBeGreaterThan(0); + // No key → no web-search call. + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('strips markdown emphasis the model emits', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ + steps: ['Open **Settings**', 'Click `Password and authentication`'], + confident: true, + }), + ); + + const result = await service.getInstructions('github.com'); + + expect(result.steps).toEqual([ + 'Open Settings', + 'Click Password and authentication', + ]); + }); + + it('falls back to universal steps when the model is not confident', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['A guessed step'], confident: false }), + ); + + const result = await service.getInstructions('obscure-vendor.example.com'); + + expect(result.source).toBe('fallback'); + expect(result.confident).toBe(false); + expect(result.grounded).toBe(false); + expect(result.steps).not.toContain('A guessed step'); + expect(result.steps.length).toBeGreaterThan(0); + }); + + it('falls back when generation throws', async () => { + mockGenerate.mockRejectedValueOnce(new Error('model unavailable')); + + const result = await service.getInstructions('aws.amazon.com'); + + expect(result.source).toBe('fallback'); + expect(result.steps.length).toBeGreaterThan(0); + }); + + it('caches per normalized host — a full URL and bare host share one entry', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Security'], confident: true }), + ); + + const first = await service.getInstructions('https://console.aws.amazon.com/iam'); + const second = await service.getInstructions('console.aws.amazon.com'); + + expect(first.hostname).toBe('console.aws.amazon.com'); + expect(second).toEqual(first); + // Second call served from cache → the model was only invoked once. + expect(mockGenerate).toHaveBeenCalledTimes(1); + }); + + it('grounds the prompt in vendor docs when Firecrawl returns results', async () => { + process.env.FIRECRAWL_API_KEY = 'test-key'; + mockFetch.mockResolvedValueOnce( + firecrawlOk('Go to Settings > Security > Add authenticator app and click "setup key".'), + ); + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Settings', 'Add authenticator app'], confident: true }), + ); + + const result = await service.getInstructions('github.com'); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(result.grounded).toBe(true); + expect(result.source).toBe('generated'); + // The docs were injected into the generation prompt. + const promptArg = mockGenerate.mock.calls[0][0].prompt as string; + expect(promptArg).toContain('CURRENT VENDOR DOCS'); + expect(promptArg).toContain('setup key'); + }); + + it('generates ungrounded when Firecrawl fails', async () => { + process.env.FIRECRAWL_API_KEY = 'test-key'; + mockFetch.mockRejectedValueOnce(new Error('firecrawl down')); + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Security'], confident: true }), + ); + + const result = await service.getInstructions('github.com'); + + expect(result.grounded).toBe(false); + expect(result.source).toBe('generated'); + const promptArg = mockGenerate.mock.calls[0][0].prompt as string; + expect(promptArg).not.toContain('CURRENT VENDOR DOCS'); + }); +}); diff --git a/apps/api/src/browserbase/browser-mfa-instructions.service.ts b/apps/api/src/browserbase/browser-mfa-instructions.service.ts new file mode 100644 index 0000000000..6defbca960 --- /dev/null +++ b/apps/api/src/browserbase/browser-mfa-instructions.service.ts @@ -0,0 +1,271 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { anthropic } from '@ai-sdk/anthropic'; +import { generateObject } from 'ai'; +import { z } from 'zod'; +import { normalizeHostnameFromUrl } from './browserbase-url'; + +// Guidance is plain natural language with no SDK-call shape to validate, so the +// cheaper/faster model is the right fit — same choice as the manual-steps +// fallback in ai-remediation.service. +const MODEL = anthropic('claude-sonnet-4-6'); + +// A vendor's MFA setup UI rarely changes, so a day keeps guidance fresh while +// making all-but-the-first request for a vendor instant and free. In-memory is +// deliberate: instructions are public and cheap to regenerate, so a per-instance +// TTL cache avoids a DB migration; promoting this to a shared table later is a +// drop-in swap behind `getInstructions`. +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; + +// Grounding: before generating, pull the vendor's CURRENT help docs via web +// search and feed them to the model, so steps track the live UI rather than the +// model's training snapshot. Best-effort — a missing key/failure/empty result +// just falls back to ungrounded generation (still confidence-gated). +const FIRECRAWL_SEARCH_URL = 'https://api.firecrawl.dev/v2/search'; +const GROUNDING_RESULT_LIMIT = 3; +const GROUNDING_TIMEOUT_MS = 20_000; +const PER_DOC_CHARS = 2_500; +const TOTAL_DOC_CHARS = 6_000; + +interface FirecrawlSearchResult { + url?: string; + title?: string; + markdown?: string | null; +} + +interface FirecrawlSearchResponse { + success?: boolean; + data?: { web?: FirecrawlSearchResult[] }; +} + +const instructionSchema = z.object({ + steps: z + .array(z.string().min(1)) + .describe( + 'Ordered, concrete steps for a user of THIS vendor to add a NEW authenticator (TOTP) app and reveal its manual "setup key" / "secret key". Each step is one short action. 3-7 steps.', + ), + confident: z + .boolean() + .describe( + "true ONLY if these steps reflect the vendor's CURRENT, real settings UI. false if you are guessing menu/button names or do not recognize the vendor — the caller then shows a generic fallback instead of shaky specifics.", + ), +}); + +export interface MfaInstructions { + hostname: string; + steps: string[]; + tips: string[]; + confident: boolean; + /** Whether the steps were grounded in the vendor's current help docs. */ + grounded: boolean; + /** ISO timestamp the steps were produced — powers the "checked on" trust line. */ + checkedAt: string; + source: 'generated' | 'fallback'; +} + +interface CacheEntry { + value: MfaInstructions; + expiresAt: number; +} + +// Always-true, vendor-agnostic pointers, safe to show alongside any steps. These +// are universal TOTP facts, NOT per-vendor hardcode. +const UNIVERSAL_TIPS = [ + 'When the vendor shows a QR code, pick "Can\'t scan?" / "Enter this code manually" and copy that long key (the setup key) — not the rotating 6-digit code.', + 'Where the vendor allows more than one device, add a dedicated authenticator just for this automation so it can be revoked without touching your personal one.', +]; + +// The single generic safety net shown when generation is not confident. One +// universal instruction — not per-vendor hardcode — so we never show invented, +// wrong steps. +const UNIVERSAL_STEPS = [ + 'Sign in to the vendor and open your account Security / Two-factor authentication settings.', + 'Choose to add an authenticator app (TOTP).', + 'When the QR code appears, select "Can\'t scan?" / "Enter this code manually" to reveal the setup key.', + 'Copy that setup key and paste it into Comp AI.', +]; + +const SYSTEM_PROMPT = `You help a user turn on an authenticator app (TOTP) for a third-party SaaS vendor so an automation can generate their 2FA codes. + +Given only the vendor's hostname, produce the exact steps to: +1. Open that vendor's account security / two-factor settings, +2. Add a NEW authenticator app (TOTP — NOT SMS, NOT email codes, NOT a security key/passkey), +3. Reveal the manual setup key (vendors usually show a QR plus a "can't scan / enter code manually" option that reveals a long alphanumeric key). + +RULES: +- If CURRENT VENDOR DOCS are provided, treat them as the source of truth for the vendor's live UI and base the steps on them. +- Otherwise, base the steps on the vendor's CURRENT, real UI. Use real menu/button names only when you are confident of them. +- Do NOT invent specific button or menu names you are unsure about. +- Only cover authenticator-app (TOTP) setup — never SMS, email, or hardware key/passkey. +- One short action per step. 3-7 steps. +- Write each step in PLAIN TEXT. Do NOT use markdown, asterisks, backticks, or bold — the UI renders raw text. +- Set confident=true ONLY if the steps reflect this vendor's actual current UI (from the provided docs, or your solid knowledge of it). If you do not recognize the vendor or are unsure of the path, set confident=false.`; + +/** Strip markdown emphasis the model sometimes emits (`**bold**`, `__`, backticks). */ +function stripEmphasis(text: string): string { + return text.replace(/\*\*/g, '').replace(/__/g, '').replace(/`/g, '').trim(); +} + +function buildPrompt(hostname: string, grounding: string | null): string { + const base = `Vendor hostname: ${hostname} + +Write the steps for a user of this exact vendor to add an authenticator app and reveal its manual setup key.`; + + if (grounding) { + return `${base} + +CURRENT VENDOR DOCS (from a live web search — treat as the source of truth for the vendor's current UI): +""" +${grounding} +""" + +Base your steps on these docs. If they clearly describe adding an authenticator app / TOTP and revealing the manual setup key, set confident=true.`; + } + + return `${base} If you do not recognize this vendor or are not confident of its current settings UI, set confident=false.`; +} + +/** + * Produces per-vendor, human-readable instructions for obtaining an authenticator + * (TOTP) setup key, so users can hand Comp AI the seed for unattended 2FA. Steps + * are AI-generated (no per-vendor hardcode), confidence-gated to a universal + * fallback, and cached per hostname. + */ +@Injectable() +export class BrowserMfaInstructionsService { + private readonly logger = new Logger(BrowserMfaInstructionsService.name); + private readonly cache = new Map(); + + async getInstructions(rawHost: string): Promise { + const hostname = this.normalizeHost(rawHost); + + const cached = this.cache.get(hostname); + if (cached && cached.expiresAt > Date.now()) return cached.value; + + let value: MfaInstructions; + try { + value = await this.generate(hostname); + } catch (err) { + this.logger.warn( + `MFA instruction generation failed for ${hostname}; using fallback. ${ + err instanceof Error ? err.message : String(err) + }`, + ); + value = this.fallback(hostname); + } + + this.cache.set(hostname, { value, expiresAt: Date.now() + CACHE_TTL_MS }); + return value; + } + + private async generate(hostname: string): Promise { + const grounding = await this.fetchGroundingDocs(hostname); + const grounded = Boolean(grounding); + + const { object } = await generateObject({ + model: MODEL, + schema: instructionSchema, + system: SYSTEM_PROMPT, + prompt: buildPrompt(hostname, grounding), + temperature: 0.2, + }); + + // Don't show shaky, possibly-invented steps — fall back to the universal + // instruction whenever the model isn't confident (or returned nothing). + if (!object.confident || object.steps.length === 0) { + this.logger.log( + `MFA instructions for ${hostname}: not confident → fallback (grounded=${grounded})`, + ); + return this.fallback(hostname); + } + + this.logger.log( + `MFA instructions for ${hostname}: generated ${object.steps.length} step(s) (grounded=${grounded})`, + ); + return { + hostname, + steps: object.steps.map(stripEmphasis).filter((step) => step.length > 0), + tips: UNIVERSAL_TIPS, + confident: true, + grounded, + checkedAt: new Date().toISOString(), + source: 'generated', + }; + } + + private fallback(hostname: string): MfaInstructions { + return { + hostname, + steps: UNIVERSAL_STEPS, + tips: UNIVERSAL_TIPS, + confident: false, + grounded: false, + checkedAt: new Date().toISOString(), + source: 'fallback', + }; + } + + /** + * Best-effort: pull the vendor's current help docs via Firecrawl web search to + * ground the generated steps in the live UI. Returns null (never throws) when + * the key is missing, the request fails/times out, or nothing useful is found — + * generation then proceeds ungrounded. + */ + private async fetchGroundingDocs(hostname: string): Promise { + const apiKey = process.env.FIRECRAWL_API_KEY; + if (!apiKey) return null; + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), GROUNDING_TIMEOUT_MS); + try { + const response = await fetch(FIRECRAWL_SEARCH_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + query: `${hostname} set up an authenticator app (TOTP) for two-factor authentication and reveal the manual setup key / secret key`, + limit: GROUNDING_RESULT_LIMIT, + sources: [{ type: 'web' }], + scrapeOptions: { formats: ['markdown'], onlyMainContent: true }, + }), + signal: controller.signal, + }); + if (!response.ok) return null; + + const body = (await response.json()) as FirecrawlSearchResponse; + const docs = (body?.data?.web ?? []) + .map((result) => { + const markdown = (result.markdown ?? '').trim(); + if (!markdown) return null; + return `# ${result.title ?? result.url ?? 'Result'}\n${ + result.url ?? '' + }\n${markdown.slice(0, PER_DOC_CHARS)}`; + }) + .filter((doc): doc is string => Boolean(doc)); + + if (docs.length === 0) return null; + return docs.join('\n\n---\n\n').slice(0, TOTAL_DOC_CHARS); + } catch (err) { + this.logger.warn( + `Firecrawl grounding failed for ${hostname}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return null; + } finally { + clearTimeout(timeout); + } + } + + /** Accepts a full URL or a bare hostname; always returns a normalized host. */ + private normalizeHost(rawHost: string): string { + const trimmed = rawHost.trim(); + const withScheme = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`; + try { + return normalizeHostnameFromUrl(withScheme); + } catch { + return trimmed.toLowerCase(); + } + } +} diff --git a/apps/api/src/browserbase/browserbase-session.service.ts b/apps/api/src/browserbase/browserbase-session.service.ts index 76e62ca6ee..bd61557917 100644 --- a/apps/api/src/browserbase/browserbase-session.service.ts +++ b/apps/api/src/browserbase/browserbase-session.service.ts @@ -10,15 +10,71 @@ import { isNoPageError } from './run-error-formatter'; type Stagehand = import('@browserbasehq/stagehand').Stagehand; -const BROWSER_WIDTH = 1440; -const BROWSER_HEIGHT = 900; -const STAGEHAND_MODEL = 'anthropic/claude-sonnet-4-6'; +export interface BrowserViewport { + width: number; + height: number; +} + +// A larger native viewport keeps unattended evidence screenshots sharp: our +// panels are well under 1920px wide, so the viewer downscales rather than +// upscaling — the latter is what looked soft, especially on HiDPI displays. +// Browserbase exposes no device-pixel-ratio setting, so viewport size is the +// only lever for capture sharpness. This is the default. +export const CAPTURE_VIEWPORT: BrowserViewport = { width: 1920, height: 1080 }; + +// A smaller viewport for human-facing sessions (sign-in, take-over, SSO, +// reconnect): the page renders larger in the embedded live view, so forms are +// easier to read and fill. Capture runs stay on CAPTURE_VIEWPORT. +export const INTERACTIVE_VIEWPORT: BrowserViewport = { width: 1280, height: 800 }; +// Model behind extract()/act() (reading pages, verdicts, form fills). Separate +// from the navigation (CUA) model and configurable via env; default unchanged. +const STAGEHAND_MODEL = + process.env.BROWSERBASE_STAGEHAND_MODEL || 'anthropic/claude-sonnet-4-6'; const BROWSERBASE_API_MAX_ATTEMPTS = 3; const BROWSERBASE_RETRY_DELAYS_MS = [250, 750]; const BROWSERBASE_DEFAULT_HEADERS = { 'accept-encoding': 'identity' }; const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** Hostname of a URL, or null if it can't be parsed (used to match live tabs). */ +function hostnameOrNull(url: string): string | null { + try { + return new URL(url).hostname; + } catch { + return null; + } +} + +/** A live-view page as returned by Browserbase `sessions.debug()`. */ +interface LiveViewPage { + url: string; + debuggerFullscreenUrl: string; +} + +/** + * Pick the live-view URL to show: the tab matching `matchUrl` (the page the AI + * is on) by exact URL, else by hostname, else the newest tab, else the + * session-level view. Pure so the tab-selection can be unit-tested. + */ +export function selectLiveViewUrl( + debug: { debuggerFullscreenUrl?: string; pages?: LiveViewPage[] }, + matchUrl?: string, +): string | null { + const pages = debug.pages ?? []; + if (pages.length === 0) return debug.debuggerFullscreenUrl ?? null; + + if (matchUrl) { + const exact = pages.find((page) => page.url === matchUrl); + if (exact) return exact.debuggerFullscreenUrl; + const host = hostnameOrNull(matchUrl); + const byHost = + host !== null ? pages.find((page) => hostnameOrNull(page.url) === host) : undefined; + if (byHost) return byHost.debuggerFullscreenUrl; + } + // Newest tab is usually the just-opened sign-in. + return pages[pages.length - 1]?.debuggerFullscreenUrl ?? debug.debuggerFullscreenUrl ?? null; +} + @Injectable() export class BrowserbaseSessionService { private readonly logger = new Logger(BrowserbaseSessionService.name); @@ -48,6 +104,7 @@ export class BrowserbaseSessionService { async createSessionWithContext( contextId: string, + viewport: BrowserViewport = CAPTURE_VIEWPORT, ): Promise<{ sessionId: string; liveViewUrl: string }> { const bb = this.getBrowserbase(); @@ -63,13 +120,13 @@ export class BrowserbaseSessionService { }, fingerprint: { screen: { - maxHeight: BROWSER_HEIGHT, - maxWidth: BROWSER_WIDTH, - minHeight: BROWSER_HEIGHT, - minWidth: BROWSER_WIDTH, + maxHeight: viewport.height, + maxWidth: viewport.width, + minHeight: viewport.height, + minWidth: viewport.width, }, }, - viewport: { width: BROWSER_WIDTH, height: BROWSER_HEIGHT }, + viewport: { width: viewport.width, height: viewport.height }, }, keepAlive: true, }), @@ -95,6 +152,23 @@ export class BrowserbaseSessionService { } } + /** + * Live-view URL for a specific tab, so the UI can follow the AI when a vendor + * opens its sign-in in a new tab (e.g. AWS from its homepage). Prefers the tab + * matching `matchUrl` (the page the AI is on), falls back to the newest tab, + * then the session-level view. Returns null if the session has no pages. + */ + async getActivePageLiveViewUrl( + sessionId: string, + matchUrl?: string, + ): Promise { + const debug = await this.withBrowserbaseRetry({ + operationName: 'session live view lookup', + operation: () => this.getBrowserbase().sessions.debug(sessionId), + }); + return selectLiveViewUrl(debug, matchUrl); + } + async closeSession(sessionId: string): Promise { await this.withBrowserbaseRetry({ operationName: 'session close', @@ -215,6 +289,7 @@ export class BrowserbaseSessionService { try { await stagehand.init(); + await this.calmPageMotion(stagehand); return stagehand; } catch (error) { await this.safeCloseStagehand(stagehand); @@ -224,6 +299,33 @@ export class BrowserbaseSessionService { }); } + /** + * Inject "reduce motion" into every page of the session so a vendor's own + * looping animations (heavy hero backgrounds, marquees, spinners) don't churn + * the live-view stream — that constant repainting is what looks laggy. Also + * steadies evidence screenshots. Passed as a string so the browser-side + * `document` isn't type-checked in this Node file. Best-effort; a calmer live + * view is a nice-to-have, not critical to the run. + */ + private async calmPageMotion(stagehand: Stagehand): Promise { + const script = `(() => { + const css = '*,*::before,*::after{animation-duration:0.001ms!important;animation-delay:0ms!important;animation-iteration-count:1!important;transition-duration:0.001ms!important;transition-delay:0ms!important;scroll-behavior:auto!important}'; + const style = document.createElement('style'); + style.setAttribute('data-comp-reduce-motion', ''); + style.textContent = css; + const attach = () => (document.head || document.documentElement).appendChild(style); + if (document.head) attach(); + else document.addEventListener('DOMContentLoaded', attach, { once: true }); + })()`; + try { + await stagehand.context.addInitScript(script); + } catch (err) { + this.logger.warn('Could not inject reduced-motion styles (ignored)', { + error: err instanceof Error ? err.message : String(err), + }); + } + } + async safeCloseStagehand(stagehand: Stagehand): Promise { try { await stagehand.close(); diff --git a/apps/api/src/browserbase/browserbase.controller.ts b/apps/api/src/browserbase/browserbase.controller.ts index a02d1fb69c..f361c3d500 100644 --- a/apps/api/src/browserbase/browserbase.controller.ts +++ b/apps/api/src/browserbase/browserbase.controller.ts @@ -25,20 +25,28 @@ import { PermissionGuard } from '../auth/permission.guard'; import { RequirePermission } from '../auth/require-permission.decorator'; import { BrowserbaseService } from './browserbase.service'; import { + AnalyzeLoginDto, + AnalyzeLoginResponseDto, AuthStatusResponseDto, BrowserAutomationResponseDto, BrowserAutomationRunResponseDto, CheckAuthDto, CloseSessionDto, ContextResponseDto, + CreateBrowserAutomationDraftDto, CreateBrowserAutomationDto, CreateSessionDto, ExecuteAutomationSessionDto, NavigateToUrlDto, RunAutomationResponseDto, SessionResponseDto, + SetTaskScheduleDto, + TestInstructionDto, + TestInstructionResponseDto, + UpdateBrowserAutomationDraftDto, UpdateBrowserAutomationDto, } from './dto/browserbase.dto'; +import { TaskFrequency } from '@db'; @ApiTags('Browserbase') @Controller({ path: 'browserbase', version: '1' }) @@ -47,6 +55,23 @@ import { export class BrowserbaseController { constructor(private readonly browserbaseService: BrowserbaseService) {} + // ===== Login Analysis ===== + + @Post('analyze-login') + @RequirePermission('integration', 'create') + @ApiOperation({ + summary: 'Analyze a vendor sign-in page', + description: + 'Opens the vendor sign-in page in a throwaway cloud browser and detects which login methods it supports, so the connect flow can recommend the most reliable setup. Reads a public page only — no credentials. Always degrades to a manual fallback.', + }) + @ApiBody({ type: AnalyzeLoginDto }) + @ApiResponse({ status: 201, type: AnalyzeLoginResponseDto }) + async analyzeLogin( + @Body() dto: AnalyzeLoginDto, + ): Promise { + return this.browserbaseService.analyzeLogin(dto.url); + } + // ===== Organization Context ===== @Post('org-context') @@ -101,9 +126,11 @@ export class BrowserbaseController { type: SessionResponseDto, }) async createSession( + @OrganizationId() organizationId: string, @Body() dto: CreateSessionDto, ): Promise { - return await this.browserbaseService.createSessionWithContext( + return await this.browserbaseService.createSessionForOrg( + organizationId, dto.contextId, ); } @@ -118,9 +145,13 @@ export class BrowserbaseController { description: 'Session closed', }) async closeSession( + @OrganizationId() organizationId: string, @Body() dto: CloseSessionDto, ): Promise<{ success: boolean }> { - await this.browserbaseService.closeSession(dto.sessionId); + await this.browserbaseService.closeSessionForOrg( + organizationId, + dto.sessionId, + ); return { success: true }; } @@ -137,9 +168,14 @@ export class BrowserbaseController { description: 'Navigation result', }) async navigateToUrl( + @OrganizationId() organizationId: string, @Body() dto: NavigateToUrlDto, ): Promise<{ success: boolean; error?: string }> { - return await this.browserbaseService.navigateToUrl(dto.sessionId, dto.url); + return await this.browserbaseService.navigateToUrlForOrg( + organizationId, + dto.sessionId, + dto.url, + ); } @Post('check-auth') @@ -153,8 +189,12 @@ export class BrowserbaseController { description: 'Auth status', type: AuthStatusResponseDto, }) - async checkAuth(@Body() dto: CheckAuthDto): Promise { - return await this.browserbaseService.checkLoginStatus( + async checkAuth( + @OrganizationId() organizationId: string, + @Body() dto: CheckAuthDto, + ): Promise { + return await this.browserbaseService.checkLoginStatusForOrg( + organizationId, dto.sessionId, dto.url, ); @@ -182,6 +222,29 @@ export class BrowserbaseController { )) as BrowserAutomationResponseDto; } + @Post('automations/test') + @RequirePermission('task', 'update') + @ApiOperation({ + summary: 'Test an instruction before saving', + description: + 'Runs a not-yet-saved instruction against the connection’s live session so the user can watch it work before committing it to the schedule. Nothing is persisted. Returns a run handle to subscribe to for live steps and the final result.', + }) + @ApiBody({ type: TestInstructionDto }) + @ApiResponse({ status: 201, type: TestInstructionResponseDto }) + async testInstruction( + @OrganizationId() organizationId: string, + @Body() dto: TestInstructionDto, + ): Promise { + return this.browserbaseService.testInstruction({ + organizationId, + taskId: dto.taskId, + profileId: dto.profileId, + targetUrl: dto.targetUrl, + instruction: dto.instruction, + evaluationCriteria: dto.evaluationCriteria, + }); + } + @Get('automations/task/:taskId') @RequirePermission('task', 'read') @ApiOperation({ @@ -247,6 +310,26 @@ export class BrowserbaseController { )) as BrowserAutomationResponseDto; } + @Patch('automations/task/:taskId/schedule') + @RequirePermission('task', 'update') + @ApiOperation({ + summary: 'Set the schedule for every browser automation on a task', + }) + @ApiParam({ name: 'taskId', description: 'Task ID' }) + @ApiResponse({ status: 200, description: 'Task schedule updated' }) + async setTaskSchedule( + @Param('taskId') taskId: string, + @OrganizationId() organizationId: string, + @Body() dto: SetTaskScheduleDto, + ): Promise<{ success: boolean; scheduleFrequency: TaskFrequency }> { + const result = await this.browserbaseService.setTaskSchedule( + taskId, + dto.scheduleFrequency, + organizationId, + ); + return { success: true, scheduleFrequency: result.scheduleFrequency }; + } + @Delete('automations/:automationId') @RequirePermission('task', 'delete') @ApiOperation({ @@ -268,6 +351,55 @@ export class BrowserbaseController { return { success: true }; } + // ===== Automation Drafts (in-progress, unsaved) ===== + + @Get('automations/task/:taskId/drafts') + @RequirePermission('task', 'read') + @ApiOperation({ summary: 'List in-progress automation drafts for a task' }) + @ApiParam({ name: 'taskId', description: 'Task ID' }) + async listDrafts( + @OrganizationId() organizationId: string, + @Param('taskId') taskId: string, + ) { + return this.browserbaseService.listAutomationDrafts(taskId, organizationId); + } + + @Post('automation-drafts') + @RequirePermission('task', 'create') + @ApiOperation({ summary: 'Create an in-progress automation draft' }) + @ApiBody({ type: CreateBrowserAutomationDraftDto }) + async createDraft( + @OrganizationId() organizationId: string, + @Body() dto: CreateBrowserAutomationDraftDto, + ) { + return this.browserbaseService.createAutomationDraft(dto, organizationId); + } + + @Patch('automation-drafts/:draftId') + @RequirePermission('task', 'update') + @ApiOperation({ summary: 'Autosave an in-progress automation draft' }) + @ApiParam({ name: 'draftId', description: 'Draft ID' }) + @ApiBody({ type: UpdateBrowserAutomationDraftDto }) + async updateDraft( + @OrganizationId() organizationId: string, + @Param('draftId') draftId: string, + @Body() dto: UpdateBrowserAutomationDraftDto, + ) { + return this.browserbaseService.updateAutomationDraft(draftId, dto, organizationId); + } + + @Delete('automation-drafts/:draftId') + @RequirePermission('task', 'delete') + @ApiOperation({ summary: 'Discard an in-progress automation draft' }) + @ApiParam({ name: 'draftId', description: 'Draft ID' }) + async deleteDraft( + @OrganizationId() organizationId: string, + @Param('draftId') draftId: string, + ): Promise<{ success: boolean }> { + await this.browserbaseService.deleteAutomationDraft(draftId, organizationId); + return { success: true }; + } + // ===== Automation Execution ===== @Post('automations/:automationId/start-live') @@ -320,6 +452,12 @@ export class BrowserbaseController { error?: string; needsReauth?: boolean; }> { + // The session is client-supplied — confirm it belongs to this org before we + // drive an automation on it (cross-tenant IDOR guard). + await this.browserbaseService.assertSessionOwnedByOrg( + organizationId, + body.sessionId, + ); return await this.browserbaseService.executeAutomationOnSession( automationId, body.runId, @@ -328,6 +466,35 @@ export class BrowserbaseController { ); } + @Post('automations/:automationId/execute-live') + @RequirePermission('task', 'update') + @ApiOperation({ + summary: 'Execute automation on a session with live step streaming', + description: + 'Runs the automation on a pre-created session as a background task so the ' + + 'live view can stream the AI’s steps. Returns a run handle to subscribe to.', + }) + @ApiParam({ name: 'automationId', description: 'Automation ID' }) + @ApiBody({ type: ExecuteAutomationSessionDto }) + @ApiResponse({ status: 200, description: 'Run handle for realtime steps' }) + async executeAutomationLive( + @Param('automationId') automationId: string, + @Body() body: ExecuteAutomationSessionDto, + @OrganizationId() organizationId: string, + ): Promise<{ runId: string; publicAccessToken: string }> { + // The session is client-supplied — confirm it belongs to this org first. + await this.browserbaseService.assertSessionOwnedByOrg( + organizationId, + body.sessionId, + ); + return await this.browserbaseService.startLiveAutomationExecution({ + automationId, + runId: body.runId, + sessionId: body.sessionId, + organizationId, + }); + } + @Post('automations/:automationId/run') @RequirePermission('task', 'update') @ApiOperation({ diff --git a/apps/api/src/browserbase/browserbase.module.ts b/apps/api/src/browserbase/browserbase.module.ts index f0497ab123..c7835b2014 100644 --- a/apps/api/src/browserbase/browserbase.module.ts +++ b/apps/api/src/browserbase/browserbase.module.ts @@ -6,11 +6,18 @@ import { BrowserAuthProfilesController } from './browser-auth-profiles.controlle import { BrowserAuthProfileContextService } from './browser-auth-profile-context.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import { BrowserInstructionTestService } from './browser-instruction-test.service'; import { BrowserbaseController } from './browserbase.controller'; import { BrowserbaseOrgContextService } from './browserbase-org-context.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserbaseService } from './browserbase.service'; +import { BrowserCredentialSigninService } from './browser-credential-signin.service'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import { BrowserLoginAnalyzerService } from './browser-login-analyzer.service'; +import { BrowserMfaInstructionsService } from './browser-mfa-instructions.service'; +import { BROWSER_CREDENTIAL_VAULT_ADAPTER } from './credential-vault'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; import { AuthModule } from '../auth/auth.module'; @Module({ @@ -27,6 +34,15 @@ import { AuthModule } from '../auth/auth.module'; BrowserbaseOrgContextService, BrowserbaseScreenshotService, BrowserEvidenceRunnerService, + BrowserInstructionTestService, + BrowserCredentialSigninService, + BrowserCredentialStorageService, + BrowserLoginAnalyzerService, + BrowserMfaInstructionsService, + { + provide: BROWSER_CREDENTIAL_VAULT_ADAPTER, + useFactory: resolveBrowserCredentialVaultAdapter, + }, ], exports: [BrowserbaseService], }) diff --git a/apps/api/src/browserbase/browserbase.service.spec.ts b/apps/api/src/browserbase/browserbase.service.spec.ts index 29d21ee2f0..53d6917cdb 100644 --- a/apps/api/src/browserbase/browserbase.service.spec.ts +++ b/apps/api/src/browserbase/browserbase.service.spec.ts @@ -6,30 +6,46 @@ import { BrowserAutomationExecutionService } from './browser-automation-executio import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; import { BrowserAuthProfileContextService } from './browser-auth-profile-context.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import { BrowserLoginAnalyzerService } from './browser-login-analyzer.service'; import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; import { BrowserbaseOrgContextService } from './browserbase-org-context.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserbaseService } from './browserbase.service'; +import { BROWSER_CREDENTIAL_VAULT_ADAPTER } from './credential-vault'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; -jest.mock('@db', () => ({ - db: { +jest.mock('@db', () => { + const db = { browserAutomationRun: { findUnique: jest.fn(), }, browserAutomation: { create: jest.fn(), update: jest.fn(), + updateMany: jest.fn(), + findFirst: jest.fn(), }, - }, - TaskFrequency: { - daily: 'daily', - weekly: 'weekly', - monthly: 'monthly', - quarterly: 'quarterly', - yearly: 'yearly', - }, -})); + browserAutomationStep: { + deleteMany: jest.fn(), + updateMany: jest.fn(), + create: jest.fn(), + }, + // Steps updates run in a transaction; pass the same mock through as `tx`. + $transaction: jest.fn((fn: (tx: unknown) => unknown) => fn(db)), + }; + return { + db, + TaskFrequency: { + daily: 'daily', + weekly: 'weekly', + monthly: 'monthly', + quarterly: 'quarterly', + yearly: 'yearly', + }, + }; +}); jest.mock('@/app/s3', () => ({ getSignedUrl: jest.fn().mockResolvedValue('https://s3.example.com/signed'), @@ -57,6 +73,12 @@ describe('BrowserbaseService.getScreenshotRedirectUrl', () => { BrowserbaseOrgContextService, BrowserbaseScreenshotService, BrowserEvidenceRunnerService, + BrowserCredentialStorageService, + BrowserLoginAnalyzerService, + { + provide: BROWSER_CREDENTIAL_VAULT_ADAPTER, + useFactory: resolveBrowserCredentialVaultAdapter, + }, ], }).compile(); service = moduleRef.get(BrowserbaseService); @@ -175,6 +197,12 @@ describe('BrowserbaseService schedule frequency passthrough', () => { BrowserbaseOrgContextService, BrowserbaseScreenshotService, BrowserEvidenceRunnerService, + BrowserCredentialStorageService, + BrowserLoginAnalyzerService, + { + provide: BROWSER_CREDENTIAL_VAULT_ADAPTER, + useFactory: resolveBrowserCredentialVaultAdapter, + }, ], }).compile(); service = moduleRef.get(BrowserbaseService); @@ -243,4 +271,105 @@ describe('BrowserbaseService schedule frequency passthrough', () => { const call = (db.browserAutomation.update as jest.Mock).mock.calls[0][0]; expect(call.data).not.toHaveProperty('scheduleFrequency'); }); + + it('inherits the task cadence for a new automation when none is given', async () => { + (db.browserAutomation.findFirst as jest.Mock).mockResolvedValue({ + scheduleFrequency: TaskFrequency.weekly, + }); + (db.browserAutomation.create as jest.Mock).mockResolvedValue({ id: 'bau_2' }); + + await service.createBrowserAutomation({ + taskId: 'tsk_1', + name: 'name', + targetUrl: 'https://example.com', + instruction: 'click', + }); + + expect(db.browserAutomation.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ scheduleFrequency: 'weekly' }), + }), + ); + }); + + it('sets one schedule for every automation on the task', async () => { + (db.browserAutomation.updateMany as jest.Mock).mockResolvedValue({ count: 3 }); + + const result = await service.setTaskSchedule('tsk_1', TaskFrequency.monthly); + + expect(db.browserAutomation.updateMany).toHaveBeenCalledWith({ + where: { taskId: 'tsk_1' }, + data: { scheduleFrequency: 'monthly' }, + }); + expect(result).toEqual({ + success: true, + scheduleFrequency: 'monthly', + updated: 3, + }); + }); + + it('stores explicit steps and mirrors the first onto the legacy columns', async () => { + (db.browserAutomation.create as jest.Mock).mockResolvedValue({ id: 'bau_1' }); + + await service.createBrowserAutomation({ + taskId: 't1', + name: 'A', + targetUrl: 'https://ignored.com', + instruction: 'ignored', + steps: [ + { + profileId: 'p1', + targetUrl: 'https://github.com', + instruction: 'screenshot 2fa', + evaluationCriteria: '2fa enforced', + }, + { targetUrl: 'https://aws.amazon.com', instruction: 'capture policy' }, + ], + }); + + const data = (db.browserAutomation.create as jest.Mock).mock.calls[0][0].data; + expect(data.targetUrl).toBe('https://github.com'); // mirrored from step 0 + expect(data.instruction).toBe('screenshot 2fa'); + expect(data.steps.create).toHaveLength(2); + expect(data.steps.create[0]).toMatchObject({ + order: 0, + profileId: 'p1', + targetUrl: 'https://github.com', + }); + expect(data.steps.create[1]).toMatchObject({ order: 1, profileId: null }); + }); + + it('wraps a single inline instruction as one step', async () => { + (db.browserAutomation.create as jest.Mock).mockResolvedValue({ id: 'bau_1' }); + + await service.createBrowserAutomation({ + taskId: 't1', + name: 'A', + targetUrl: 'https://x.com', + instruction: 'do it', + }); + + const data = (db.browserAutomation.create as jest.Mock).mock.calls[0][0].data; + expect(data.steps.create).toHaveLength(1); + expect(data.steps.create[0]).toMatchObject({ + order: 0, + targetUrl: 'https://x.com', + instruction: 'do it', + }); + }); + + it('replaces the step list when steps are supplied on update', async () => { + (db.browserAutomation.update as jest.Mock).mockResolvedValue({ id: 'bau_1' }); + + await service.updateBrowserAutomation('bau_1', { + steps: [{ targetUrl: 'https://okta.com', instruction: 'sso' }], + }); + + expect(db.browserAutomationStep.deleteMany).toHaveBeenCalledWith({ + where: { automationId: 'bau_1' }, + }); + const data = (db.browserAutomation.update as jest.Mock).mock.calls[0][0].data; + expect(data.targetUrl).toBe('https://okta.com'); + expect(data.steps.create).toHaveLength(1); + }); }); diff --git a/apps/api/src/browserbase/browserbase.service.ts b/apps/api/src/browserbase/browserbase.service.ts index 696133553f..3b3d9f2e81 100644 --- a/apps/api/src/browserbase/browserbase.service.ts +++ b/apps/api/src/browserbase/browserbase.service.ts @@ -1,8 +1,18 @@ import { Injectable } from '@nestjs/common'; import { TaskFrequency } from '@db'; -import { BrowserAutomationCrudService } from './browser-automation-crud.service'; +import { tasks } from '@trigger.dev/sdk'; +import { + BrowserAutomationCrudService, + type BrowserAutomationStepInput, +} from './browser-automation-crud.service'; +import { BrowserAutomationDraftService } from './browser-automation-draft.service'; import { BrowserAutomationExecutionService } from './browser-automation-execution.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import type { + BrowserRunLivePhase, + EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; import { BrowserbaseSessionService } from './browserbase-session.service'; @@ -23,10 +33,73 @@ export class BrowserbaseService { private readonly automationCrud: BrowserAutomationCrudService = new BrowserAutomationCrudService( screenshots, ), - private readonly automationExecution: BrowserAutomationExecutionService = - new BrowserAutomationExecutionService(sessions, profiles, runner), + private readonly automationExecution: BrowserAutomationExecutionService = new BrowserAutomationExecutionService( + sessions, + profiles, + runner, + ), + private readonly credentialStorage: BrowserCredentialStorageService = new BrowserCredentialStorageService(), ) {} + // Drafts have no dependencies; a field avoids Nest trying to DI-resolve it. + private readonly automationDrafts = new BrowserAutomationDraftService(); + + /** + * Kicks off login analysis as a background Trigger.dev run (browser + AI, which + * can outlast an HTTP/browser timeout) and returns a handle the client + * subscribes to for the result. + */ + async analyzeLogin(url: string): Promise<{ + runId: string; + publicAccessToken: string; + }> { + const handle = await tasks.trigger('analyze-vendor-login', { url }); + return { runId: handle.id, publicAccessToken: handle.publicAccessToken }; + } + + /** + * Kicks off a live test of an instruction the user hasn't saved yet. Creates + * the session up front (so the client shows it as a live view), then runs the + * instruction as a background Trigger.dev task that streams its steps. Nothing + * is persisted — this only proves the instruction out before it's saved. + */ + async testInstruction(input: { + organizationId: string; + taskId?: string; + profileId?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + }): Promise<{ + runId: string; + publicAccessToken: string; + sessionId: string; + liveViewUrl: string; + }> { + const profile = await this.profiles.resolveProfileForTarget({ + organizationId: input.organizationId, + targetUrl: input.targetUrl, + profileId: input.profileId, + }); + const { sessionId, liveViewUrl } = + await this.createSessionWithContext(profile.contextId); + const handle = await tasks.trigger('test-vendor-instruction', { + organizationId: input.organizationId, + taskId: input.taskId, + profileId: profile.id, + targetUrl: input.targetUrl, + instruction: input.instruction, + evaluationCriteria: input.evaluationCriteria, + sessionId, + }); + return { + runId: handle.id, + publicAccessToken: handle.publicAccessToken, + sessionId, + liveViewUrl, + }; + } + async listAuthProfiles(organizationId: string) { return this.profiles.listProfiles(organizationId); } @@ -67,6 +140,73 @@ export class BrowserbaseService { return this.profiles.markNeedsReauth(input); } + async updateAuthProfile(input: { + organizationId: string; + profileId: string; + displayName?: string; + url?: string; + }) { + return this.profiles.updateProfile(input); + } + + async deleteAuthProfile(input: { organizationId: string; profileId: string }) { + // Best-effort: remove the stored login from 1Password before dropping the + // profile, so we don't leave orphaned secrets behind. + const profile = await this.profiles.getProfile(input); + if (profile?.vaultExternalItemRef) { + await this.credentialStorage.deleteProfileCredentialItem(profile); + } + return this.profiles.deleteProfile(input); + } + + async storeAuthProfileCredentials(input: { + organizationId: string; + profileId: string; + username: string; + password: string; + totpSeed?: string; + extraFields?: { label: string; value: string }[]; + usernameLabel?: string; + }) { + return this.credentialStorage.storeProfileCredentials(input); + } + + /** + * Kicks off the connect flow's first automated sign-in. Creates the browser + * session up front (so the client can show it as a live view — the user + * watches the auto-fill and takes over in place if it can't finish), then runs + * the sign-in as a background Trigger.dev task on that session (browser + AI, + * which can outlast an HTTP/browser timeout). + */ + async signInAuthProfile(input: { + organizationId: string; + profileId: string; + url: string; + mode?: 'password' | 'sso'; + /** Vendor's identifier-field label, forwarded so the streamed step is truthful. */ + usernameLabel?: string; + }): Promise<{ + runId: string; + publicAccessToken: string; + sessionId: string; + liveViewUrl: string; + }> { + const { sessionId, liveViewUrl } = await this.profiles.startProfileSession({ + organizationId: input.organizationId, + profileId: input.profileId, + }); + const handle = await tasks.trigger('sign-in-vendor-profile', { + ...input, + sessionId, + }); + return { + runId: handle.id, + publicAccessToken: handle.publicAccessToken, + sessionId, + liveViewUrl, + }; + } + async getOrCreateOrgContext(organizationId: string) { return this.profiles.getOrCreateOrgContext(organizationId); } @@ -91,6 +231,59 @@ export class BrowserbaseService { return this.sessions.checkLoginStatus(sessionId, url); } + // ─── Tenant-safe wrappers for the raw session/context endpoints ──────────── + // Every raw session/context operation must confirm the target belongs to the + // caller's org before acting on it (the persisted context holds the org's + // authenticated vendor cookies) — otherwise it's a cross-tenant IDOR. + + async createSessionForOrg(organizationId: string, contextId: string) { + await this.profiles.assertContextOwnedByOrg({ organizationId, contextId }); + return this.sessions.createSessionWithContext(contextId); + } + + async closeSessionForOrg( + organizationId: string, + sessionId: string, + ): Promise { + // Tolerant: a session that can no longer be resolved is already gone, so + // closing is a no-op. A resolvable session that isn't the org's is rejected. + let contextId: string | undefined; + try { + contextId = await this.sessions.getSessionContextId(sessionId); + } catch { + return; + } + if (!contextId) return; + await this.profiles.assertContextOwnedByOrg({ organizationId, contextId }); + await this.sessions.closeSession(sessionId); + } + + async navigateToUrlForOrg( + organizationId: string, + sessionId: string, + url: string, + ) { + await this.profiles.assertSessionOwnedByOrg({ organizationId, sessionId }); + return this.sessions.navigateToUrl(sessionId, url); + } + + async checkLoginStatusForOrg( + organizationId: string, + sessionId: string, + url: string, + ) { + await this.profiles.assertSessionOwnedByOrg({ organizationId, sessionId }); + return this.sessions.checkLoginStatus(sessionId, url); + } + + /** Confirms a client-supplied session belongs to the caller's org. */ + async assertSessionOwnedByOrg( + organizationId: string, + sessionId: string, + ): Promise { + await this.profiles.assertSessionOwnedByOrg({ organizationId, sessionId }); + } + async createBrowserAutomation( data: { taskId: string; @@ -99,6 +292,7 @@ export class BrowserbaseService { targetUrl: string; instruction: string; evaluationCriteria?: string; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -107,11 +301,17 @@ export class BrowserbaseService { } async getBrowserAutomation(automationId: string, organizationId?: string) { - return this.automationCrud.getBrowserAutomation(automationId, organizationId); + return this.automationCrud.getBrowserAutomation( + automationId, + organizationId, + ); } async getBrowserAutomationsForTask(taskId: string, organizationId?: string) { - return this.automationCrud.getBrowserAutomationsForTask(taskId, organizationId); + return this.automationCrud.getBrowserAutomationsForTask( + taskId, + organizationId, + ); } async updateBrowserAutomation( @@ -123,6 +323,7 @@ export class BrowserbaseService { instruction?: string; evaluationCriteria?: string; isEnabled?: boolean; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -135,10 +336,53 @@ export class BrowserbaseService { } async deleteBrowserAutomation(automationId: string, organizationId?: string) { - return this.automationCrud.deleteBrowserAutomation(automationId, organizationId); + return this.automationCrud.deleteBrowserAutomation( + automationId, + organizationId, + ); + } + + async setTaskSchedule( + taskId: string, + scheduleFrequency: TaskFrequency, + organizationId?: string, + ) { + return this.automationCrud.setTaskSchedule( + taskId, + scheduleFrequency, + organizationId, + ); + } + + // ===== Drafts (in-progress, unsaved automations) ===== + + listAutomationDrafts(taskId: string, organizationId: string) { + return this.automationDrafts.listDraftsForTask(taskId, organizationId); } - async startAutomationWithLiveView(automationId: string, organizationId: string) { + createAutomationDraft( + data: { taskId: string; name?: string; steps: unknown; createdById?: string | null }, + organizationId: string, + ) { + return this.automationDrafts.createDraft(data, organizationId); + } + + updateAutomationDraft( + draftId: string, + data: { name?: string; steps?: unknown }, + organizationId: string, + ) { + return this.automationDrafts.updateDraft(draftId, data, organizationId); + } + + deleteAutomationDraft(draftId: string, organizationId: string) { + return this.automationDrafts.deleteDraft(draftId, organizationId); + } + + async startAutomationWithLiveView( + automationId: string, + organizationId: string, + ) { return this.automationExecution.startAutomationWithLiveView( automationId, organizationId, @@ -150,15 +394,53 @@ export class BrowserbaseService { runId: string, sessionId: string, organizationId: string, + onSteps?: (steps: EvidenceTimelineStep[]) => void, ) { return this.automationExecution.executeAutomationOnSession( automationId, runId, sessionId, organizationId, + onSteps, ); } + /** Runs the FULL step sequence on a live session, streaming the timeline. */ + async executeAutomationLive( + automationId: string, + runId: string, + sessionId: string, + organizationId: string, + onSteps?: (steps: EvidenceTimelineStep[]) => void, + onLiveView?: (url: string) => void, + onLivePhase?: (phase: BrowserRunLivePhase) => void, + ) { + return this.automationExecution.executeAutomationLive( + automationId, + runId, + sessionId, + organizationId, + onSteps, + onLiveView, + onLivePhase, + ); + } + + /** + * Kick off the interactive Run as a background task so the live view can + * stream the AI's steps (same realtime mechanism as the Test flow). Returns a + * handle the composer subscribes to for `runSteps` + the final result. + */ + async startLiveAutomationExecution(input: { + automationId: string; + runId: string; + sessionId: string; + organizationId: string; + }): Promise<{ runId: string; publicAccessToken: string }> { + const handle = await tasks.trigger('execute-automation-live', input); + return { runId: handle.id, publicAccessToken: handle.publicAccessToken }; + } + async runBrowserAutomation(automationId: string, organizationId: string) { return this.automationExecution.runBrowserAutomation( automationId, @@ -178,7 +460,10 @@ export class BrowserbaseService { return this.automationCrud.getRunWithPresignedUrl(runId, organizationId); } - async getAutomationsWithPresignedUrls(taskId: string, organizationId?: string) { + async getAutomationsWithPresignedUrls( + taskId: string, + organizationId?: string, + ) { return this.automationCrud.getAutomationsWithPresignedUrls( taskId, organizationId, diff --git a/apps/api/src/browserbase/credential-vault.ts b/apps/api/src/browserbase/credential-vault.ts index aa9f111411..f87964f370 100644 --- a/apps/api/src/browserbase/credential-vault.ts +++ b/apps/api/src/browserbase/credential-vault.ts @@ -2,6 +2,8 @@ export interface RuntimeCredentialMaterial { username?: string; password?: string; totpCode?: string; + /** Extra site-specific fields (e.g. workspace, subdomain), filled by label. */ + extraFields?: { label: string; value: string }[]; } export const BROWSER_CREDENTIAL_VAULT_ADAPTER = @@ -16,9 +18,7 @@ export interface BrowserCredentialVaultAdapter { }): Promise; } -export class NoopBrowserCredentialVaultAdapter - implements BrowserCredentialVaultAdapter -{ +export class NoopBrowserCredentialVaultAdapter implements BrowserCredentialVaultAdapter { async resolveCredentialReference(_params: { profileId: string; provider?: string | null; diff --git a/apps/api/src/browserbase/dto/browserbase.dto.ts b/apps/api/src/browserbase/dto/browserbase.dto.ts index 1fb7b29b18..87ce68feec 100644 --- a/apps/api/src/browserbase/dto/browserbase.dto.ts +++ b/apps/api/src/browserbase/dto/browserbase.dto.ts @@ -1,11 +1,14 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; import { + IsArray, IsEnum, IsNotEmpty, IsOptional, IsString, IsBoolean, IsUrl, + ValidateNested, } from 'class-validator'; import { TaskFrequency } from '@db'; import { IsSafeUrl } from '../validators/url-safety.validator'; @@ -19,6 +22,16 @@ export class CreateSessionDto { contextId: string; } +export class SetAuthProfileTotpDto { + @ApiProperty({ + description: + "The authenticator setup key (TOTP seed) shown once during the vendor's authenticator-app setup — not the rotating 6-digit code.", + }) + @IsString() + @IsNotEmpty() + totpSeed: string; +} + export class NavigateToUrlDto { @ApiProperty({ description: 'Browserbase session ID' }) @IsString() @@ -62,10 +75,61 @@ export class AuthStatusResponseDto { username?: string; } +// ===== Login analysis DTOs ===== + +export class AnalyzeLoginDto { + @ApiProperty({ description: 'Vendor sign-in URL to analyze' }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + url: string; +} + +export class LoginRecommendationDto { + @ApiProperty({ enum: ['ready', 'works_with_checkins', 'manual'] }) + category: string; + + @ApiProperty() + headline: string; + + @ApiProperty() + detail: string; +} + +export class AnalyzeLoginResponseDto { + @ApiProperty({ + description: 'Trigger.dev run id for the background analysis', + }) + runId: string; + + @ApiProperty({ description: 'Public access token to subscribe to the run' }) + publicAccessToken: string; +} + +export class LoginAnalysisResponseDto { + @ApiProperty() + reachable: boolean; + + @ApiProperty({ type: [String] }) + detectedMethods: string[]; + + @ApiProperty({ enum: ['email', 'username', 'either', 'unknown'] }) + identifierType: string; + + @ApiProperty({ type: [Object] }) + extraFields: { label: string }[]; + + @ApiProperty({ type: () => LoginRecommendationDto }) + recommendation: LoginRecommendationDto; +} + // ===== Auth Profile DTOs ===== export class ResolveAuthProfileDto { - @ApiProperty({ description: 'Website URL to normalize into an auth profile hostname' }) + @ApiProperty({ + description: 'Website URL to normalize into an auth profile hostname', + }) @IsUrl({}, { message: 'url must be a valid URL' }) @IsSafeUrl({ message: 'The provided URL is not allowed.' }) @IsString() @@ -114,13 +178,130 @@ export class VerifyAuthProfileSessionDto { url: string; } +export class UpdateAuthProfileDto { + @ApiPropertyOptional({ description: 'Display name for the connection' }) + @IsString() + @IsOptional() + displayName?: string; + + @ApiPropertyOptional({ + description: + 'Sign-in URL. Changing to a different hostname signs the connection out.', + }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsOptional() + url?: string; +} + export class MarkAuthProfileNeedsReauthDto { - @ApiPropertyOptional({ description: 'Reason the profile needs re-authentication' }) + @ApiPropertyOptional({ + description: 'Reason the profile needs re-authentication', + }) @IsString() @IsOptional() reason?: string; } +export class SignInAuthProfileDto { + @ApiProperty({ + description: 'Vendor URL to sign in to (the profile hostname).', + }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + url: string; + + @ApiPropertyOptional({ + enum: ['password', 'sso'], + description: + "'password' fills stored credentials; 'sso' has the AI open the identity provider for the user to finish.", + }) + @IsEnum({ password: 'password', sso: 'sso' }) + @IsOptional() + mode?: 'password' | 'sso'; + + @ApiPropertyOptional({ + description: + "The vendor's detected identifier-field label (e.g. 'IAM username'), shown in the live sign-in steps instead of a generic 'username'.", + }) + @IsString() + @IsOptional() + usernameLabel?: string; +} + +export class SignInAuthProfileResponseDto { + @ApiProperty({ + description: 'Trigger.dev run id for the background automated sign-in', + }) + runId: string; + + @ApiProperty({ description: 'Public access token to subscribe to the run' }) + publicAccessToken: string; + + @ApiProperty({ + description: 'Browserbase session id the sign-in runs on (for take-over)', + }) + sessionId: string; + + @ApiProperty({ + description: 'Live view URL so the user can watch and take over the sign-in', + }) + liveViewUrl: string; +} + +export class CredentialExtraFieldDto { + @ApiProperty({ description: 'Field label as shown on the vendor login' }) + @IsString() + @IsNotEmpty() + label: string; + + @ApiProperty({ description: 'Field value' }) + @IsString() + @IsNotEmpty() + value: string; +} + +export class StoreAuthProfileCredentialsDto { + @ApiProperty({ description: 'Username or email for the vendor login' }) + @IsString() + @IsNotEmpty() + username: string; + + @ApiProperty({ description: 'Password for the vendor login' }) + @IsString() + @IsNotEmpty() + password: string; + + @ApiPropertyOptional({ + description: + 'Authenticator app setup key (TOTP secret or otpauth:// URI). When provided, one-time codes are generated for automated sign-in.', + }) + @IsString() + @IsOptional() + totpSeed?: string; + + @ApiPropertyOptional({ + type: [CredentialExtraFieldDto], + description: 'Extra site-specific fields (e.g. workspace, subdomain).', + }) + @IsArray() + @IsOptional() + @ValidateNested({ each: true }) + @Type(() => CredentialExtraFieldDto) + extraFields?: CredentialExtraFieldDto[]; + + @ApiPropertyOptional({ + description: + "The vendor's own label for the identifier field (e.g. \"IAM username\"), stored so sign-in steps and reconnects show the real field name.", + }) + @IsString() + @IsOptional() + usernameLabel?: string; +} + export class BrowserAuthProfileResponseDto { @ApiProperty() id: string; @@ -166,6 +347,11 @@ export class BrowserAuthProfileResponseDto { @ApiProperty() updatedAt: Date; + + @ApiPropertyOptional({ + description: 'Number of browser automations in the org that run on this connection', + }) + automationCount?: number; } export class ResolveAuthProfileResponseDto { @@ -186,6 +372,32 @@ export class VerifyAuthProfileResponseDto { // ===== Browser Automation DTOs ===== +export class BrowserAutomationStepDto { + @ApiPropertyOptional({ + description: 'Connection (browser auth profile) this step runs on', + }) + @IsString() + @IsOptional() + profileId?: string; + + @ApiProperty({ description: 'Starting URL for this step' }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + targetUrl: string; + + @ApiProperty({ description: 'Natural language instruction for this step' }) + @IsString() + @IsNotEmpty() + instruction: string; + + @ApiPropertyOptional({ description: 'Optional pass/fail criterion for this step' }) + @IsString() + @IsOptional() + evaluationCriteria?: string; +} + export class CreateBrowserAutomationDto { @ApiProperty({ description: 'Task ID this automation belongs to' }) @IsString() @@ -222,6 +434,17 @@ export class CreateBrowserAutomationDto { @IsOptional() evaluationCriteria?: string; + @ApiPropertyOptional({ + type: [BrowserAutomationStepDto], + description: + 'Ordered steps for a multi-vendor automation. When provided, these define the run; the top-level targetUrl/instruction are taken from the first step.', + }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => BrowserAutomationStepDto) + @IsOptional() + steps?: BrowserAutomationStepDto[]; + @ApiPropertyOptional({ enum: TaskFrequency, description: 'Automation schedule cadence', @@ -231,6 +454,15 @@ export class CreateBrowserAutomationDto { scheduleFrequency?: TaskFrequency; } +export class SetTaskScheduleDto { + @ApiProperty({ + enum: TaskFrequency, + description: 'Cadence applied to every browser automation on the task', + }) + @IsEnum(TaskFrequency) + scheduleFrequency: TaskFrequency; +} + export class UpdateBrowserAutomationDto { @ApiPropertyOptional({ description: 'Automation name' }) @IsString() @@ -267,6 +499,17 @@ export class UpdateBrowserAutomationDto { @IsOptional() isEnabled?: boolean; + @ApiPropertyOptional({ + type: [BrowserAutomationStepDto], + description: + 'Ordered steps for a multi-vendor automation. When provided, they replace the automation’s existing steps.', + }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => BrowserAutomationStepDto) + @IsOptional() + steps?: BrowserAutomationStepDto[]; + @ApiPropertyOptional({ enum: TaskFrequency, description: 'Automation schedule cadence', @@ -276,6 +519,61 @@ export class UpdateBrowserAutomationDto { scheduleFrequency?: TaskFrequency; } +/** A draft step — everything optional, since a draft can be half-written. */ +export class DraftStepDto { + @ApiPropertyOptional() + @IsString() + @IsOptional() + profileId?: string; + + @ApiPropertyOptional() + @IsString() + @IsOptional() + targetUrl?: string; + + @ApiPropertyOptional() + @IsString() + @IsOptional() + instruction?: string; + + @ApiPropertyOptional() + @IsString() + @IsOptional() + evaluationCriteria?: string; +} + +export class CreateBrowserAutomationDraftDto { + @ApiProperty({ description: 'Task the draft belongs to' }) + @IsString() + @IsNotEmpty() + taskId: string; + + @ApiPropertyOptional({ description: 'Preview name, derived from the first step' }) + @IsString() + @IsOptional() + name?: string; + + @ApiProperty({ type: [DraftStepDto], description: 'The composer step state' }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => DraftStepDto) + steps: DraftStepDto[]; +} + +export class UpdateBrowserAutomationDraftDto { + @ApiPropertyOptional() + @IsString() + @IsOptional() + name?: string; + + @ApiPropertyOptional({ type: [DraftStepDto] }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => DraftStepDto) + @IsOptional() + steps?: DraftStepDto[]; +} + export class ExecuteAutomationSessionDto { @ApiProperty({ description: 'Browser automation run ID' }) @IsString() @@ -413,3 +711,51 @@ export class RunAutomationResponseDto { @ApiPropertyOptional() blockedReason?: string; } + +// ===== Instruction Test (coach loop) DTOs ===== + +export class TestInstructionDto { + @ApiProperty({ description: 'URL the AI should start from' }) + @IsUrl({}, { message: 'targetUrl must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + targetUrl: string; + + @ApiProperty({ description: 'Natural language instruction to test' }) + @IsString() + @IsNotEmpty() + instruction: string; + + @ApiPropertyOptional({ + description: + 'Optional pass/fail criteria. When set, the test run gets a verdict.', + }) + @IsString() + @IsOptional() + evaluationCriteria?: string; + + @ApiPropertyOptional({ description: 'Connection (browser auth profile) to run under' }) + @IsString() + @IsOptional() + profileId?: string; + + @ApiPropertyOptional({ description: 'Task the instruction belongs to' }) + @IsString() + @IsOptional() + taskId?: string; +} + +export class TestInstructionResponseDto { + @ApiProperty({ description: 'Trigger.dev run id to subscribe to' }) + runId: string; + + @ApiProperty({ description: 'Public token for realtime subscription' }) + publicAccessToken: string; + + @ApiProperty({ description: 'Browserbase session id backing the live view' }) + sessionId: string; + + @ApiProperty({ description: 'Live view URL for watching the test run' }) + liveViewUrl: string; +} diff --git a/apps/api/src/browserbase/onepassword-client.ts b/apps/api/src/browserbase/onepassword-client.ts new file mode 100644 index 0000000000..5edd97e900 --- /dev/null +++ b/apps/api/src/browserbase/onepassword-client.ts @@ -0,0 +1,57 @@ +import { Logger } from '@nestjs/common'; + +type OnePasswordModule = typeof import('@1password/sdk'); +export type OnePasswordClient = Awaited< + ReturnType +>; + +const OP_INTEGRATION_NAME = 'Comp AI Browser Automations'; +const OP_INTEGRATION_VERSION = '1.0.0'; + +const logger = new Logger('OnePasswordClient'); +let clientPromise: Promise | null = null; + +export function getOnePasswordServiceAccountToken(): string | undefined { + const token = process.env.OP_SERVICE_ACCOUNT_TOKEN?.trim(); + return token ? token : undefined; +} + +export function isOnePasswordConfigured(): boolean { + return Boolean(getOnePasswordServiceAccountToken()); +} + +// Dynamic import keeps the WASM-backed SDK out of the module graph until it is +// actually needed, so runtimes that never touch 1Password don't pay to load it +// and bundlers don't have to resolve the WASM core eagerly. Exported so the +// write path can read the SDK's runtime enums (ItemCategory/ItemFieldType) +// without a static import that would defeat the lazy load. +export async function loadOnePasswordModule(): Promise { + return import('@1password/sdk'); +} + +export async function getOnePasswordClient(): Promise { + const token = getOnePasswordServiceAccountToken(); + if (!token) { + throw new Error( + 'OP_SERVICE_ACCOUNT_TOKEN is not configured; cannot reach 1Password.', + ); + } + + if (!clientPromise) { + clientPromise = (async () => { + const { createClient } = await loadOnePasswordModule(); + logger.log('Initializing 1Password service account client.'); + return createClient({ + auth: token, + integrationName: OP_INTEGRATION_NAME, + integrationVersion: OP_INTEGRATION_VERSION, + }); + })().catch((error: unknown) => { + // Reset so a transient failure doesn't permanently poison the singleton. + clientPromise = null; + throw error; + }); + } + + return clientPromise; +} diff --git a/apps/api/src/browserbase/onepassword-credential-item.ts b/apps/api/src/browserbase/onepassword-credential-item.ts new file mode 100644 index 0000000000..29a84746f6 --- /dev/null +++ b/apps/api/src/browserbase/onepassword-credential-item.ts @@ -0,0 +1,40 @@ +// Shared contract between the write path (storing a login into 1Password) and the +// read path (resolving it at run time). Both sides MUST use the same field titles +// and reference format, so they live here as the single source of truth. + +export const ONEPASSWORD_PROVIDER = '1password'; + +// Field titles match 1Password's built-in Login item fields so secret references +// (`op:////`) resolve them by label. +export const USERNAME_FIELD_TITLE = 'username'; +export const PASSWORD_FIELD_TITLE = 'password'; +export const TOTP_FIELD_TITLE = 'one-time password'; + +export function buildOrgVaultTitle(organizationId: string): string { + return `Comp AI Browser Automations — ${organizationId}`; +} + +export function buildItemReference(vaultId: string, itemId: string): string { + return `op://${vaultId}/${itemId}`; +} + +export function parseItemReference(itemRef: string): { + vaultId?: string; + itemId?: string; +} { + const [vaultId, itemId] = itemRef.replace(/^op:\/\//, '').split('/'); + return { vaultId, itemId }; +} + +export function buildFieldReference( + itemRef: string, + fieldTitle: string, +): string { + return `${itemRef}/${fieldTitle}`; +} + +// The `?attribute=otp` modifier makes 1Password return the *computed* 6-digit +// code for a Totp field rather than the stored seed. +export function buildTotpReference(itemRef: string): string { + return `${itemRef}/${TOTP_FIELD_TITLE}?attribute=otp`; +} diff --git a/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts b/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts new file mode 100644 index 0000000000..3a1e843d7d --- /dev/null +++ b/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts @@ -0,0 +1,141 @@ +import { OnePasswordCredentialVaultAdapter } from './onepassword-credential-vault.adapter'; +import { + getOnePasswordClient, + type OnePasswordClient, +} from './onepassword-client'; + +jest.mock('./onepassword-client', () => ({ + getOnePasswordClient: jest.fn(), +})); + +const mockedGetClient = jest.mocked(getOnePasswordClient); + +function clientWith( + resolve: jest.Mock, + itemsGet?: jest.Mock, +): OnePasswordClient { + return { + secrets: { resolve }, + items: { + get: itemsGet ?? jest.fn().mockRejectedValue(new Error('no items')), + }, + } as unknown as OnePasswordClient; +} + +describe('OnePasswordCredentialVaultAdapter', () => { + const adapter = new OnePasswordCredentialVaultAdapter(); + + beforeEach(() => jest.clearAllMocks()); + + it('returns null (and never touches 1Password) when provider is not 1password', async () => { + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: 'lastpass', + externalItemRef: 'op://v/i', + }); + + expect(result).toBeNull(); + expect(mockedGetClient).not.toHaveBeenCalled(); + }); + + it('returns null when the item reference is missing', async () => { + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: '1password', + externalItemRef: ' ', + }); + + expect(result).toBeNull(); + expect(mockedGetClient).not.toHaveBeenCalled(); + }); + + it('resolves username, password, and the live TOTP code', async () => { + const resolve = jest.fn((reference: string) => { + if (reference.endsWith('/username')) return 'alice@example.com'; + if (reference.endsWith('/password')) return 's3cret'; + if (reference.includes('one-time password')) return '123456'; + throw new Error(`unexpected reference: ${reference}`); + }); + mockedGetClient.mockResolvedValue(clientWith(resolve)); + + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: '1password', + externalItemRef: 'op://vault123/item456', + }); + + expect(result).toEqual({ + username: 'alice@example.com', + password: 's3cret', + totpCode: '123456', + }); + expect(resolve).toHaveBeenCalledWith( + 'op://vault123/item456/one-time password?attribute=otp', + ); + }); + + it('treats a missing TOTP field as absent rather than failing', async () => { + const resolve = jest.fn((reference: string) => { + if (reference.endsWith('/username')) return 'alice'; + if (reference.endsWith('/password')) return 'pw'; + throw new Error('field not found'); + }); + mockedGetClient.mockResolvedValue(clientWith(resolve)); + + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: '1password', + externalItemRef: 'op://v/i', + }); + + expect(result).toEqual({ + username: 'alice', + password: 'pw', + totpCode: undefined, + }); + }); + + it('returns null when no fields resolve', async () => { + const resolve = jest.fn(() => { + throw new Error('nothing here'); + }); + mockedGetClient.mockResolvedValue(clientWith(resolve)); + + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: '1password', + externalItemRef: 'op://v/i', + }); + + expect(result).toBeNull(); + }); + + it('resolves extra custom fields, ignoring the reserved login fields', async () => { + const resolve = jest.fn((reference: string) => { + if (reference.endsWith('/username')) return 'alice'; + if (reference.endsWith('/password')) return 'pw'; + throw new Error('no totp'); + }); + const itemsGet = jest.fn().mockResolvedValue({ + fields: [ + { title: 'username', value: 'alice' }, + { title: 'password', value: 'pw' }, + { title: 'one-time password', value: 'seed' }, + { title: 'Workspace URL', value: 'acme.example.com' }, + { title: 'Empty', value: ' ' }, + ], + }); + mockedGetClient.mockResolvedValue(clientWith(resolve, itemsGet)); + + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: '1password', + externalItemRef: 'op://vault123/item456', + }); + + expect(itemsGet).toHaveBeenCalledWith('vault123', 'item456'); + expect(result?.extraFields).toEqual([ + { label: 'Workspace URL', value: 'acme.example.com' }, + ]); + }); +}); diff --git a/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts b/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts new file mode 100644 index 0000000000..3e3324ea59 --- /dev/null +++ b/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts @@ -0,0 +1,117 @@ +import { Logger } from '@nestjs/common'; +import { + type BrowserCredentialVaultAdapter, + type RuntimeCredentialMaterial, +} from './credential-vault'; +import { + getOnePasswordClient, + type OnePasswordClient, +} from './onepassword-client'; +import { + ONEPASSWORD_PROVIDER, + buildFieldReference, + buildTotpReference, + PASSWORD_FIELD_TITLE, + TOTP_FIELD_TITLE, + USERNAME_FIELD_TITLE, +} from './onepassword-credential-item'; + +const RESERVED_FIELD_TITLES = new Set([ + USERNAME_FIELD_TITLE, + PASSWORD_FIELD_TITLE, + TOTP_FIELD_TITLE, +]); + +function parseItemRef(itemRef: string): { vaultId?: string; itemId?: string } { + const [vaultId, itemId] = itemRef.replace(/^op:\/\//, '').split('/'); + return { vaultId, itemId }; +} + +/** + * Resolves a browser auth profile's stored login (username, password, live TOTP) + * from 1Password at run time using its `op:///` reference. Secrets + * are fetched just-in-time and never persisted or logged. + */ +export class OnePasswordCredentialVaultAdapter implements BrowserCredentialVaultAdapter { + private readonly logger = new Logger(OnePasswordCredentialVaultAdapter.name); + + async resolveCredentialReference(params: { + profileId: string; + provider?: string | null; + externalItemRef?: string | null; + connectionId?: string | null; + }): Promise { + if (params.provider !== ONEPASSWORD_PROVIDER) return null; + + const itemRef = params.externalItemRef?.trim(); + if (!itemRef) return null; + + const client = await getOnePasswordClient(); + + const [username, password] = await Promise.all([ + this.resolveField( + client, + buildFieldReference(itemRef, USERNAME_FIELD_TITLE), + ), + this.resolveField( + client, + buildFieldReference(itemRef, PASSWORD_FIELD_TITLE), + ), + ]); + // Resolve the OTP separately so its rotation window is as fresh as possible. + const totpCode = await this.resolveField( + client, + buildTotpReference(itemRef), + ); + const extraFields = await this.resolveExtraFields(client, itemRef); + + if (!username && !password && !totpCode && extraFields.length === 0) { + return null; + } + + return { + username: username ?? undefined, + password: password ?? undefined, + totpCode: totpCode ?? undefined, + extraFields: extraFields.length > 0 ? extraFields : undefined, + }; + } + + // Reads any site-specific custom fields (workspace, subdomain, …) the customer + // added. Best-effort: a vault without custom fields (or an older SDK) just + // yields none, and never blocks the standard username/password/TOTP path. + private async resolveExtraFields( + client: OnePasswordClient, + itemRef: string, + ): Promise<{ label: string; value: string }[]> { + try { + const { vaultId, itemId } = parseItemRef(itemRef); + if (!vaultId || !itemId) return []; + const item = await client.items.get(vaultId, itemId); + return item.fields + .filter( + (field) => + !RESERVED_FIELD_TITLES.has(field.title) && + Boolean(field.value?.trim()), + ) + .map((field) => ({ label: field.title, value: field.value })); + } catch { + return []; + } + } + + private async resolveField( + client: OnePasswordClient, + reference: string, + ): Promise { + try { + const value = await client.secrets.resolve(reference); + return value?.trim() ? value : null; + } catch { + // A missing optional field (e.g. a login with no TOTP configured) resolves + // as an error; treat it as absent rather than failing the whole sign-in. + this.logger.debug(`1Password reference not resolvable: ${reference}`); + return null; + } + } +} diff --git a/apps/api/src/browserbase/select-live-view-url.spec.ts b/apps/api/src/browserbase/select-live-view-url.spec.ts new file mode 100644 index 0000000000..75cab0e7c5 --- /dev/null +++ b/apps/api/src/browserbase/select-live-view-url.spec.ts @@ -0,0 +1,34 @@ +import { selectLiveViewUrl } from './browserbase-session.service'; + +const page = (url: string, tag: string) => ({ + url, + debuggerFullscreenUrl: `live://${tag}`, +}); + +describe('selectLiveViewUrl', () => { + const home = page('https://aws.amazon.com/', 'home'); + const signin = page('https://us-east-2.signin.aws.amazon.com/oauth', 'signin'); + const debug = { debuggerFullscreenUrl: 'live://session', pages: [home, signin] }; + + it('follows the tab matching the AI page by exact URL', () => { + expect(selectLiveViewUrl(debug, 'https://us-east-2.signin.aws.amazon.com/oauth')).toBe( + 'live://signin', + ); + }); + + it('matches by hostname when the exact URL has drifted (redirects)', () => { + expect( + selectLiveViewUrl(debug, 'https://us-east-2.signin.aws.amazon.com/console?x=1'), + ).toBe('live://signin'); + }); + + it('falls back to the newest tab when nothing matches', () => { + expect(selectLiveViewUrl(debug, 'https://unrelated.example.com/')).toBe('live://signin'); + }); + + it('uses the session-level view when there are no pages', () => { + expect(selectLiveViewUrl({ debuggerFullscreenUrl: 'live://session', pages: [] })).toBe( + 'live://session', + ); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/analyze-vendor-login.ts b/apps/api/src/trigger/browser-automation/analyze-vendor-login.ts new file mode 100644 index 0000000000..fd47c1737c --- /dev/null +++ b/apps/api/src/trigger/browser-automation/analyze-vendor-login.ts @@ -0,0 +1,24 @@ +import { task } from '@trigger.dev/sdk'; +import { BrowserLoginAnalyzerService } from '../../browserbase/browser-login-analyzer.service'; +import type { LoginAnalysis } from '../../browserbase/browser-login-analysis'; + +const analyzer = new BrowserLoginAnalyzerService(); + +/** + * Runs the vendor login analysis (open a cloud browser, navigate to the sign-in + * page, and detect the login methods) in the background. Kept off the HTTP + * request path because the browser + AI work can outlast request/browser + * timeouts. The connect flow subscribes to the run for the result. + */ +export const analyzeVendorLogin = task({ + id: 'analyze-vendor-login', + // Browser open + AI navigate + AI extract; generous headroom, still well under + // the Browserbase session lifetime. + maxDuration: 180, + // Analysis already degrades to a manual fallback on failure, so a blind retry + // just wastes a browser session. + retry: { maxAttempts: 1 }, + run: async (payload: { url: string }): Promise => { + return analyzer.analyzeLogin(payload.url); + }, +}); diff --git a/apps/api/src/trigger/browser-automation/execute-automation-live.ts b/apps/api/src/trigger/browser-automation/execute-automation-live.ts new file mode 100644 index 0000000000..dd08c7c8e2 --- /dev/null +++ b/apps/api/src/trigger/browser-automation/execute-automation-live.ts @@ -0,0 +1,52 @@ +import { metadata, task } from '@trigger.dev/sdk'; +import { BrowserbaseService } from '../../browserbase/browserbase.service'; + +const browserbaseService = new BrowserbaseService(); + +/** + * Runs a saved automation on an already-started live session (from start-live) + * so the user can watch the AI work AND see its steps — the same live timeline + * the Test/connect flow shows. Kept off the HTTP request path because the + * browser + AI work can outlast request timeouts; the Run view subscribes to + * this run for live steps (`runSteps`) and the final result. + */ +export const executeAutomationLive = task({ + id: 'execute-automation-live', + // Matches the scheduled run + test caps: a multi-step task on a complex site + // can take a while; give it a safe budget instead of being cut off mid-run. + maxDuration: 60 * 10, + // A browser + AI run isn't safe to blindly retry (the session may be gone). + retry: { maxAttempts: 1 }, + run: async (payload: { + automationId: string; + runId: string; + sessionId: string; + organizationId: string; + }) => { + // Runs EVERY step (all vendors), not just the first — step 0 on the live + // session, later vendors in their own sessions. + return browserbaseService.executeAutomationLive( + payload.automationId, + payload.runId, + payload.sessionId, + payload.organizationId, + // Live activity timeline — surfaced to the Run view via realtime metadata. + // Steps are plain JSON; cast to the SDK's value type (named interfaces lack + // the index signature DeserializedJson structurally requires). + (steps) => + metadata.set( + 'runSteps', + steps as unknown as Parameters[1], + ), + // Each vendor's live view, so the Run iframe follows the run across vendors. + // A fresh view means we're live again — clear any transition state. + (liveViewUrl) => { + metadata.set('liveViewUrl', liveViewUrl); + metadata.set('livePhase', 'running'); + }, + // Live-view phase: 'switching' between vendors, 'finishing' at the end, so + // the Run view covers the (soon-to-disconnect) iframe with a calm overlay. + (livePhase) => metadata.set('livePhase', livePhase), + ); + }, +}); diff --git a/apps/api/src/trigger/browser-automation/sign-in-vendor-profile.ts b/apps/api/src/trigger/browser-automation/sign-in-vendor-profile.ts new file mode 100644 index 0000000000..8ee0f497fc --- /dev/null +++ b/apps/api/src/trigger/browser-automation/sign-in-vendor-profile.ts @@ -0,0 +1,46 @@ +import { metadata, task } from '@trigger.dev/sdk'; +import { + BrowserCredentialSigninService, + type AutoSignInResult, +} from '../../browserbase/browser-credential-signin.service'; + +const signin = new BrowserCredentialSigninService(); + +/** + * Runs the connect flow's first automated sign-in in the background: opens a + * cloud browser, fills the credentials the user just stored, and verifies the + * session. Kept off the HTTP request path because the browser + AI work can + * outlast request/browser timeouts. The connect flow subscribes to the run and, + * if it can't complete unattended, hands the live browser to the user. + */ +export const signInVendorProfile = task({ + id: 'sign-in-vendor-profile', + maxDuration: 240, + // A browser + AI sign-in isn't safe to blindly retry (it would re-enter + // credentials on a session that may already be gone), so run it once. + retry: { maxAttempts: 1 }, + run: async (payload: { + organizationId: string; + profileId: string; + url: string; + sessionId: string; + mode?: 'password' | 'sso'; + /** Vendor's identifier-field label (e.g. "IAM username") for a truthful step. */ + usernameLabel?: string; + }): Promise => { + return signin.signInWithStoredCredentials({ + ...payload, + // Live activity timeline — surfaced to the connect flow via realtime metadata. + // Steps are plain JSON; cast to the SDK's value type (named interfaces lack + // the index signature DeserializedJson structurally requires). + onSteps: (steps) => + metadata.set( + 'signinSteps', + steps as unknown as Parameters[1], + ), + // The tab the AI is on — so the connect flow's iframe follows it across + // new tabs (e.g. AWS opening its sign-in in a new tab). + onLiveView: (liveViewUrl) => metadata.set('signinLiveViewUrl', liveViewUrl), + }); + }, +}); diff --git a/apps/api/src/trigger/browser-automation/test-vendor-instruction.ts b/apps/api/src/trigger/browser-automation/test-vendor-instruction.ts new file mode 100644 index 0000000000..65b0489ac4 --- /dev/null +++ b/apps/api/src/trigger/browser-automation/test-vendor-instruction.ts @@ -0,0 +1,48 @@ +import { metadata, task } from '@trigger.dev/sdk'; +import { + BrowserInstructionTestService, + type InstructionTestResult, +} from '../../browserbase/browser-instruction-test.service'; + +const tester = new BrowserInstructionTestService(); + +/** + * Runs a not-yet-saved instruction against the connection's live session so the + * user can watch it work before committing it to the schedule. Kept off the HTTP + * request path because the browser + AI work can outlast request timeouts; the + * composer subscribes to the run for live steps and the final result. + */ +export const testVendorInstruction = task({ + id: 'test-vendor-instruction', + // Sign-in can take ~40s before navigation even starts, so give the run a + // generous, safe budget to finish a multi-step task on a complex site instead + // of being cut off mid-way. Matches the scheduled run's 10-minute cap. + maxDuration: 60 * 10, + // A browser + AI run isn't safe to blindly retry (the session may be gone), + // so run it once. + retry: { maxAttempts: 1 }, + run: async (payload: { + organizationId: string; + taskId?: string; + profileId?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + sessionId: string; + }): Promise => { + return tester.testInstructionOnSession({ + ...payload, + // Live activity timeline — surfaced to the composer via realtime metadata. + // Steps are plain JSON; cast to the SDK's value type (named interfaces lack + // the index signature DeserializedJson structurally requires). + onSteps: (steps) => + metadata.set( + 'testSteps', + steps as unknown as Parameters[1], + ), + // Follow the agent across tabs so the composer's live view tracks the + // page it's actually working on, not the stale starting tab. + onLiveView: (liveViewUrl) => metadata.set('testLiveViewUrl', liveViewUrl), + }); + }, +}); diff --git a/apps/api/trigger.config.ts b/apps/api/trigger.config.ts index 44f4e88364..1a54130e05 100644 --- a/apps/api/trigger.config.ts +++ b/apps/api/trigger.config.ts @@ -10,6 +10,9 @@ export default defineConfig({ logLevel: 'log', maxDuration: 300, // 5 minutes build: { + // The 1Password SDK ships a native WASM core; keep it external so it's + // resolved from node_modules at runtime instead of being bundled by esbuild. + external: ['@1password/sdk'], extensions: [ caBundleExtension(), prismaExtension({ diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx index be1e1bdca1..2b4b098625 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx @@ -1,106 +1,123 @@ -import { render, screen } from '@testing-library/react'; -import type { - ButtonHTMLAttributes, - HTMLAttributes, - InputHTMLAttributes, - LabelHTMLAttributes, - ReactNode, -} from 'react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; import { - setMockPermissions, ADMIN_PERMISSIONS, AUDITOR_PERMISSIONS, mockHasPermission, + setMockPermissions, } from '@/test-utils/mocks/permissions'; +import { fireEvent, render, screen } from '@testing-library/react'; +import type { ButtonHTMLAttributes, InputHTMLAttributes, ReactNode } from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Connection } from './connection-format'; vi.mock('@/hooks/use-permissions', () => ({ - usePermissions: () => ({ - permissions: {}, - hasPermission: mockHasPermission, - }), + usePermissions: () => ({ permissions: {}, hasPermission: mockHasPermission }), })); vi.mock('@/lib/api-client', () => ({ + // Never resolves, so the passed initialProfiles stay in state for the assertion. apiClient: { get: vi.fn(() => new Promise(() => undefined)), post: vi.fn().mockResolvedValue({ data: {} }), + patch: vi.fn().mockResolvedValue({ data: {} }), + delete: vi.fn().mockResolvedValue({ data: {} }), }, })); vi.mock('@trycompai/design-system', () => ({ - Badge: ({ children }: { children?: ReactNode }) => {children}, Button: ({ children, iconLeft, - iconRight, - loading, ...props - }: ButtonHTMLAttributes & { - iconLeft?: ReactNode; - iconRight?: ReactNode; - loading?: boolean; - }) => ( - ), - Card: ({ children }: HTMLAttributes) =>
{children}
, - CardContent: ({ children }: HTMLAttributes) =>
{children}
, - CardDescription: ({ children }: HTMLAttributes) =>

{children}

, - CardHeader: ({ children }: HTMLAttributes) =>
{children}
, - CardTitle: ({ children }: HTMLAttributes) =>

{children}

, Input: (props: InputHTMLAttributes) => , - Label: ({ children, ...props }: LabelHTMLAttributes) => ( - + Spinner: () => , + Section: ({ + title, + description, + actions, + children, + }: { + title?: ReactNode; + description?: ReactNode; + actions?: ReactNode; + children?: ReactNode; + }) => ( +
+ {title &&

{title}

} + {description &&

{description}

} + {actions} + {children} +
), - Spinner: () => , })); vi.mock('@trycompai/design-system/icons', () => ({ - Globe: () => , - Renew: () => , - Screen: () => , + Add: () => , })); +// Sub-components are covered on their own; stub them to focus on the client. +vi.mock('./ConnectionsTable', () => ({ + ConnectionsTable: ({ connections }: { connections: Connection[] }) => ( +
{connections.length} rows
+ ), +})); +vi.mock('./ManageConnectionSheet', () => ({ ManageConnectionSheet: () => null })); +// The shared connect flow is covered by the task suite; stub it so we only test +// that the client renders it (and doesn't pull the whole task subsystem). +vi.mock( + '@/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/ConnectVendorLoginFlow', + () => ({ ConnectVendorLoginFlow: () =>
}), +); + import { BrowserConnectionClient } from './BrowserConnectionClient'; +const profile: Connection = { + id: 'bap_1', + hostname: 'github.com', + loginIdentity: 'ci-bot@acme.com', + displayName: 'GitHub', + status: 'verified', + vaultExternalItemRef: 'op://vault/item', +}; + describe('BrowserConnectionClient permission gating', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); + beforeEach(() => vi.clearAllMocks()); - it('shows Connect Browser button when user has integration:create permission', () => { + it('shows "Connect a vendor" and the table when the user can create integrations', () => { setMockPermissions(ADMIN_PERMISSIONS); - render(); + render(); - const connectButton = screen.getByRole('button', { name: /connect browser/i }); - expect(connectButton).toBeInTheDocument(); - expect(connectButton).not.toBeDisabled(); + expect(screen.getByRole('button', { name: /connect a vendor/i })).toBeInTheDocument(); + expect(screen.getByTestId('table')).toHaveTextContent('1 rows'); }); - it('hides Connect Browser button when user lacks integration:create permission', () => { + it('hides "Connect a vendor" for a read-only user but still lists connections', () => { setMockPermissions(AUDITOR_PERMISSIONS); - render(); + render(); - expect(screen.queryByRole('button', { name: /connect browser/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /open browser/i })).not.toBeInTheDocument(); + expect(screen.queryByRole('button', { name: /connect a vendor/i })).not.toBeInTheDocument(); + expect(screen.getByTestId('table')).toBeInTheDocument(); }); - it('hides Connect Browser button when user has no permissions', () => { - setMockPermissions({}); - render(); + it('shows an empty state when there are no connections', () => { + setMockPermissions(ADMIN_PERMISSIONS); + render(); - expect(screen.queryByRole('button', { name: /connect browser/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /open browser/i })).not.toBeInTheDocument(); + expect(screen.getByText(/no connections yet/i)).toBeInTheDocument(); + expect(screen.queryByTestId('table')).not.toBeInTheDocument(); }); - it('still shows the URL input regardless of permissions', () => { - setMockPermissions({}); - render(); + it('opens the shared connect flow when "Connect a vendor" is clicked', () => { + setMockPermissions(ADMIN_PERMISSIONS); + render(); - expect(screen.getByLabelText('Website URL')).toBeInTheDocument(); + expect(screen.queryByTestId('connect-flow')).not.toBeInTheDocument(); + fireEvent.click(screen.getAllByRole('button', { name: /connect a vendor/i })[0]); + expect(screen.getByTestId('connect-flow')).toBeInTheDocument(); }); }); diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx index 5064647673..6786182efd 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx @@ -1,284 +1,275 @@ 'use client'; +// Reuse the task flow's proven connect/reconnect experience here so org-level +// connections use the same reliable path (method detection + automated +// credential entry + a working live takeover) instead of a bespoke, flaky one. +import { ConnectVendorLoginFlow } from '@/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/ConnectVendorLoginFlow'; +import { clearConnectState } from '@/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/connect-flow-storage'; import { usePermissions } from '@/hooks/use-permissions'; import { apiClient } from '@/lib/api-client'; -import { - Badge, - Button, - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, - Input, - Label, - Spinner, -} from '@trycompai/design-system'; -import { Globe, Screen } from '@trycompai/design-system/icons'; +import { Button, Section } from '@trycompai/design-system'; +import { Add } from '@trycompai/design-system/icons'; import { useCallback, useEffect, useState } from 'react'; -import { BrowserConnectionInstructions } from './BrowserConnectionInstructions'; -import { BrowserConnectionLiveView } from './BrowserConnectionLiveView'; -import { - BrowserConnectionProfileList, - type BrowserConnectionProfile, -} from './BrowserConnectionProfileList'; +import { toast } from 'sonner'; +import { methodOf, type Connection } from './connection-format'; +import { ConnectionsTable } from './ConnectionsTable'; +import { ManageConnectionSheet } from './ManageConnectionSheet'; -interface ResolveProfileResponse { - profile: BrowserConnectionProfile & { contextId: string }; - isNew: boolean; -} - -interface SessionResponse { - sessionId: string; - liveViewUrl: string; +interface BrowserConnectionClientProps { + organizationId: string; + initialProfiles?: Connection[]; } -interface AuthStatusResponse { - isLoggedIn: boolean; - username?: string; -} +/** The connect/reconnect flow the page is currently showing (full-screen). */ +type ActiveFlow = { kind: 'connect' } | { kind: 'reconnect'; connection: Connection }; -interface VerifyProfileResponse { - profile: BrowserConnectionProfile; - auth: AuthStatusResponse; -} +// Stable resume-state key for the shared connect flow on this page. Cleared each +// time the flow opens so "Connect a vendor" always starts fresh (a settings page +// has no "resume where I left off" — that would trap the user on a stale step). +const CONNECT_FLOW_KEY = 'org-connections'; -type Status = 'idle' | 'loading' | 'session-active' | 'checking'; +/** + * Org-level browser connections manager. Lists every vendor login the org has, + * with health, and wires the shared connect flow for connect / reconnect plus + * the profile API for rename / change-login / remove. + */ +export function BrowserConnectionClient({ + organizationId: _organizationId, + initialProfiles = [], +}: BrowserConnectionClientProps) { + const { hasPermission } = usePermissions(); + const canConnect = hasPermission('integration', 'create'); + const canUpdate = hasPermission('integration', 'update'); + const canDelete = hasPermission('integration', 'delete'); -interface BrowserConnectionClientProps { - organizationId: string; -} + const [profiles, setProfiles] = useState(initialProfiles); + const [flow, setFlow] = useState(null); + const [busy, setBusy] = useState(false); -export function BrowserConnectionClient({ organizationId }: BrowserConnectionClientProps) { - const { hasPermission } = usePermissions(); - const canManageBrowser = hasPermission('integration', 'create'); - const [status, setStatus] = useState('idle'); - const [profileId, setProfileId] = useState(null); - const [profiles, setProfiles] = useState([]); - const [sessionId, setSessionId] = useState(null); - const [liveViewUrl, setLiveViewUrl] = useState(null); - const [urlToCheck, setUrlToCheck] = useState('https://github.com'); - const [authStatus, setAuthStatus] = useState(null); - const [error, setError] = useState(null); - const hasContext = profiles.some((profile) => profile.status === 'verified'); + const [manageConnection, setManageConnection] = useState(null); + const [manageOpen, setManageOpen] = useState(false); - const checkContextStatus = useCallback(async () => { - try { - const res = await apiClient.get('/v1/browserbase/profiles'); - if (res.data) { - setProfiles(res.data); - const verifiedProfile = res.data.find((profile) => profile.status === 'verified'); - const firstProfile = verifiedProfile ?? res.data[0]; - setProfileId(firstProfile?.id ?? null); - } - } catch { - // Ignore - } + const fetchProfiles = useCallback(async () => { + const res = await apiClient.get('/v1/browserbase/profiles'); + setProfiles(Array.isArray(res.data) ? res.data : []); }, []); useEffect(() => { - checkContextStatus(); - }, [checkContextStatus]); - - const handleStartSession = async () => { - let startedSessionId: string | null = null; - try { - setError(null); - setStatus('loading'); + void fetchProfiles(); + }, [fetchProfiles]); - const profileRes = await apiClient.post( - '/v1/browserbase/profiles/resolve', - { url: urlToCheck }, - ); - if (profileRes.error || !profileRes.data) { - throw new Error(profileRes.error || 'Failed to create auth profile'); - } - setProfileId(profileRes.data.profile.id); - setProfiles((currentProfiles) => { - const rest = currentProfiles.filter( - (profile) => profile.id !== profileRes.data?.profile.id, - ); - return profileRes.data ? [profileRes.data.profile, ...rest] : currentProfiles; - }); + // After the shared flow connects or reconnects, refresh the list and return. + const handleFlowDone = useCallback( + (message: string) => { + void fetchProfiles(); + setFlow(null); + toast.success(message); + }, + [fetchProfiles], + ); - const sessionRes = await apiClient.post( - `/v1/browserbase/profiles/${profileRes.data.profile.id}/session`, - {}, - ); - if (sessionRes.error || !sessionRes.data) { - throw new Error(sessionRes.error || 'Failed to create session'); - } - startedSessionId = sessionRes.data.sessionId; - setSessionId(startedSessionId); - setLiveViewUrl(sessionRes.data.liveViewUrl); + // Always start the shared flow from a clean slate on this page. + const openConnect = useCallback(() => { + clearConnectState(CONNECT_FLOW_KEY); + setFlow({ kind: 'connect' }); + }, []); - // Navigate to the URL - await apiClient.post('/v1/browserbase/navigate', { - sessionId: startedSessionId, - url: urlToCheck, - }); + const handleReconnect = useCallback((connection: Connection) => { + setManageOpen(false); + clearConnectState(CONNECT_FLOW_KEY); + setFlow({ kind: 'reconnect', connection }); + }, []); - setStatus('session-active'); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to start session'); - setSessionId(null); - setLiveViewUrl(null); - setStatus('idle'); + const handleManage = useCallback((connection: Connection) => { + setManageConnection(connection); + setManageOpen(true); + }, []); - // If we created a session but navigation failed, close it to avoid orphaned sessions - if (startedSessionId) { - try { - await apiClient.post('/v1/browserbase/session/close', { sessionId: startedSessionId }); - } catch { - // Ignore cleanup errors (don't mask original error) - } + const handleRename = useCallback( + async (connection: Connection, name: string) => { + setBusy(true); + try { + await apiClient.patch(`/v1/browserbase/profiles/${connection.id}`, { + displayName: name, + }); + await fetchProfiles(); + setManageOpen(false); + toast.success('Connection renamed.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not rename.'); + } finally { + setBusy(false); } - } - }; - - const handleCheckAuth = async () => { - if (!sessionId || !profileId) return; - - try { - setError(null); - setStatus('checking'); + }, + [fetchProfiles], + ); - const res = await apiClient.post( - `/v1/browserbase/profiles/${profileId}/verify`, - { sessionId, url: urlToCheck }, - ); - if (res.error || !res.data) { - throw new Error(res.error || 'Failed to check auth'); + const handleChangeLogin = useCallback( + async (connection: Connection, creds: { username: string; password: string }) => { + setBusy(true); + try { + await apiClient.post(`/v1/browserbase/profiles/${connection.id}/credentials`, creds); + await fetchProfiles(); + setManageOpen(false); + toast.success('Login updated. Reconnect to verify it works.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not update the login.'); + } finally { + setBusy(false); } + }, + [fetchProfiles], + ); - setAuthStatus(res.data.auth); - setProfiles((currentProfiles) => { - const rest = currentProfiles.filter((profile) => profile.id !== res.data?.profile.id); - return res.data ? [res.data.profile, ...rest] : currentProfiles; - }); + const handleSetTotp = useCallback(async (connection: Connection, totpSeed: string) => { + setBusy(true); + try { + await apiClient.post(`/v1/browserbase/profiles/${connection.id}/totp`, { totpSeed }); + toast.success('Automatic 2FA is on. Scheduled runs generate the code for you.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not save the authenticator key.'); + } finally { + setBusy(false); + } + }, []); - // Close the session after checking - await apiClient.post('/v1/browserbase/session/close', { sessionId }); - setSessionId(null); - setLiveViewUrl(null); - setProfileId(null); - setStatus('idle'); + const handleClearTotp = useCallback(async (connection: Connection) => { + setBusy(true); + try { + await apiClient.delete(`/v1/browserbase/profiles/${connection.id}/totp`); + toast.success('Automatic 2FA turned off.'); } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to check auth'); - setStatus('session-active'); + toast.error(err instanceof Error ? err.message : 'Could not turn off automatic 2FA.'); + } finally { + setBusy(false); } - }; + }, []); - const handleCloseSession = async () => { - if (sessionId) { + const handleRemove = useCallback( + async (connection: Connection) => { + setBusy(true); try { - await apiClient.post('/v1/browserbase/session/close', { sessionId }); - } catch { - // Ignore + await apiClient.delete(`/v1/browserbase/profiles/${connection.id}`); + await fetchProfiles(); + setManageOpen(false); + toast.success('Connection removed.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not remove.'); + } finally { + setBusy(false); } - } - setSessionId(null); - setLiveViewUrl(null); - setProfileId(null); - setStatus('idle'); - }; + }, + [fetchProfiles], + ); + + // Connect / reconnect take over the whole panel via the shared flow — the same + // one the evidence task uses, so behaviour is identical and reliable. + if (flow) { + const reconnect = + flow.kind === 'reconnect' + ? { + url: flow.connection.lastAuthCheckUrl || `https://${flow.connection.hostname}`, + mode: methodOf(flow.connection), + } + : undefined; + return ( + handleFlowDone('Connection added.')} + onReconnected={() => handleFlowDone('Connection reconnected.')} + onCancel={() => setFlow(null)} + /> + ); + } return ( -
- {/* Status Card */} - - -
-
-
- -
-
- Browser Session - - {hasContext - ? 'At least one browser auth profile is verified' - : 'No verified browser auth profile yet'} - -
-
- - {hasContext ? 'Connected' : 'Not Connected'} - +
}> + Connect a vendor + + ) : undefined + } + > +
+ {/* Orient anyone who lands here from Settings without knowing the feature: + what it does, that it's unattended, and where the automations live. */} +
+
+ How it works
- - - {error &&

{error}

} +
    +
  1. + 1. + + Connect a vendor login here — Comp AI signs in for you, including any + two-factor codes. + +
    2. +
  2. + 2. + + On a schedule it signs in, screenshots the required page as audit evidence, + and re-signs in on its own when a session expires. + +
    3. +
  3. + 3. + + You add the automations that use these logins inside an{' '} + evidence task, in its + “Browser evidence” section. + +
    4. +
+
- {status === 'idle' && ( -
-
- -
-
- setUrlToCheck(e.target.value)} - /> -
- {canManageBrowser && ( - - )} -
-

- Open a browser session to authenticate with websites. Your login session will be - saved and used for browser automations. -

+ {profiles.length === 0 ? ( +
+
+
No connections yet
+

+ Connect a vendor login so Comp AI can sign in and capture evidence for your + browser automations. +

+ {canConnect && ( +
+
- - {authStatus && ( -
-
-
- - {authStatus.isLoggedIn - ? `Logged in${authStatus.username ? ` as ${authStatus.username}` : ''}` - : 'Not logged in'} - -
-
- )} -
- )} - - {status === 'loading' && ( -
- - Starting browser session... -
- )} - - - - {(status === 'session-active' || status === 'checking') && liveViewUrl && ( - +
+ ) : ( + )} - - -
+ +
+
); } diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx deleted file mode 100644 index fa9e4fbed4..0000000000 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx +++ /dev/null @@ -1,33 +0,0 @@ -'use client'; - -import { Card, CardContent, CardHeader, CardTitle } from '@trycompai/design-system'; - -export function BrowserConnectionInstructions() { - return ( - - - How it works - - -
    -
  1. - Create a service account - Use a - dedicated account for browser automations when possible. -
    2. -
  2. - Authenticate per site - Each hostname gets - its own auth profile and Browserbase context. -
    3. -
  3. - Complete 2FA manually - Live View supports - manual approvals; Comp does not store passwords or TOTP secrets. -
    4. -
  4. - Reconnect when needed - Expired or blocked - profiles are shown above and skipped by scheduled runs. -
    5. -
- - - ); -} diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx deleted file mode 100644 index 43ffa900d4..0000000000 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx +++ /dev/null @@ -1,65 +0,0 @@ -'use client'; - -import { - Button, - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from '@trycompai/design-system'; -import { Renew } from '@trycompai/design-system/icons'; - -export function BrowserConnectionLiveView({ - liveViewUrl, - isChecking, - canManageBrowser, - onCheckAuth, - onClose, -}: { - liveViewUrl: string; - isChecking: boolean; - canManageBrowser: boolean; - onCheckAuth: () => void; - onClose: () => void; -}) { - return ( - - -
-
- Browser Session - - Log in to websites below. Your session will be saved for automations. - -
-
- - -
-
- - -
-