Skip to content

Commit 21edbde

Browse files
author
stlc-bot
committed
feat(scrape): add product extraction format to POST /web/scrape (#1212)
Stainless-Generated-From: e801ba5
1 parent 9a81729 commit 21edbde

2 files changed

Lines changed: 188 additions & 12 deletions

File tree

‎src/resources/web.ts‎

Lines changed: 186 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -86,16 +86,18 @@ export class Web extends APIResource {
8686
/**
8787
* Reuse cached outputs independently and capture missing formats in one page
8888
* visit. Each cache key includes only the settings that affect that output. HTML
89-
* is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached
90-
* outputs can come from different visits within maxAgeMs; use 0 for a fresh
91-
* capture. HTML-only requests use the existing fast acquisition path. Highlights
92-
* return the plain-text passages most relevant to highlightsParams.query. One
93-
* credit per request, including cache hits and missing pages, or two with browser
94-
* actions; highlights add 3 credits when passages are returned; JSON extraction
95-
* adds four credits and runs an LLM over the page Markdown on every request that
96-
* has text to extract; PDF OCR adds one credit per recovered page on fresh
97-
* extraction. Original response bytes and screenshots are limited to 20 MiB each,
98-
* screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
89+
* is shared with Markdown, parsed fields, product data, highlights, and JSON
90+
* extraction. Cached outputs can come from different visits within maxAgeMs; use 0
91+
* for a fresh capture. HTML-only requests use the existing fast acquisition path.
92+
* Highlights return the plain-text passages most relevant to
93+
* highlightsParams.query. One credit per request, including cache hits and missing
94+
* pages, or two with browser actions; highlights add 3 credits when passages are
95+
* returned; JSON extraction adds four credits and runs an LLM over the page
96+
* Markdown on every request that has text to extract; PDF OCR adds one credit per
97+
* recovered page on fresh extraction; the product output adds one credit, plus six
98+
* more when the specialized model is used. Original response bytes and screenshots
99+
* are limited to 20 MiB each, screenshots to 40 megapixels, and the combined
100+
* browser capture to 60 MiB.
99101
*
100102
* @example
101103
* ```ts
@@ -983,6 +985,11 @@ export interface WebScrapeResponse {
983985
*/
984986
parsed: WebScrapeResponse.Parsed;
985987

988+
/**
989+
* Product detail page classification and the extracted product.
990+
*/
991+
product: WebScrapeResponse.Product;
992+
986993
/**
987994
* Unique id of this API call, also sent in the X-Request-Id response header. Quote
988995
* it when contacting support about a failed request.
@@ -1000,8 +1007,10 @@ export interface WebScrapeResponse {
10001007
url: string;
10011008

10021009
/**
1003-
* Present when return-partial captures a page that is still loading or returns
1004-
* images before image processing finishes. Partial responses are not cached.
1010+
* Present when return-partial captures a page that is still loading, returns
1011+
* images before image processing finishes, or cuts product AI extraction short.
1012+
* Also present if the optional product AI fallback fails. Partial responses are
1013+
* not cached.
10051014
*/
10061015
isPartial?: true;
10071016

@@ -1277,6 +1286,148 @@ export namespace WebScrapeResponse {
12771286
requested: boolean;
12781287
}
12791288

1289+
/**
1290+
* Product detail page classification and the extracted product.
1291+
*/
1292+
export interface Product {
1293+
data: Product.Data | null;
1294+
1295+
requested: boolean;
1296+
}
1297+
1298+
export namespace Product {
1299+
export interface Data {
1300+
/**
1301+
* Whether the page is a product detail page.
1302+
*/
1303+
isProductPage: boolean;
1304+
1305+
/**
1306+
* The extracted product, or null when the page is not a product detail page.
1307+
*/
1308+
product: Data.Product | null;
1309+
}
1310+
1311+
export namespace Data {
1312+
/**
1313+
* The extracted product, or null when the page is not a product detail page.
1314+
*/
1315+
export interface Product {
1316+
/**
1317+
* Stock or ordering availability.
1318+
*/
1319+
availability:
1320+
| 'in_stock'
1321+
| 'out_of_stock'
1322+
| 'limited_availability'
1323+
| 'preorder'
1324+
| 'backorder'
1325+
| 'made_to_order'
1326+
| 'discontinued'
1327+
| null;
1328+
1329+
/**
1330+
* Brand or vendor.
1331+
*/
1332+
brand: string | null;
1333+
1334+
/**
1335+
* Product category.
1336+
*/
1337+
category: string | null;
1338+
1339+
/**
1340+
* ISO 4217 currency code.
1341+
*/
1342+
currency: string | null;
1343+
1344+
/**
1345+
* Product description.
1346+
*/
1347+
description: string | null;
1348+
1349+
/**
1350+
* Product dimensions as shown on the page.
1351+
*/
1352+
dimensions: Array<string>;
1353+
1354+
/**
1355+
* Key features and specifications.
1356+
*/
1357+
features: Array<string>;
1358+
1359+
/**
1360+
* Product image URLs, main image first.
1361+
*/
1362+
images: Array<string>;
1363+
1364+
/**
1365+
* Main product image URL.
1366+
*/
1367+
imageUrl: string | null;
1368+
1369+
/**
1370+
* Product name.
1371+
*/
1372+
name: string;
1373+
1374+
/**
1375+
* Current price.
1376+
*/
1377+
price: number | null;
1378+
1379+
/**
1380+
* List price before any discount.
1381+
*/
1382+
regularPrice: number | null;
1383+
1384+
/**
1385+
* Product identifier such as a SKU or model number.
1386+
*/
1387+
sku: string | null;
1388+
1389+
/**
1390+
* Product tags.
1391+
*/
1392+
tags: Array<string>;
1393+
1394+
/**
1395+
* Intended audience.
1396+
*/
1397+
targetAudience: Array<string>;
1398+
1399+
/**
1400+
* Product variations, such as different colors or sizes, with their attributes and
1401+
* images. Empty if none are found. May not include every variation offered by the
1402+
* store.
1403+
*/
1404+
variants: Array<Product.Variant>;
1405+
}
1406+
1407+
export namespace Product {
1408+
export interface Variant {
1409+
/**
1410+
* Explicit variant attributes such as color, size, material, pattern and
1411+
* properties declared by page.
1412+
*/
1413+
attributes: { [key: string]: string };
1414+
1415+
/**
1416+
* Original source image URLs explicitly attached to this variant.
1417+
*/
1418+
images: Array<string>;
1419+
1420+
sku: string | null;
1421+
1422+
/**
1423+
* Variant or offer URL when provided by the source. May be shared by variants.
1424+
*/
1425+
url: string | null;
1426+
}
1427+
}
1428+
}
1429+
}
1430+
12801431
/**
12811432
* An image data URL. Use directly as an image src.
12821433
*/
@@ -2124,6 +2275,11 @@ export interface WebScrapeParams {
21242275
*/
21252276
parseParams?: WebScrapeParams.ParseParams;
21262277

2278+
/**
2279+
* Product options. Requires formats.product: true.
2280+
*/
2281+
productParams?: WebScrapeParams.ProductParams;
2282+
21272283
/**
21282284
* Screenshot options. Requires formats.screenshot: true.
21292285
*/
@@ -2204,6 +2360,11 @@ export namespace WebScrapeParams {
22042360
*/
22052361
parse?: boolean;
22062362

2363+
/**
2364+
* Structured product data for product detail pages. Adds one credit.
2365+
*/
2366+
product?: boolean;
2367+
22072368
/**
22082369
* An inline image of the page.
22092370
*/
@@ -2297,6 +2458,19 @@ export namespace WebScrapeParams {
22972458
}
22982459
}
22992460

2461+
/**
2462+
* Product options. Requires formats.product: true.
2463+
*/
2464+
export interface ProductParams {
2465+
/**
2466+
* Extract the product with a specialized model when the page has no structured
2467+
* product data. Adds six credits when the model returns a verdict. If the fallback
2468+
* fails, returns a partial response with the deterministic result and no fallback
2469+
* charge. Request deadlines and client disconnects still apply.
2470+
*/
2471+
useAIFallback?: boolean;
2472+
}
2473+
23002474
/**
23012475
* Screenshot options. Requires formats.screenshot: true.
23022476
*/

‎tests/api-resources/web.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ describe('resource web', () => {
142142
json: true,
143143
markdown: true,
144144
parse: true,
145+
product: true,
145146
screenshot: true,
146147
},
147148
url: 'https://example.com',
@@ -172,6 +173,7 @@ describe('resource web', () => {
172173
},
173174
},
174175
},
176+
productParams: { useAIFallback: true },
175177
screenshotParams: { area: 'viewport', format: 'png' },
176178
sharedParams: {
177179
actions: [{ action: 'Click the product details tab', type: 'perform' }],

0 commit comments

Comments
 (0)