Skip to content

Commit c5235fc

Browse files
author
stlc-bot
committed
feat(scrape): add CSS extraction rules to HTML scraping (#1146)
Stainless-Generated-From: e183d7360dba568f99f553bbb2af7bd984682d69
1 parent 9a80da2 commit c5235fc

2 files changed

Lines changed: 48 additions & 6 deletions

File tree

‎src/resources/web.ts‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -168,12 +168,16 @@ export class Web extends APIResource {
168168
}
169169

170170
/**
171-
* Scrapes the given URL and returns the raw HTML content of the page. The base
172-
* request costs 1 credit; requests with browser actions cost 2 credits. A request
173-
* that hits its timeoutOpts.milliseconds deadline fails with 408 and is not
174-
* billed, unless timeoutOpts.behavior=return-partial is set — then the page as
175-
* rendered so far is returned with `finalDOMState: "still-loading"` and billed at
176-
* the base cost of 1 credit.
171+
* Scrapes the given URL and returns the HTML content of the page. Optional
172+
* extractRules return deterministic structured data in extracted using CSS
173+
* selectors, attributes, lists, and nested rules, without an LLM or additional
174+
* credits. Rules run on the returned HTML after selector and main-content
175+
* filtering. Send extractRules as a JSON-encoded query parameter. The base request
176+
* costs 1 credit; requests with browser actions cost 2 credits. A request that
177+
* hits its timeoutOpts.milliseconds deadline fails with 408 and is not billed,
178+
* unless timeoutOpts.behavior=return-partial is set — then the page as rendered so
179+
* far is returned with `finalDOMState: "still-loading"` and billed at the base
180+
* cost of 1 credit.
177181
*
178182
* @example
179183
* ```ts
@@ -1917,6 +1921,14 @@ export interface WebWebScrapeHTMLResponse {
19171921
*/
19181922
actionsHtmlStale?: boolean;
19191923

1924+
/**
1925+
* Present only when extractRules is supplied. Keys match the requested fields.
1926+
* Values are normalized text, raw attribute strings, outer HTML, nested objects,
1927+
* or lists. Missing items are null; lists with no matches are empty. Rules run on
1928+
* the returned HTML after filtering.
1929+
*/
1930+
extracted?: { [key: string]: string | unknown | Array<string | unknown | null> | null };
1931+
19201932
/**
19211933
* Credit usage, included whenever a valid API key is provided.
19221934
*/
@@ -4749,6 +4761,17 @@ export interface WebWebScrapeHTMLParams {
47494761
*/
47504762
excludeSelectors?: Array<string> | null;
47514763

4764+
/**
4765+
* Optional CSS extraction rules applied to the returned HTML after selector and
4766+
* main-content filtering. Use selector strings ("h1", "a@href") or objects with
4767+
* selector, type (item or list), and output (text, html, @attribute, or nested
4768+
* rules). Text whitespace is normalized; html includes the matched element;
4769+
* attributes are returned as written. Missing items are null and missing lists are
4770+
* empty. CSS only; XPath is not supported. Maximum: 100 fields across 5 levels.
4771+
* Send a JSON-encoded string in the extractRules query parameter.
4772+
*/
4773+
extractRules?: { [key: string]: string | WebWebScrapeHTMLParams.UnionMember1 };
4774+
47524775
/**
47534776
* Optional outbound HTTP headers forwarded only to the target URL, sent as
47544777
* deep-object query params such as headers[X-Custom]=value. When provided, caching
@@ -4873,6 +4896,24 @@ export namespace WebWebScrapeHTMLParams {
48734896
maxScrolls?: number;
48744897
}
48754898

4899+
export interface UnionMember1 {
4900+
selector: string;
4901+
4902+
output?: 'text' | 'html' | string | { [key: string]: string | UnionMember1.UnionMember1 };
4903+
4904+
type?: 'item' | 'list';
4905+
}
4906+
4907+
export namespace UnionMember1 {
4908+
export interface UnionMember1 {
4909+
selector: string;
4910+
4911+
output?: 'text' | 'html' | string | unknown;
4912+
4913+
type?: 'item' | 'list';
4914+
}
4915+
}
4916+
48764917
/**
48774918
* PDF parsing controls. Use start/end to limit text extraction and embedded-image
48784919
* detection/OCR to an inclusive 1-based page range.

‎tests/api-resources/web.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ describe('resource web', () => {
334334
actions: [{ do: 'wait', timeMs: 0 }],
335335
country: 'de',
336336
excludeSelectors: ['x'],
337+
extractRules: { foo: 'x' },
337338
headers: { foo: 'J!' },
338339
includeFrames: true,
339340
includeSelectors: ['x'],

0 commit comments

Comments
 (0)