Skip to content

Commit d4ad692

Browse files
author
stlc-bot
committed
feat(scrape): add product extraction format to POST /web/scrape (#1212)
Stainless-Generated-From: 2532392
1 parent 89485cb commit d4ad692

13 files changed

Lines changed: 936 additions & 5 deletions

File tree

‎src/ServiceContracts/WebContract.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
use ContextDev\Web\WebScrapeParams\JsonParams;
2121
use ContextDev\Web\WebScrapeParams\MarkdownParams;
2222
use ContextDev\Web\WebScrapeParams\ParseParams;
23+
use ContextDev\Web\WebScrapeParams\ProductParams;
2324
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
2425
use ContextDev\Web\WebScrapeParams\SharedParams;
2526
use ContextDev\Web\WebScrapeResponse;
@@ -45,6 +46,7 @@
4546
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
4647
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
4748
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
49+
* @phpstan-import-type ProductParamsShape from \ContextDev\Web\WebScrapeParams\ProductParams
4850
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
4951
* @phpstan-import-type SharedParamsShape from \ContextDev\Web\WebScrapeParams\SharedParams
5052
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebScrapeParams\TimeoutOpts as TimeoutOptsShape4
@@ -169,6 +171,7 @@ public function mapUrls(
169171
* @param MarkdownParams|MarkdownParamsShape $markdownParams Markdown options. Requires formats.markdown: true.
170172
* @param int $maxAgeMs Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and updates the requested outputs. Compatible outputs are shared with the individual scrape endpoints. Image results with hosted files refresh after 23 hours; other outputs retain their own freshness.
171173
* @param ParseParams|ParseParamsShape $parseParams Required when formats.parse is true.
174+
* @param ProductParams|ProductParamsShape $productParams Product options. Requires formats.product: true.
172175
* @param ScreenshotParams|ScreenshotParamsShape $screenshotParams Screenshot options. Requires formats.screenshot: true.
173176
* @param SharedParams|SharedParamsShape $sharedParams Shared browser and content settings. Content filters leave screenshots and original bytes unchanged.
174177
* @param list<string> $tags Labels for tracking request usage. Not retained when zdr is enabled.
@@ -187,6 +190,7 @@ public function scrape(
187190
MarkdownParams|array|null $markdownParams = null,
188191
int $maxAgeMs = 86400000,
189192
ParseParams|array|null $parseParams = null,
193+
ProductParams|array|null $productParams = null,
190194
ScreenshotParams|array|null $screenshotParams = null,
191195
SharedParams|array|null $sharedParams = null,
192196
?array $tags = null,

‎src/Services/WebRawService.php‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
use ContextDev\Web\WebScrapeParams\JsonParams;
3030
use ContextDev\Web\WebScrapeParams\MarkdownParams;
3131
use ContextDev\Web\WebScrapeParams\ParseParams;
32+
use ContextDev\Web\WebScrapeParams\ProductParams;
3233
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
3334
use ContextDev\Web\WebScrapeParams\SharedParams;
3435
use ContextDev\Web\WebScrapeResponse;
@@ -57,6 +58,7 @@
5758
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
5859
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
5960
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
61+
* @phpstan-import-type ProductParamsShape from \ContextDev\Web\WebScrapeParams\ProductParams
6062
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
6163
* @phpstan-import-type SharedParamsShape from \ContextDev\Web\WebScrapeParams\SharedParams
6264
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebScrapeParams\TimeoutOpts as TimeoutOptsShape4
@@ -238,7 +240,7 @@ public function mapUrls(
238240
/**
239241
* @api
240242
*
241-
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
243+
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, product data, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction; the product output adds one credit, plus six more when the specialized model is used. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
242244
*
243245
* @param array{
244246
* formats: Formats|FormatsShape,
@@ -249,6 +251,7 @@ public function mapUrls(
249251
* markdownParams?: MarkdownParams|MarkdownParamsShape,
250252
* maxAgeMs?: int,
251253
* parseParams?: ParseParams|ParseParamsShape,
254+
* productParams?: ProductParams|ProductParamsShape,
252255
* screenshotParams?: ScreenshotParams|ScreenshotParamsShape,
253256
* sharedParams?: SharedParams|SharedParamsShape,
254257
* tags?: list<string>,

‎src/Services/WebService.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
use ContextDev\Web\WebScrapeParams\JsonParams;
2424
use ContextDev\Web\WebScrapeParams\MarkdownParams;
2525
use ContextDev\Web\WebScrapeParams\ParseParams;
26+
use ContextDev\Web\WebScrapeParams\ProductParams;
2627
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
2728
use ContextDev\Web\WebScrapeParams\SharedParams;
2829
use ContextDev\Web\WebScrapeResponse;
@@ -48,6 +49,7 @@
4849
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
4950
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
5051
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
52+
* @phpstan-import-type ProductParamsShape from \ContextDev\Web\WebScrapeParams\ProductParams
5153
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
5254
* @phpstan-import-type SharedParamsShape from \ContextDev\Web\WebScrapeParams\SharedParams
5355
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebScrapeParams\TimeoutOpts as TimeoutOptsShape4
@@ -253,7 +255,7 @@ public function mapUrls(
253255
/**
254256
* @api
255257
*
256-
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
258+
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, product data, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction; the product output adds one credit, plus six more when the specialized model is used. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
257259
*
258260
* @param Formats|FormatsShape $formats Outputs to return. Enable at least one; omitted formats are false.
259261
* @param string $url the URL to scrape
@@ -263,6 +265,7 @@ public function mapUrls(
263265
* @param MarkdownParams|MarkdownParamsShape $markdownParams Markdown options. Requires formats.markdown: true.
264266
* @param int $maxAgeMs Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and updates the requested outputs. Compatible outputs are shared with the individual scrape endpoints. Image results with hosted files refresh after 23 hours; other outputs retain their own freshness.
265267
* @param ParseParams|ParseParamsShape $parseParams Required when formats.parse is true.
268+
* @param ProductParams|ProductParamsShape $productParams Product options. Requires formats.product: true.
266269
* @param ScreenshotParams|ScreenshotParamsShape $screenshotParams Screenshot options. Requires formats.screenshot: true.
267270
* @param SharedParams|SharedParamsShape $sharedParams Shared browser and content settings. Content filters leave screenshots and original bytes unchanged.
268271
* @param list<string> $tags Labels for tracking request usage. Not retained when zdr is enabled.
@@ -281,6 +284,7 @@ public function scrape(
281284
MarkdownParams|array|null $markdownParams = null,
282285
int $maxAgeMs = 86400000,
283286
ParseParams|array|null $parseParams = null,
287+
ProductParams|array|null $productParams = null,
284288
ScreenshotParams|array|null $screenshotParams = null,
285289
SharedParams|array|null $sharedParams = null,
286290
?array $tags = null,
@@ -300,6 +304,7 @@ public function scrape(
300304
'markdownParams' => $markdownParams,
301305
'maxAgeMs' => $maxAgeMs,
302306
'parseParams' => $parseParams,
307+
'productParams' => $productParams,
303308
'screenshotParams' => $screenshotParams,
304309
'sharedParams' => $sharedParams,
305310
'tags' => $tags,

‎src/Web/WebScrapeParams.php‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,14 @@
1515
use ContextDev\Web\WebScrapeParams\JsonParams;
1616
use ContextDev\Web\WebScrapeParams\MarkdownParams;
1717
use ContextDev\Web\WebScrapeParams\ParseParams;
18+
use ContextDev\Web\WebScrapeParams\ProductParams;
1819
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
1920
use ContextDev\Web\WebScrapeParams\SharedParams;
2021
use ContextDev\Web\WebScrapeParams\TimeoutOpts;
2122
use ContextDev\Web\WebScrapeParams\Zdr;
2223

2324
/**
24-
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
25+
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, product data, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction; the product output adds one credit, plus six more when the specialized model is used. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
2526
*
2627
* @see ContextDev\Services\WebService::scrape()
2728
*
@@ -31,6 +32,7 @@
3132
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
3233
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
3334
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
35+
* @phpstan-import-type ProductParamsShape from \ContextDev\Web\WebScrapeParams\ProductParams
3436
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
3537
* @phpstan-import-type SharedParamsShape from \ContextDev\Web\WebScrapeParams\SharedParams
3638
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebScrapeParams\TimeoutOpts
@@ -44,6 +46,7 @@
4446
* markdownParams?: null|MarkdownParams|MarkdownParamsShape,
4547
* maxAgeMs?: int|null,
4648
* parseParams?: null|ParseParams|ParseParamsShape,
49+
* productParams?: null|ProductParams|ProductParamsShape,
4750
* screenshotParams?: null|ScreenshotParams|ScreenshotParamsShape,
4851
* sharedParams?: null|SharedParams|SharedParamsShape,
4952
* tags?: list<string>|null,
@@ -105,6 +108,12 @@ final class WebScrapeParams implements BaseModel
105108
#[Optional]
106109
public ?ParseParams $parseParams;
107110

111+
/**
112+
* Product options. Requires formats.product: true.
113+
*/
114+
#[Optional]
115+
public ?ProductParams $productParams;
116+
108117
/**
109118
* Screenshot options. Requires formats.screenshot: true.
110119
*/
@@ -169,6 +178,7 @@ public function __construct()
169178
* @param JsonParams|JsonParamsShape|null $jsonParams
170179
* @param MarkdownParams|MarkdownParamsShape|null $markdownParams
171180
* @param ParseParams|ParseParamsShape|null $parseParams
181+
* @param ProductParams|ProductParamsShape|null $productParams
172182
* @param ScreenshotParams|ScreenshotParamsShape|null $screenshotParams
173183
* @param SharedParams|SharedParamsShape|null $sharedParams
174184
* @param list<string>|null $tags
@@ -184,6 +194,7 @@ public static function with(
184194
MarkdownParams|array|null $markdownParams = null,
185195
?int $maxAgeMs = null,
186196
ParseParams|array|null $parseParams = null,
197+
ProductParams|array|null $productParams = null,
187198
ScreenshotParams|array|null $screenshotParams = null,
188199
SharedParams|array|null $sharedParams = null,
189200
?array $tags = null,
@@ -201,6 +212,7 @@ public static function with(
201212
null !== $markdownParams && $self['markdownParams'] = $markdownParams;
202213
null !== $maxAgeMs && $self['maxAgeMs'] = $maxAgeMs;
203214
null !== $parseParams && $self['parseParams'] = $parseParams;
215+
null !== $productParams && $self['productParams'] = $productParams;
204216
null !== $screenshotParams && $self['screenshotParams'] = $screenshotParams;
205217
null !== $sharedParams && $self['sharedParams'] = $sharedParams;
206218
null !== $tags && $self['tags'] = $tags;
@@ -312,6 +324,19 @@ public function withParseParams(ParseParams|array $parseParams): self
312324
return $self;
313325
}
314326

327+
/**
328+
* Product options. Requires formats.product: true.
329+
*
330+
* @param ProductParams|ProductParamsShape $productParams
331+
*/
332+
public function withProductParams(ProductParams|array $productParams): self
333+
{
334+
$self = clone $this;
335+
$self['productParams'] = $productParams;
336+
337+
return $self;
338+
}
339+
315340
/**
316341
* Screenshot options. Requires formats.screenshot: true.
317342
*

‎src/Web/WebScrapeParams/Formats.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
* json?: bool|null,
2020
* markdown?: bool|null,
2121
* parse?: bool|null,
22+
* product?: bool|null,
2223
* screenshot?: bool|null,
2324
* }
2425
*/
@@ -69,6 +70,12 @@ final class Formats implements BaseModel
6970
#[Optional]
7071
public ?bool $parse;
7172

73+
/**
74+
* Structured product data for product detail pages. Adds one credit.
75+
*/
76+
#[Optional]
77+
public ?bool $product;
78+
7279
/**
7380
* An inline image of the page.
7481
*/
@@ -93,6 +100,7 @@ public static function with(
93100
?bool $json = null,
94101
?bool $markdown = null,
95102
?bool $parse = null,
103+
?bool $product = null,
96104
?bool $screenshot = null,
97105
): self {
98106
$self = new self;
@@ -104,6 +112,7 @@ public static function with(
104112
null !== $json && $self['json'] = $json;
105113
null !== $markdown && $self['markdown'] = $markdown;
106114
null !== $parse && $self['parse'] = $parse;
115+
null !== $product && $self['product'] = $product;
107116
null !== $screenshot && $self['screenshot'] = $screenshot;
108117

109118
return $self;
@@ -186,6 +195,17 @@ public function withParse(bool $parse): self
186195
return $self;
187196
}
188197

198+
/**
199+
* Structured product data for product detail pages. Adds one credit.
200+
*/
201+
public function withProduct(bool $product): self
202+
{
203+
$self = clone $this;
204+
$self['product'] = $product;
205+
206+
return $self;
207+
}
208+
189209
/**
190210
* An inline image of the page.
191211
*/

‎src/Web/WebScrapeParams/ProductParams.php‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace ContextDev\Web\WebScrapeParams;
6+
7+
use ContextDev\Core\Attributes\Optional;
8+
use ContextDev\Core\Concerns\SdkModel;
9+
use ContextDev\Core\Contracts\BaseModel;
10+
11+
/**
12+
* Product options. Requires formats.product: true.
13+
*
14+
* @phpstan-type ProductParamsShape = array{useAIFallback?: bool|null}
15+
*/
16+
final class ProductParams implements BaseModel
17+
{
18+
/** @use SdkModel<ProductParamsShape> */
19+
use SdkModel;
20+
21+
/**
22+
* Extract the product with a specialized model when the page has no structured product data. Adds six credits when the model returns a verdict. If the fallback fails, returns a partial response with the deterministic result and no fallback charge. Request deadlines and client disconnects still apply.
23+
*/
24+
#[Optional]
25+
public ?bool $useAIFallback;
26+
27+
public function __construct()
28+
{
29+
$this->initialize();
30+
}
31+
32+
/**
33+
* Construct an instance from the required parameters.
34+
*
35+
* You must use named parameters to construct any parameters with a default value.
36+
*/
37+
public static function with(?bool $useAIFallback = null): self
38+
{
39+
$self = new self;
40+
41+
null !== $useAIFallback && $self['useAIFallback'] = $useAIFallback;
42+
43+
return $self;
44+
}
45+
46+
/**
47+
* Extract the product with a specialized model when the page has no structured product data. Adds six credits when the model returns a verdict. If the fallback fails, returns a partial response with the deterministic result and no fallback charge. Request deadlines and client disconnects still apply.
48+
*/
49+
public function withUseAIFallback(bool $useAIFallback): self
50+
{
51+
$self = clone $this;
52+
$self['useAIFallback'] = $useAIFallback;
53+
54+
return $self;
55+
}
56+
}

0 commit comments

Comments
 (0)