Skip to content

Commit babaf1d

Browse files
author
stlc-bot
committed
feat(scrape): add json output format to POST /web/scrape (#1213)
Stainless-Generated-From: 29fd9c2
1 parent a0fac10 commit babaf1d

9 files changed

Lines changed: 279 additions & 3 deletions

File tree

‎src/ServiceContracts/WebContract.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use ContextDev\Web\WebMapURLsResponse;
1717
use ContextDev\Web\WebScrapeParams\Formats;
1818
use ContextDev\Web\WebScrapeParams\ImageParams;
19+
use ContextDev\Web\WebScrapeParams\JsonParams;
1920
use ContextDev\Web\WebScrapeParams\MarkdownParams;
2021
use ContextDev\Web\WebScrapeParams\ParseParams;
2122
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
@@ -39,6 +40,7 @@
3940
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebMapURLsParams\TimeoutOpts as TimeoutOptsShape3
4041
* @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
4142
* @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
43+
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
4244
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
4345
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
4446
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
@@ -160,6 +162,7 @@ public function mapUrls(
160162
* @param Formats|FormatsShape $formats Outputs to return. Enable at least one; omitted formats are false.
161163
* @param string $url the URL to scrape
162164
* @param ImageParams|ImageParamsShape $imageParams Image options. Requires formats.images: true.
165+
* @param JsonParams|JsonParamsShape $jsonParams Required when formats.json is true.
163166
* @param MarkdownParams|MarkdownParamsShape $markdownParams Markdown options. Requires formats.markdown: true.
164167
* @param int $maxAgeMs Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and updates the requested outputs. Compatible outputs are shared with the individual scrape endpoints. Image results with hosted files refresh after 23 hours; other outputs retain their own freshness.
165168
* @param ParseParams|ParseParamsShape $parseParams Required when formats.parse is true.
@@ -176,6 +179,7 @@ public function scrape(
176179
Formats|array $formats,
177180
string $url,
178181
ImageParams|array|null $imageParams = null,
182+
JsonParams|array|null $jsonParams = null,
179183
MarkdownParams|array|null $markdownParams = null,
180184
int $maxAgeMs = 86400000,
181185
ParseParams|array|null $parseParams = null,

‎src/Services/WebRawService.php‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
use ContextDev\Web\WebScrapeParams;
2626
use ContextDev\Web\WebScrapeParams\Formats;
2727
use ContextDev\Web\WebScrapeParams\ImageParams;
28+
use ContextDev\Web\WebScrapeParams\JsonParams;
2829
use ContextDev\Web\WebScrapeParams\MarkdownParams;
2930
use ContextDev\Web\WebScrapeParams\ParseParams;
3031
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
@@ -51,6 +52,7 @@
5152
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebMapURLsParams\TimeoutOpts as TimeoutOptsShape3
5253
* @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
5354
* @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
55+
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
5456
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
5557
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
5658
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
@@ -234,12 +236,13 @@ public function mapUrls(
234236
/**
235237
* @api
236238
*
237-
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown and parsed fields. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits, or two with browser actions; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
239+
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits and missing pages, or two with browser actions; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
238240
*
239241
* @param array{
240242
* formats: Formats|FormatsShape,
241243
* url: string,
242244
* imageParams?: ImageParams|ImageParamsShape,
245+
* jsonParams?: JsonParams|JsonParamsShape,
243246
* markdownParams?: MarkdownParams|MarkdownParamsShape,
244247
* maxAgeMs?: int,
245248
* parseParams?: ParseParams|ParseParamsShape,

‎src/Services/WebService.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
use ContextDev\Web\WebMapURLsResponse;
2020
use ContextDev\Web\WebScrapeParams\Formats;
2121
use ContextDev\Web\WebScrapeParams\ImageParams;
22+
use ContextDev\Web\WebScrapeParams\JsonParams;
2223
use ContextDev\Web\WebScrapeParams\MarkdownParams;
2324
use ContextDev\Web\WebScrapeParams\ParseParams;
2425
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
@@ -42,6 +43,7 @@
4243
* @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebMapURLsParams\TimeoutOpts as TimeoutOptsShape3
4344
* @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
4445
* @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
46+
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
4547
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
4648
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
4749
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
@@ -249,11 +251,12 @@ public function mapUrls(
249251
/**
250252
* @api
251253
*
252-
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown and parsed fields. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits, or two with browser actions; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
254+
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits and missing pages, or two with browser actions; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
253255
*
254256
* @param Formats|FormatsShape $formats Outputs to return. Enable at least one; omitted formats are false.
255257
* @param string $url the URL to scrape
256258
* @param ImageParams|ImageParamsShape $imageParams Image options. Requires formats.images: true.
259+
* @param JsonParams|JsonParamsShape $jsonParams Required when formats.json is true.
257260
* @param MarkdownParams|MarkdownParamsShape $markdownParams Markdown options. Requires formats.markdown: true.
258261
* @param int $maxAgeMs Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and updates the requested outputs. Compatible outputs are shared with the individual scrape endpoints. Image results with hosted files refresh after 23 hours; other outputs retain their own freshness.
259262
* @param ParseParams|ParseParamsShape $parseParams Required when formats.parse is true.
@@ -270,6 +273,7 @@ public function scrape(
270273
Formats|array $formats,
271274
string $url,
272275
ImageParams|array|null $imageParams = null,
276+
JsonParams|array|null $jsonParams = null,
273277
MarkdownParams|array|null $markdownParams = null,
274278
int $maxAgeMs = 86400000,
275279
ParseParams|array|null $parseParams = null,
@@ -287,6 +291,7 @@ public function scrape(
287291
'formats' => $formats,
288292
'url' => $url,
289293
'imageParams' => $imageParams,
294+
'jsonParams' => $jsonParams,
290295
'markdownParams' => $markdownParams,
291296
'maxAgeMs' => $maxAgeMs,
292297
'parseParams' => $parseParams,

‎src/Web/WebScrapeParams.php‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
use ContextDev\Core\Contracts\BaseModel;
1212
use ContextDev\Web\WebScrapeParams\Formats;
1313
use ContextDev\Web\WebScrapeParams\ImageParams;
14+
use ContextDev\Web\WebScrapeParams\JsonParams;
1415
use ContextDev\Web\WebScrapeParams\MarkdownParams;
1516
use ContextDev\Web\WebScrapeParams\ParseParams;
1617
use ContextDev\Web\WebScrapeParams\ScreenshotParams;
@@ -19,12 +20,13 @@
1920
use ContextDev\Web\WebScrapeParams\Zdr;
2021

2122
/**
22-
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown and parsed fields. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits, or two with browser actions; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
23+
* Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits and missing pages, or two with browser actions; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
2324
*
2425
* @see ContextDev\Services\WebService::scrape()
2526
*
2627
* @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
2728
* @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
29+
* @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
2830
* @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
2931
* @phpstan-import-type ParseParamsShape from \ContextDev\Web\WebScrapeParams\ParseParams
3032
* @phpstan-import-type ScreenshotParamsShape from \ContextDev\Web\WebScrapeParams\ScreenshotParams
@@ -35,6 +37,7 @@
3537
* formats: Formats|FormatsShape,
3638
* url: string,
3739
* imageParams?: null|ImageParams|ImageParamsShape,
40+
* jsonParams?: null|JsonParams|JsonParamsShape,
3841
* markdownParams?: null|MarkdownParams|MarkdownParamsShape,
3942
* maxAgeMs?: int|null,
4043
* parseParams?: null|ParseParams|ParseParamsShape,
@@ -69,6 +72,12 @@ final class WebScrapeParams implements BaseModel
6972
#[Optional]
7073
public ?ImageParams $imageParams;
7174

75+
/**
76+
* Required when formats.json is true.
77+
*/
78+
#[Optional]
79+
public ?JsonParams $jsonParams;
80+
7281
/**
7382
* Markdown options. Requires formats.markdown: true.
7483
*/
@@ -147,6 +156,7 @@ public function __construct()
147156
*
148157
* @param Formats|FormatsShape $formats
149158
* @param ImageParams|ImageParamsShape|null $imageParams
159+
* @param JsonParams|JsonParamsShape|null $jsonParams
150160
* @param MarkdownParams|MarkdownParamsShape|null $markdownParams
151161
* @param ParseParams|ParseParamsShape|null $parseParams
152162
* @param ScreenshotParams|ScreenshotParamsShape|null $screenshotParams
@@ -159,6 +169,7 @@ public static function with(
159169
Formats|array $formats,
160170
string $url,
161171
ImageParams|array|null $imageParams = null,
172+
JsonParams|array|null $jsonParams = null,
162173
MarkdownParams|array|null $markdownParams = null,
163174
?int $maxAgeMs = null,
164175
ParseParams|array|null $parseParams = null,
@@ -174,6 +185,7 @@ public static function with(
174185
$self['url'] = $url;
175186

176187
null !== $imageParams && $self['imageParams'] = $imageParams;
188+
null !== $jsonParams && $self['jsonParams'] = $jsonParams;
177189
null !== $markdownParams && $self['markdownParams'] = $markdownParams;
178190
null !== $maxAgeMs && $self['maxAgeMs'] = $maxAgeMs;
179191
null !== $parseParams && $self['parseParams'] = $parseParams;
@@ -223,6 +235,19 @@ public function withImageParams(ImageParams|array $imageParams): self
223235
return $self;
224236
}
225237

238+
/**
239+
* Required when formats.json is true.
240+
*
241+
* @param JsonParams|JsonParamsShape $jsonParams
242+
*/
243+
public function withJsonParams(JsonParams|array $jsonParams): self
244+
{
245+
$self = clone $this;
246+
$self['jsonParams'] = $jsonParams;
247+
248+
return $self;
249+
}
250+
226251
/**
227252
* Markdown options. Requires formats.markdown: true.
228253
*

‎src/Web/WebScrapeParams/Formats.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
* bytes?: bool|null,
1616
* html?: bool|null,
1717
* images?: bool|null,
18+
* json?: bool|null,
1819
* markdown?: bool|null,
1920
* parse?: bool|null,
2021
* screenshot?: bool|null,
@@ -43,6 +44,12 @@ final class Formats implements BaseModel
4344
#[Optional]
4445
public ?bool $images;
4546

47+
/**
48+
* Page data extracted by an LLM from the page Markdown into jsonParams.schema; values carried only in attributes or CSS classes need formats.parse instead. Adds four credits when the page has text to extract; when shared content filters leave no text the result is an empty object and only the base price applies.
49+
*/
50+
#[Optional]
51+
public ?bool $json;
52+
4653
/**
4754
* Page content as Markdown.
4855
*/
@@ -75,6 +82,7 @@ public static function with(
7582
?bool $bytes = null,
7683
?bool $html = null,
7784
?bool $images = null,
85+
?bool $json = null,
7886
?bool $markdown = null,
7987
?bool $parse = null,
8088
?bool $screenshot = null,
@@ -84,6 +92,7 @@ public static function with(
8492
null !== $bytes && $self['bytes'] = $bytes;
8593
null !== $html && $self['html'] = $html;
8694
null !== $images && $self['images'] = $images;
95+
null !== $json && $self['json'] = $json;
8796
null !== $markdown && $self['markdown'] = $markdown;
8897
null !== $parse && $self['parse'] = $parse;
8998
null !== $screenshot && $self['screenshot'] = $screenshot;
@@ -124,6 +133,17 @@ public function withImages(bool $images): self
124133
return $self;
125134
}
126135

136+
/**
137+
* Page data extracted by an LLM from the page Markdown into jsonParams.schema; values carried only in attributes or CSS classes need formats.parse instead. Adds four credits when the page has text to extract; when shared content filters leave no text the result is an empty object and only the base price applies.
138+
*/
139+
public function withJson(bool $json): self
140+
{
141+
$self = clone $this;
142+
$self['json'] = $json;
143+
144+
return $self;
145+
}
146+
127147
/**
128148
* Page content as Markdown.
129149
*/

‎src/Web/WebScrapeParams/JsonParams.php‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace ContextDev\Web\WebScrapeParams;
6+
7+
use ContextDev\Core\Attributes\Optional;
8+
use ContextDev\Core\Attributes\Required;
9+
use ContextDev\Core\Concerns\SdkModel;
10+
use ContextDev\Core\Contracts\BaseModel;
11+
12+
/**
13+
* Required when formats.json is true.
14+
*
15+
* @phpstan-type JsonParamsShape = array{
16+
* schema: array<string,mixed>, instructions?: string|null
17+
* }
18+
*/
19+
final class JsonParams implements BaseModel
20+
{
21+
/** @use SdkModel<JsonParamsShape> */
22+
use SdkModel;
23+
24+
/**
25+
* JSON Schema for the returned object. Must describe a top-level object; at most 50 KB serialized. Optional fields the page does not state are omitted, or null when their type allows null, while required non-nullable fields always receive a best-effort value, so prefer nullable or optional fields for data a page may omit. Zod users can pass the output of z.toJSONSchema().
26+
*
27+
* @var array<string,mixed> $schema
28+
*/
29+
#[Required(map: 'mixed')]
30+
public array $schema;
31+
32+
/**
33+
* Optional guidance on which facts to prioritize or how to interpret schema fields.
34+
*/
35+
#[Optional]
36+
public ?string $instructions;
37+
38+
/**
39+
* `new JsonParams()` is missing required properties by the API.
40+
*
41+
* To enforce required parameters use
42+
* ```
43+
* JsonParams::with(schema: ...)
44+
* ```
45+
*
46+
* Otherwise ensure the following setters are called
47+
*
48+
* ```
49+
* (new JsonParams)->withSchema(...)
50+
* ```
51+
*/
52+
public function __construct()
53+
{
54+
$this->initialize();
55+
}
56+
57+
/**
58+
* Construct an instance from the required parameters.
59+
*
60+
* You must use named parameters to construct any parameters with a default value.
61+
*
62+
* @param array<string,mixed> $schema
63+
*/
64+
public static function with(
65+
array $schema,
66+
?string $instructions = null
67+
): self {
68+
$self = new self;
69+
70+
$self['schema'] = $schema;
71+
72+
null !== $instructions && $self['instructions'] = $instructions;
73+
74+
return $self;
75+
}
76+
77+
/**
78+
* JSON Schema for the returned object. Must describe a top-level object; at most 50 KB serialized. Optional fields the page does not state are omitted, or null when their type allows null, while required non-nullable fields always receive a best-effort value, so prefer nullable or optional fields for data a page may omit. Zod users can pass the output of z.toJSONSchema().
79+
*
80+
* @param array<string,mixed> $schema
81+
*/
82+
public function withSchema(array $schema): self
83+
{
84+
$self = clone $this;
85+
$self['schema'] = $schema;
86+
87+
return $self;
88+
}
89+
90+
/**
91+
* Optional guidance on which facts to prioritize or how to interpret schema fields.
92+
*/
93+
public function withInstructions(string $instructions): self
94+
{
95+
$self = clone $this;
96+
$self['instructions'] = $instructions;
97+
98+
return $self;
99+
}
100+
}

0 commit comments

Comments
 (0)