feat(scrape): add highlights format to POST /web/scrape (#1210)

Stainless-Generated-From: 3aeff5b07cbc2ffae72b3f43c05f4268a8d14650

Author: stlc-bot

src/ServiceContracts/WebContract.php

@@ -15,6 +15,7 @@
 use ContextDev\Web\WebExtractStyleguideResponse;
 use ContextDev\Web\WebMapURLsResponse;
 use ContextDev\Web\WebScrapeParams\Formats;
+use ContextDev\Web\WebScrapeParams\HighlightsParams;
 use ContextDev\Web\WebScrapeParams\ImageParams;
 use ContextDev\Web\WebScrapeParams\JsonParams;
 use ContextDev\Web\WebScrapeParams\MarkdownParams;
@@ -39,6 +40,7 @@
  * @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebExtractStyleguideParams\TimeoutOpts as TimeoutOptsShape2
  * @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebMapURLsParams\TimeoutOpts as TimeoutOptsShape3
  * @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
+ * @phpstan-import-type HighlightsParamsShape from \ContextDev\Web\WebScrapeParams\HighlightsParams
  * @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
  * @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
  * @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
@@ -161,6 +163,7 @@ public function mapUrls(
      *
      * @param Formats|FormatsShape $formats Outputs to return. Enable at least one; omitted formats are false.
      * @param string $url the URL to scrape
+     * @param HighlightsParams|HighlightsParamsShape $highlightsParams Highlight options. Requires formats.highlights: true.
      * @param ImageParams|ImageParamsShape $imageParams Image options. Requires formats.images: true.
      * @param JsonParams|JsonParamsShape $jsonParams Required when formats.json is true.
      * @param MarkdownParams|MarkdownParamsShape $markdownParams Markdown options. Requires formats.markdown: true.
@@ -170,14 +173,15 @@ public function mapUrls(
      * @param SharedParams|SharedParamsShape $sharedParams Shared browser and content settings. Content filters leave screenshots and original bytes unchanged.
      * @param list<string> $tags Labels for tracking request usage. Not retained when zdr is enabled.
      * @param \ContextDev\Web\WebScrapeParams\TimeoutOpts|TimeoutOptsShape4 $timeoutOpts Total deadline, including navigation, actions, waiting, and all outputs. Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture the current page state and return captured images if image processing cannot finish before the deadline; these responses set isPartial and are not cached. Every requested format must still be available. Fixed waits must fit before a response reserve of up to 5000 milliseconds (at most one quarter of the timeout) when using return-partial.
-     * @param \ContextDev\Web\WebScrapeParams\Zdr|value-of<\ContextDev\Web\WebScrapeParams\Zdr> $zdr Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization.
+     * @param \ContextDev\Web\WebScrapeParams\Zdr|value-of<\ContextDev\Web\WebScrapeParams\Zdr> $zdr Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization. Not available with the highlights output.
      * @param RequestOpts|null $requestOptions
      *
      * @throws APIException
      */
     public function scrape(
         Formats|array $formats,
         string $url,
+        HighlightsParams|array|null $highlightsParams = null,
         ImageParams|array|null $imageParams = null,
         JsonParams|array|null $jsonParams = null,
         MarkdownParams|array|null $markdownParams = null,

src/Services/WebRawService.php

@@ -24,6 +24,7 @@
 use ContextDev\Web\WebMapURLsResponse;
 use ContextDev\Web\WebScrapeParams;
 use ContextDev\Web\WebScrapeParams\Formats;
+use ContextDev\Web\WebScrapeParams\HighlightsParams;
 use ContextDev\Web\WebScrapeParams\ImageParams;
 use ContextDev\Web\WebScrapeParams\JsonParams;
 use ContextDev\Web\WebScrapeParams\MarkdownParams;
@@ -51,6 +52,7 @@
  * @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebExtractStyleguideParams\TimeoutOpts as TimeoutOptsShape2
  * @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebMapURLsParams\TimeoutOpts as TimeoutOptsShape3
  * @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
+ * @phpstan-import-type HighlightsParamsShape from \ContextDev\Web\WebScrapeParams\HighlightsParams
  * @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
  * @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
  * @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
@@ -236,11 +238,12 @@ public function mapUrls(
     /**
      * @api
      *
-     * Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits and missing pages, or two with browser actions; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+     * Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
      *
      * @param array{
      *   formats: Formats|FormatsShape,
      *   url: string,
+     *   highlightsParams?: HighlightsParams|HighlightsParamsShape,
      *   imageParams?: ImageParams|ImageParamsShape,
      *   jsonParams?: JsonParams|JsonParamsShape,
      *   markdownParams?: MarkdownParams|MarkdownParamsShape,

src/Services/WebService.php

@@ -18,6 +18,7 @@
 use ContextDev\Web\WebExtractStyleguideResponse;
 use ContextDev\Web\WebMapURLsResponse;
 use ContextDev\Web\WebScrapeParams\Formats;
+use ContextDev\Web\WebScrapeParams\HighlightsParams;
 use ContextDev\Web\WebScrapeParams\ImageParams;
 use ContextDev\Web\WebScrapeParams\JsonParams;
 use ContextDev\Web\WebScrapeParams\MarkdownParams;
@@ -42,6 +43,7 @@
  * @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebExtractStyleguideParams\TimeoutOpts as TimeoutOptsShape2
  * @phpstan-import-type TimeoutOptsShape from \ContextDev\Web\WebMapURLsParams\TimeoutOpts as TimeoutOptsShape3
  * @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
+ * @phpstan-import-type HighlightsParamsShape from \ContextDev\Web\WebScrapeParams\HighlightsParams
  * @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
  * @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
  * @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
@@ -251,10 +253,11 @@ public function mapUrls(
     /**
      * @api
      *
-     * Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits and missing pages, or two with browser actions; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+     * Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
      *
      * @param Formats|FormatsShape $formats Outputs to return. Enable at least one; omitted formats are false.
      * @param string $url the URL to scrape
+     * @param HighlightsParams|HighlightsParamsShape $highlightsParams Highlight options. Requires formats.highlights: true.
      * @param ImageParams|ImageParamsShape $imageParams Image options. Requires formats.images: true.
      * @param JsonParams|JsonParamsShape $jsonParams Required when formats.json is true.
      * @param MarkdownParams|MarkdownParamsShape $markdownParams Markdown options. Requires formats.markdown: true.
@@ -264,14 +267,15 @@ public function mapUrls(
      * @param SharedParams|SharedParamsShape $sharedParams Shared browser and content settings. Content filters leave screenshots and original bytes unchanged.
      * @param list<string> $tags Labels for tracking request usage. Not retained when zdr is enabled.
      * @param \ContextDev\Web\WebScrapeParams\TimeoutOpts|TimeoutOptsShape4 $timeoutOpts Total deadline, including navigation, actions, waiting, and all outputs. Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture the current page state and return captured images if image processing cannot finish before the deadline; these responses set isPartial and are not cached. Every requested format must still be available. Fixed waits must fit before a response reserve of up to 5000 milliseconds (at most one quarter of the timeout) when using return-partial.
-     * @param \ContextDev\Web\WebScrapeParams\Zdr|value-of<\ContextDev\Web\WebScrapeParams\Zdr> $zdr Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization.
+     * @param \ContextDev\Web\WebScrapeParams\Zdr|value-of<\ContextDev\Web\WebScrapeParams\Zdr> $zdr Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization. Not available with the highlights output.
      * @param RequestOpts|null $requestOptions
      *
      * @throws APIException
      */
     public function scrape(
         Formats|array $formats,
         string $url,
+        HighlightsParams|array|null $highlightsParams = null,
         ImageParams|array|null $imageParams = null,
         JsonParams|array|null $jsonParams = null,
         MarkdownParams|array|null $markdownParams = null,
@@ -290,6 +294,7 @@ public function scrape(
             [
                 'formats' => $formats,
                 'url' => $url,
+                'highlightsParams' => $highlightsParams,
                 'imageParams' => $imageParams,
                 'jsonParams' => $jsonParams,
                 'markdownParams' => $markdownParams,

src/Web/WebScrapeParams.php

@@ -10,6 +10,7 @@
 use ContextDev\Core\Concerns\SdkParams;
 use ContextDev\Core\Contracts\BaseModel;
 use ContextDev\Web\WebScrapeParams\Formats;
+use ContextDev\Web\WebScrapeParams\HighlightsParams;
 use ContextDev\Web\WebScrapeParams\ImageParams;
 use ContextDev\Web\WebScrapeParams\JsonParams;
 use ContextDev\Web\WebScrapeParams\MarkdownParams;
@@ -20,11 +21,12 @@
 use ContextDev\Web\WebScrapeParams\Zdr;
 
 /**
- * Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. One credit per request, including cache hits and missing pages, or two with browser actions; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+ * Reuse cached outputs independently and capture missing formats in one page visit. Each cache key includes only the settings that affect that output. HTML is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached outputs can come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests use the existing fast acquisition path. Highlights return the plain-text passages most relevant to highlightsParams.query. One credit per request, including cache hits and missing pages, or two with browser actions; highlights add 3 credits when passages are returned; JSON extraction adds four credits and runs an LLM over the page Markdown on every request that has text to extract; PDF OCR adds one credit per recovered page on fresh extraction. Original response bytes and screenshots are limited to 20 MiB each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
  *
  * @see ContextDev\Services\WebService::scrape()
  *
  * @phpstan-import-type FormatsShape from \ContextDev\Web\WebScrapeParams\Formats
+ * @phpstan-import-type HighlightsParamsShape from \ContextDev\Web\WebScrapeParams\HighlightsParams
  * @phpstan-import-type ImageParamsShape from \ContextDev\Web\WebScrapeParams\ImageParams
  * @phpstan-import-type JsonParamsShape from \ContextDev\Web\WebScrapeParams\JsonParams
  * @phpstan-import-type MarkdownParamsShape from \ContextDev\Web\WebScrapeParams\MarkdownParams
@@ -36,6 +38,7 @@
  * @phpstan-type WebScrapeParamsShape = array{
  *   formats: Formats|FormatsShape,
  *   url: string,
+ *   highlightsParams?: null|HighlightsParams|HighlightsParamsShape,
  *   imageParams?: null|ImageParams|ImageParamsShape,
  *   jsonParams?: null|JsonParams|JsonParamsShape,
  *   markdownParams?: null|MarkdownParams|MarkdownParamsShape,
@@ -66,6 +69,12 @@ final class WebScrapeParams implements BaseModel
     #[Required]
     public string $url;
 
+    /**
+     * Highlight options. Requires formats.highlights: true.
+     */
+    #[Optional]
+    public ?HighlightsParams $highlightsParams;
+
     /**
      * Image options. Requires formats.images: true.
      */
@@ -123,7 +132,7 @@ final class WebScrapeParams implements BaseModel
     public ?TimeoutOpts $timeoutOpts;
 
     /**
-     * Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization.
+     * Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization. Not available with the highlights output.
      *
      * @var value-of<Zdr>|null $zdr
      */
@@ -155,6 +164,7 @@ public function __construct()
      * You must use named parameters to construct any parameters with a default value.
      *
      * @param Formats|FormatsShape $formats
+     * @param HighlightsParams|HighlightsParamsShape|null $highlightsParams
      * @param ImageParams|ImageParamsShape|null $imageParams
      * @param JsonParams|JsonParamsShape|null $jsonParams
      * @param MarkdownParams|MarkdownParamsShape|null $markdownParams
@@ -168,6 +178,7 @@ public function __construct()
     public static function with(
         Formats|array $formats,
         string $url,
+        HighlightsParams|array|null $highlightsParams = null,
         ImageParams|array|null $imageParams = null,
         JsonParams|array|null $jsonParams = null,
         MarkdownParams|array|null $markdownParams = null,
@@ -184,6 +195,7 @@ public static function with(
         $self['formats'] = $formats;
         $self['url'] = $url;
 
+        null !== $highlightsParams && $self['highlightsParams'] = $highlightsParams;
         null !== $imageParams && $self['imageParams'] = $imageParams;
         null !== $jsonParams && $self['jsonParams'] = $jsonParams;
         null !== $markdownParams && $self['markdownParams'] = $markdownParams;
@@ -222,6 +234,20 @@ public function withURL(string $url): self
         return $self;
     }
 
+    /**
+     * Highlight options. Requires formats.highlights: true.
+     *
+     * @param HighlightsParams|HighlightsParamsShape $highlightsParams
+     */
+    public function withHighlightsParams(
+        HighlightsParams|array $highlightsParams
+    ): self {
+        $self = clone $this;
+        $self['highlightsParams'] = $highlightsParams;
+
+        return $self;
+    }
+
     /**
      * Image options. Requires formats.images: true.
      *
@@ -340,7 +366,7 @@ public function withTimeoutOpts(TimeoutOpts|array $timeoutOpts): self
     }
 
     /**
-     * Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization.
+     * Zero data retention. Bypasses caches and uploads; excludes request/response content and tags from logs. Must be enabled for your organization. Not available with the highlights output.
      *
      * @param Zdr|value-of<Zdr> $zdr
      */

src/Web/WebScrapeParams/Formats.php

@@ -13,6 +13,7 @@
  *
  * @phpstan-type FormatsShape = array{
  *   bytes?: bool|null,
+ *   highlights?: bool|null,
  *   html?: bool|null,
  *   images?: bool|null,
  *   json?: bool|null,
@@ -32,6 +33,12 @@ final class Formats implements BaseModel
     #[Optional]
     public ?bool $bytes;
 
+    /**
+     * Plain-text passages from the page that are most relevant to highlightsParams.query, each prefixed with its section heading. Adds 3 credits. Not available with zdr enabled.
+     */
+    #[Optional]
+    public ?bool $highlights;
+
     /**
      * Rendered HTML.
      */
@@ -80,6 +87,7 @@ public function __construct()
      */
     public static function with(
         ?bool $bytes = null,
+        ?bool $highlights = null,
         ?bool $html = null,
         ?bool $images = null,
         ?bool $json = null,
@@ -90,6 +98,7 @@ public static function with(
         $self = new self;
 
         null !== $bytes && $self['bytes'] = $bytes;
+        null !== $highlights && $self['highlights'] = $highlights;
         null !== $html && $self['html'] = $html;
         null !== $images && $self['images'] = $images;
         null !== $json && $self['json'] = $json;
@@ -111,6 +120,17 @@ public function withBytes(bool $bytes): self
         return $self;
     }
 
+    /**
+     * Plain-text passages from the page that are most relevant to highlightsParams.query, each prefixed with its section heading. Adds 3 credits. Not available with zdr enabled.
+     */
+    public function withHighlights(bool $highlights): self
+    {
+        $self = clone $this;
+        $self['highlights'] = $highlights;
+
+        return $self;
+    }
+
     /**
      * Rendered HTML.
      */