feat(sitemap): add subdomain discovery

Stainless-Generated-From: 46e084a

Author: stlc-bot

src/ServiceContracts/WebContract.php

@@ -405,6 +405,7 @@ public function webScrapeMd(
      *
      * @param string $domain Domain to build a sitemap for
      * @param array<string,string> $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.
+     * @param bool $includeSubdomains When true, discover and include public pages and sitemaps on subdomains of the requested domain. Defaults to false.
      * @param int $maxLinks Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000.
      * @param string $search Optional search phrase. When provided, the crawled sitemap is filtered to the pages whose URLs are about that phrase, most relevant first, and the request costs 2 credits instead of 1.
      * @param string $sitemapURL Optional explicit sitemap URL. When provided, exactly this sitemap is crawled instead of discovering the domain's sitemaps.
@@ -419,6 +420,7 @@ public function webScrapeMd(
     public function webScrapeSitemap(
         string $domain,
         ?array $headers = null,
+        bool $includeSubdomains = false,
         int $maxLinks = 10000,
         ?string $search = null,
         ?string $sitemapURL = null,

src/Services/WebRawService.php

@@ -532,11 +532,12 @@ public function webScrapeMd(
     /**
      * @api
      *
-     * Crawl an entire website's sitemap and return all discovered page URLs. Pass `search` to have the crawled sitemap filtered down to the pages about a phrase (for example `pricing and plans` or `api authentication docs`), most relevant first — a searched crawl scans the whole sitemap and costs 2 credits instead of 1.
+     * Crawl an entire website's sitemap and return all discovered page URLs. Set `includeSubdomains=true` to also discover public pages and sitemaps on child hosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have the discovered URLs filtered down to the pages about a phrase (for example `pricing and plans` or `api authentication docs`), most relevant first — a searched crawl scans the whole sitemap and costs 2 credits instead of 1.
      *
      * @param array{
      *   domain: string,
      *   headers?: array<string,string>,
+     *   includeSubdomains?: bool,
      *   maxLinks?: int,
      *   search?: string,
      *   sitemapURL?: string,

src/Services/WebService.php

@@ -677,10 +677,11 @@ public function webScrapeMd(
     /**
      * @api
      *
-     * Crawl an entire website's sitemap and return all discovered page URLs. Pass `search` to have the crawled sitemap filtered down to the pages about a phrase (for example `pricing and plans` or `api authentication docs`), most relevant first — a searched crawl scans the whole sitemap and costs 2 credits instead of 1.
+     * Crawl an entire website's sitemap and return all discovered page URLs. Set `includeSubdomains=true` to also discover public pages and sitemaps on child hosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have the discovered URLs filtered down to the pages about a phrase (for example `pricing and plans` or `api authentication docs`), most relevant first — a searched crawl scans the whole sitemap and costs 2 credits instead of 1.
      *
      * @param string $domain Domain to build a sitemap for
      * @param array<string,string> $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.
+     * @param bool $includeSubdomains When true, discover and include public pages and sitemaps on subdomains of the requested domain. Defaults to false.
      * @param int $maxLinks Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000.
      * @param string $search Optional search phrase. When provided, the crawled sitemap is filtered to the pages whose URLs are about that phrase, most relevant first, and the request costs 2 credits instead of 1.
      * @param string $sitemapURL Optional explicit sitemap URL. When provided, exactly this sitemap is crawled instead of discovering the domain's sitemaps.
@@ -695,6 +696,7 @@ public function webScrapeMd(
     public function webScrapeSitemap(
         string $domain,
         ?array $headers = null,
+        bool $includeSubdomains = false,
         int $maxLinks = 10000,
         ?string $search = null,
         ?string $sitemapURL = null,
@@ -708,6 +710,7 @@ public function webScrapeSitemap(
             [
                 'domain' => $domain,
                 'headers' => $headers,
+                'includeSubdomains' => $includeSubdomains,
                 'maxLinks' => $maxLinks,
                 'search' => $search,
                 'sitemapURL' => $sitemapURL,

src/Web/WebWebScrapeSitemapParams.php

@@ -12,13 +12,14 @@
 use ContextDev\Web\WebWebScrapeSitemapParams\Zdr;
 
 /**
- * Crawl an entire website's sitemap and return all discovered page URLs. Pass `search` to have the crawled sitemap filtered down to the pages about a phrase (for example `pricing and plans` or `api authentication docs`), most relevant first — a searched crawl scans the whole sitemap and costs 2 credits instead of 1.
+ * Crawl an entire website's sitemap and return all discovered page URLs. Set `includeSubdomains=true` to also discover public pages and sitemaps on child hosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have the discovered URLs filtered down to the pages about a phrase (for example `pricing and plans` or `api authentication docs`), most relevant first — a searched crawl scans the whole sitemap and costs 2 credits instead of 1.
  *
  * @see ContextDev\Services\WebService::webScrapeSitemap()
  *
  * @phpstan-type WebWebScrapeSitemapParamsShape = array{
  *   domain: string,
  *   headers?: array<string,string>|null,
+ *   includeSubdomains?: bool|null,
  *   maxLinks?: int|null,
  *   search?: string|null,
  *   sitemapURL?: string|null,
@@ -48,6 +49,12 @@ final class WebWebScrapeSitemapParams implements BaseModel
     #[Optional(map: 'string')]
     public ?array $headers;
 
+    /**
+     * When true, discover and include public pages and sitemaps on subdomains of the requested domain. Defaults to false.
+     */
+    #[Optional]
+    public ?bool $includeSubdomains;
+
     /**
      * Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000.
      */
@@ -125,6 +132,7 @@ public function __construct()
     public static function with(
         string $domain,
         ?array $headers = null,
+        ?bool $includeSubdomains = null,
         ?int $maxLinks = null,
         ?string $search = null,
         ?string $sitemapURL = null,
@@ -138,6 +146,7 @@ public static function with(
         $self['domain'] = $domain;
 
         null !== $headers && $self['headers'] = $headers;
+        null !== $includeSubdomains && $self['includeSubdomains'] = $includeSubdomains;
         null !== $maxLinks && $self['maxLinks'] = $maxLinks;
         null !== $search && $self['search'] = $search;
         null !== $sitemapURL && $self['sitemapURL'] = $sitemapURL;
@@ -173,6 +182,17 @@ public function withHeaders(array $headers): self
         return $self;
     }
 
+    /**
+     * When true, discover and include public pages and sitemaps on subdomains of the requested domain. Defaults to false.
+     */
+    public function withIncludeSubdomains(bool $includeSubdomains): self
+    {
+        $self = clone $this;
+        $self['includeSubdomains'] = $includeSubdomains;
+
+        return $self;
+    }
+
     /**
      * Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000.
      */

tests/Services/WebTest.php

@@ -412,6 +412,7 @@ public function testWebScrapeSitemapWithOptionalParams(): void
         $result = $this->client->web->webScrapeSitemap(
             domain: 'xxx',
             headers: ['foo' => 'J!'],
+            includeSubdomains: true,
             maxLinks: 1,
             search: 'help center and troubleshooting articles',
             sitemapURL: 'https://example.com',