Skip to content

Commit 534fd28

Browse files
YahiaBakourYahiaBakour
authored
Merge pull request #48 from context-dot-dev/docs/context-dev-readme-opening
README: lead with what the SDK does
2 parents f32b3e9 + 2907ebc commit 534fd28

1 file changed

Lines changed: 152 additions & 34 deletions

File tree

‎README.md‎

Lines changed: 152 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@
66

77
<!-- x-release-please-end -->
88

9-
The Context Dev Go library provides convenient access to the [Context Dev REST API](https://docs.context.dev/)
10-
from applications written in Go.
9+
Context.dev is a web scraping API for AI agents and LLMs. This SDK turns any URL into clean, LLM-ready markdown, crawls whole sites, searches the web, takes screenshots and extracts structured JSON against a schema you define, all with one API key. Proxies, JavaScript rendering and anti-bot handling run on Context.dev's side, so there is no headless browser to host.
1110

12-
It is generated with [Stainless](https://www.stainless.com/).
11+
The REST API documentation can be found on [docs.context.dev](https://docs.context.dev/).
1312

1413
## Installation
1514

@@ -39,6 +38,10 @@ This library requires Go 1.22+.
3938

4039
## Usage
4140

41+
Set `CONTEXT_DEV_API_KEY` to your API key; the client reads it automatically.
42+
43+
### Scrape markdown and HTML
44+
4245
The full API of this library can be found in [api.md](api.md).
4346

4447
```go
@@ -49,26 +52,142 @@ import (
4952
"fmt"
5053

5154
"github.com/context-dot-dev/context-go-sdk/v2"
52-
"github.com/context-dot-dev/context-go-sdk/v2/option"
5355
)
5456

5557
func main() {
56-
client := contextdev.NewClient(
57-
option.WithAPIKey("My API Key"), // defaults to os.LookupEnv("CONTEXT_DEV_API_KEY")
58-
)
59-
brand, err := client.Brand.Get(context.TODO(), contextdev.BrandGetParams{
60-
OfByDomain: &contextdev.BrandGetParamsBodyByDomain{
61-
Domain: "stripe.com",
58+
client := contextdev.NewClient()
59+
page, err := client.Web.Scrape(context.Background(), contextdev.WebScrapeParams{
60+
URL: "https://example.com",
61+
Formats: contextdev.WebScrapeParamsFormats{
62+
Markdown: contextdev.Bool(true),
63+
HTML: contextdev.Bool(true),
64+
},
65+
})
66+
if err != nil {
67+
panic(err)
68+
}
69+
fmt.Println(page.Markdown.Data)
70+
fmt.Println(page.HTML.Data)
71+
}
72+
```
73+
74+
### Extract structured JSON
75+
76+
```go
77+
package main
78+
79+
import (
80+
"context"
81+
"fmt"
82+
83+
"github.com/context-dot-dev/context-go-sdk/v2"
84+
)
85+
86+
func main() {
87+
client := contextdev.NewClient()
88+
page, err := client.Web.Scrape(context.Background(), contextdev.WebScrapeParams{
89+
URL: "https://example.com",
90+
Formats: contextdev.WebScrapeParamsFormats{
91+
Json: contextdev.Bool(true),
92+
},
93+
JsonParams: contextdev.WebScrapeParamsJsonParams{
94+
Schema: map[string]any{
95+
"type": "object",
96+
"properties": map[string]any{
97+
"title": map[string]any{"type": []string{"string", "null"}},
98+
"description": map[string]any{"type": []string{"string", "null"}},
99+
},
100+
"required": []string{"title", "description"},
101+
"additionalProperties": false,
102+
},
103+
},
104+
})
105+
if err != nil {
106+
panic(err)
107+
}
108+
fmt.Printf("%+v\n", page.Json.Data)
109+
}
110+
```
111+
112+
### Extract relevant highlights
113+
114+
Return the passages that answer a question about the page.
115+
116+
```go
117+
package main
118+
119+
import (
120+
"context"
121+
"fmt"
122+
123+
"github.com/context-dot-dev/context-go-sdk/v2"
124+
)
125+
126+
func main() {
127+
client := contextdev.NewClient()
128+
page, err := client.Web.Scrape(context.Background(), contextdev.WebScrapeParams{
129+
URL: "https://example.com",
130+
Formats: contextdev.WebScrapeParamsFormats{
131+
Highlights: contextdev.Bool(true),
132+
},
133+
HighlightsParams: contextdev.WebScrapeParamsHighlightsParams{
134+
Query: "What is this domain used for?",
62135
},
63136
})
64137
if err != nil {
65-
panic(err.Error())
138+
panic(err)
66139
}
67-
fmt.Printf("%+v\n", brand.RequestID)
140+
fmt.Println(page.Highlights.Data)
68141
}
142+
```
143+
144+
### Take a screenshot
145+
146+
The screenshot is returned as a base64 image data URL.
147+
148+
```go
149+
package main
150+
151+
import (
152+
"context"
153+
"fmt"
69154

155+
"github.com/context-dot-dev/context-go-sdk/v2"
156+
)
157+
158+
func main() {
159+
client := contextdev.NewClient()
160+
page, err := client.Web.Scrape(context.Background(), contextdev.WebScrapeParams{
161+
URL: "https://example.com",
162+
Formats: contextdev.WebScrapeParamsFormats{
163+
Screenshot: contextdev.Bool(true),
164+
},
165+
})
166+
if err != nil {
167+
panic(err)
168+
}
169+
fmt.Println(page.Screenshot.Data)
170+
}
70171
```
71172

173+
## What you can do
174+
175+
| Task | Method |
176+
| --- | --- |
177+
| Scrape a URL to markdown, HTML, JSON, highlights or a screenshot | `client.Web.Scrape` |
178+
| Crawl a site and get every page as markdown | `client.Web.WebCrawlMd` |
179+
| Map every URL on a domain | `client.Web.MapURLs` |
180+
| Search the web | `client.Web.Search` |
181+
| Take a screenshot of a page | `client.Web.Screenshot` |
182+
| Parse PDFs and documents | `client.Parse.Handle` |
183+
| Run thousands of URLs as a batch | `client.Batch.Submit` |
184+
| Watch a page for changes | `client.Monitors.New` |
185+
| Look up a company's logo, colors and brand data | `client.Brand.Get` |
186+
187+
## Use it from an AI agent
188+
189+
Context.dev also ships as a plugin for [Claude](https://github.com/context-dot-dev/claude-plugin), [Cursor](https://github.com/context-dot-dev/cursor-plugin) and [Gemini CLI](https://github.com/context-dot-dev/gemini-cli-context), and as tools for [LangChain](https://github.com/context-dot-dev/langchain-context) and [Haystack](https://github.com/context-dot-dev/context-haystack).
190+
72191
### Request fields
73192

74193
The contextdev library uses the [`omitzero`](https://tip.golang.org/doc/go1.24#encodingjsonpkgencodingjson)
@@ -270,7 +389,10 @@ client := contextdev.NewClient(
270389
option.WithHeader("X-Some-Header", "custom_header_info"),
271390
)
272391

273-
client.Brand.Get(context.TODO(), ...,
392+
client.Web.Scrape(context.TODO(), contextdev.WebScrapeParams{
393+
URL: "https://example.com",
394+
Formats: contextdev.WebScrapeParamsFormats{Markdown: contextdev.Bool(true)},
395+
},
274396
// Override the header
275397
option.WithHeader("X-Some-Header", "some_other_custom_header_info"),
276398
// Add an undocumented field to the request body, using sjson syntax
@@ -301,18 +423,17 @@ When the API returns a non-success status code, we return an error with type
301423
To handle errors, we recommend that you use the `errors.As` pattern:
302424

303425
```go
304-
_, err := client.Brand.Get(context.TODO(), contextdev.BrandGetParams{
305-
OfByDomain: &contextdev.BrandGetParamsBodyByDomain{
306-
Domain: "stripe.com",
307-
},
426+
_, err := client.Web.Scrape(context.TODO(), contextdev.WebScrapeParams{
427+
URL: "https://example.com",
428+
Formats: contextdev.WebScrapeParamsFormats{Markdown: contextdev.Bool(true)},
308429
})
309430
if err != nil {
310431
var apierr *contextdev.Error
311432
if errors.As(err, &apierr) {
312433
println(string(apierr.DumpRequest(true))) // Prints the serialized HTTP request
313434
println(string(apierr.DumpResponse(true))) // Prints the serialized HTTP response
314435
}
315-
panic(err.Error()) // GET "/brand/retrieve": 400 Bad Request { ... }
436+
panic(err.Error())
316437
}
317438
```
318439

@@ -330,12 +451,11 @@ To set a per-retry timeout, use `option.WithRequestTimeout()`.
330451
// This sets the timeout for the request, including all the retries.
331452
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
332453
defer cancel()
333-
client.Brand.Get(
454+
client.Web.Scrape(
334455
ctx,
335-
contextdev.BrandGetParams{
336-
OfByDomain: &contextdev.BrandGetParamsBodyByDomain{
337-
Domain: "stripe.com",
338-
},
456+
contextdev.WebScrapeParams{
457+
URL: "https://example.com",
458+
Formats: contextdev.WebScrapeParamsFormats{Markdown: contextdev.Bool(true)},
339459
},
340460
// This sets the per-retry timeout
341461
option.WithRequestTimeout(20*time.Second),
@@ -370,12 +490,11 @@ client := contextdev.NewClient(
370490
)
371491

372492
// Override per-request:
373-
client.Brand.Get(
493+
client.Web.Scrape(
374494
context.TODO(),
375-
contextdev.BrandGetParams{
376-
OfByDomain: &contextdev.BrandGetParamsBodyByDomain{
377-
Domain: "stripe.com",
378-
},
495+
contextdev.WebScrapeParams{
496+
URL: "https://example.com",
497+
Formats: contextdev.WebScrapeParamsFormats{Markdown: contextdev.Bool(true)},
379498
},
380499
option.WithMaxRetries(5),
381500
)
@@ -389,19 +508,18 @@ you need to examine response headers, status codes, or other details.
389508
```go
390509
// Create a variable to store the HTTP response
391510
var response *http.Response
392-
brand, err := client.Brand.Get(
511+
page, err := client.Web.Scrape(
393512
context.TODO(),
394-
contextdev.BrandGetParams{
395-
OfByDomain: &contextdev.BrandGetParamsBodyByDomain{
396-
Domain: "stripe.com",
397-
},
513+
contextdev.WebScrapeParams{
514+
URL: "https://example.com",
515+
Formats: contextdev.WebScrapeParamsFormats{Markdown: contextdev.Bool(true)},
398516
},
399517
option.WithResponseInto(&response),
400518
)
401519
if err != nil {
402520
// handle error
403521
}
404-
fmt.Printf("%+v\n", brand)
522+
fmt.Printf("%+v\n", page)
405523

406524
fmt.Printf("Status Code: %d\n", response.StatusCode)
407525
fmt.Printf("Headers: %+#v\n", response.Header)

0 commit comments

Comments
 (0)