diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 73e527ca23..2905d573d5 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -18,7 +18,9 @@ "./skills/firecrawl-monitor", "./skills/firecrawl-parse", "./skills/firecrawl-scrape", - "./skills/firecrawl-search" + "./skills/firecrawl-search", + "./skills/firecrawl-developer-index", + "./skills/firecrawl-research-index" ] } ] diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 499f8cd18b..51eda56822 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -15,6 +15,8 @@ "./skills/firecrawl-monitor", "./skills/firecrawl-parse", "./skills/firecrawl-scrape", - "./skills/firecrawl-search" + "./skills/firecrawl-search", + "./skills/firecrawl-developer-index", + "./skills/firecrawl-research-index" ] } diff --git a/skills/firecrawl-developer-index/SKILL.md b/skills/firecrawl-developer-index/SKILL.md new file mode 100644 index 0000000000..f804e24dfa --- /dev/null +++ b/skills/firecrawl-developer-index/SKILL.md @@ -0,0 +1,41 @@ +--- +name: firecrawl-developer-index +description: | + Search issues, merged pull requests, READMEs, and documentation. Use when the question is how a library or API behaves, what an error means, or whether a bug was fixed; prefer this over a general web page. +allowed-tools: + - Bash(firecrawl *) + - Bash(npx firecrawl-cli *) +--- + +# firecrawl developer + +Answer a developer question from the primary source: the issue, the merged pull request that fixed it, or the README/docs passage that states the contract. + +## Quick start + +```bash +mkdir -p .firecrawl +firecrawl developer "how do I configure retries" --limit 10 -o .firecrawl/developer.json --json +jq -r '.results[] | .id, .url, .passages[].text' .firecrawl/developer.json +``` + +Run `firecrawl developer --help` for the full option list. + +HTTP: `GET|POST https://api.firecrawl.dev/v2/search/developer`. MCP: `firecrawl_developer_search`. Each hit carries `id`, `url`, `passages`. Kind is the `id` prefix (`doc:`, `issue:`, `pull_request:`, `readme:`). Hits do not carry a `type` field. + +**Done when:** the answer quotes a matched passage and cites its `url` (fall back to `url` when `title` is absent), or you have moved to the open web because the index had nothing to say. + +## Tips + +- Default first move is `firecrawl developer`. Use `search --categories developer` only when you are already running a web search and want developer hits in the same call (no passage control, no index filters). +- Literal error or stack trace: search the string plus the library name. On HTTP, `types=["issue","pull_request"]`. Strip paths, line numbers, and ids, then retry. +- API contract: `readme` and `doc` are authoritative. A merged PR supersedes an issue report. Never answer from an opening report alone. +- Scope last: search the whole index, then narrow with HTTP `types`, `repos`, or `sources`. If a scoped search is empty, read the echoed `indexed` flag before concluding the repo is missing. +- Repository filters (`language`, `topic`, `license`, `min_stars`, …) drop `doc` results unless you also pass `sources`. `types`, `repos`, `sources`, `passages`, and those repository filters are HTTP-only. +- Comparison, opinion, news, or an unindexed project: `firecrawl search`, then `firecrawl scrape`. + +## See also + +- [firecrawl-search](../firecrawl-search/SKILL.md) — open web, or `search --categories developer` in the same call +- [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — full page when a hit is right but you need all of it +- [firecrawl-research-index](../firecrawl-research-index/SKILL.md) — papers, not this index diff --git a/skills/firecrawl-research-index/SKILL.md b/skills/firecrawl-research-index/SKILL.md new file mode 100644 index 0000000000..77dc4aa881 --- /dev/null +++ b/skills/firecrawl-research-index/SKILL.md @@ -0,0 +1,43 @@ +--- +name: firecrawl-research-index +description: | + Find papers in Firecrawl's research paper index (PubMed, bioRxiv, medRxiv, arXiv). Use for literature-finding of any kind, including clinical and biomedical questions; `search --categories research` is a website filter, not this index. +allowed-tools: + - Bash(firecrawl *) + - Bash(npx firecrawl-cli *) +--- + +# firecrawl research + +Find the papers that answer a research query. When in doubt, return the relevant set (most relevant first) rather than one hit. + +## Quick start + +```bash +mkdir -p .firecrawl +firecrawl research search-papers "CRISPR base editing off-target effects" \ + --limit 20 -o .firecrawl/papers.json --json +jq -r '.results[] | .primaryId, .title' .firecrawl/papers.json +``` + +Run `firecrawl research --help` for flags. MCP arguments use `paperId`, not `id`. + +A successful `search-papers` response is `{success, results}`. Each hit carries `paperId`, `primaryId` (`pmid:`, `pmcid:`, `doi:`, or `arxiv:`), `ids`, `title`, `abstract`, and `score`. + +**Done when:** the answer is a cited paper set (or the one named paper), each kept or dropped against a verified constraint, with `search-papers` as the first move unless the query already named an id. + +## Tips + +- `search-papers` is the first move. If results look thin or all-alike, re-run with a different framing (sibling domain, rival method, dataset/benchmark name). +- `related-papers` needs `--intent`. `mode=similar` for siblings, `citers` for who builds on the seeds, `references` for what they build on. +- `inspect-paper` is metadata for one id. `read-paper` is in-body passages for one constraint (sample size, method, affiliation). Use it to rule a paper out, not to gatekeep. +- `search --categories research` is a website filter. It returns pages from academic domains, not paper records in this index. +- Named paper ("the Qwen3 report") → one `search-papers`. Method / family / "papers that do X" → expand with `related-papers` and keep neighbors. +- Superlative / leaderboard questions live on the web: `firecrawl search` / `firecrawl scrape`, then `search-papers` each top entry. +- PubMed, bioRxiv, and medRxiv are the largest part of the corpus. Do not send a biomedical query to the open web on the assumption the index is arXiv-only. + +## See also + +- [firecrawl-search](../firecrawl-search/SKILL.md) — web pages, including `search --categories research` +- [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — leaderboards and other non-paper pages +- [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) — issues, PRs, READMEs, and docs diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 0c1f6687f1..d4c50ecc13 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -1,7 +1,7 @@ --- name: firecrawl-search description: | - Web search with full page content, plus a research-paper index (PubMed, arXiv, bioRxiv, medRxiv). Use when no URL is known: finding sources, articles, news, or scientific literature. + Web search with full page content. Use when no URL is known: finding sources, articles, or news. For papers use firecrawl-research-index; for library, API, error, or bug questions use firecrawl-developer-index. allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl-cli *) @@ -9,7 +9,7 @@ allowed-tools: # firecrawl search -Web search with optional content scraping. Returns search results as JSON, optionally with full page content. For research papers, route to [Paper search](#paper-search) (`firecrawl research`). +Web search with optional content scraping. Returns search results as JSON, optionally with full page content. ## Quick start @@ -22,73 +22,13 @@ firecrawl search "your query" --scrape -o .firecrawl/scraped.json --json # News from the past day firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json --json - -# Programming question: search GitHub issues, merged PRs, READMEs, and docs -firecrawl search "your query" --categories developer -o .firecrawl/developer.json --json - -# Research papers: use the paper index (`research`, not `search --categories research`) -firecrawl research search-papers "your query" -o .firecrawl/papers.json --json ``` Run `firecrawl search --help` for the full option list. -**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). - -## Developer search - -`--categories developer` adds an index built for coding agents. It covers GitHub -issues, merged pull requests, repository READMEs, and curated documentation -sites. Use it for a programming question: an error message, an API contract, a -library behaviour, or a known bug. - -The hits arrive in their own `data.developer` group beside `data.web`. Each hit -holds `url`, `title`, and `description`, where `description` is the matched -passage. Read the passages with -`jq -r '.data.developer[] | .url, .description' .firecrawl/developer.json`. - -The dedicated `firecrawl developer` command searches only that index and keeps -the full matched passages: - -```bash -# Developer search only, with full passages -firecrawl developer "your query" --limit 10 -o .firecrawl/developer.json --json -``` - -Each result holds `id`, `type` (`issue`, `pull_request`, `readme`, `doc`), -`url`, `title`, and `passages`. Read them with -`jq -r '.results[] | .url, .passages[].text' .firecrawl/developer.json`. +`--categories developer` weighs the developer index beside ordinary web results in this same call (no passage control, no index filters). `--categories research` is a website filter, not the paper index. Dedicated skills: [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) and [firecrawl-research-index](../firecrawl-research-index/SKILL.md). -## Paper search - -For actual papers use the `firecrawl research` command group, which searches -roughly 43M abstracts, around 90% biomedical (PubMed, bioRxiv, medRxiv) plus -arXiv. (`search --categories research` is a website filter — it only narrows -ordinary web results to research-affiliated sites.) - -Reach for it on any biomedical, clinical, or scientific-literature question — -it replaces web-searching or scraping PubMed, bioRxiv, medRxiv, or Google -Scholar by hand: - -```bash -# Find papers by topic -- start here, and run several distinct framings -firecrawl research search-papers "CRISPR base editing off-target effects" \ - --limit 20 -o .firecrawl/papers.json --json - -# Expand from your strongest hits along the citation graph -firecrawl research related-papers pmid:40953549 --intent "in vivo delivery" \ - -o .firecrawl/papers-related.json --json - -# Verify a specific claim against the full text before you cite it -firecrawl research read-paper pmcid:PMC12530322 --question "What was the sample size?" \ - -o .firecrawl/paper-passages.json --json -``` - -Paper ids accept `pmid:`, `pmcid:`, `doi:`, and `arxiv:` forms. `inspect-paper` -returns canonical metadata for one id. Read hits with -`jq -r '.results[] | .primaryId, .title' .firecrawl/papers.json`. - -See [firecrawl](../firecrawl/SKILL.md) for how paper search fits the -overall command routing. +**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). ## Tips @@ -145,3 +85,5 @@ fi - [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — scrape a specific URL - [firecrawl-map](../firecrawl-map/SKILL.md) — discover URLs within a site - [firecrawl-crawl](../firecrawl-crawl/SKILL.md) — bulk extract from a site +- [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) — issues, merged PRs, READMEs, and docs +- [firecrawl-research-index](../firecrawl-research-index/SKILL.md) — published papers, not `search --categories research` diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index 31070e181b..163e110877 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -1,7 +1,7 @@ --- name: firecrawl description: | - Any live-web task via the Firecrawl CLI — including ordinary web research: searching the web, reading or extracting pages, gathering sources, discovering site URLs, bulk extraction, downloading a site, change alerts, or pages needing clicks/login — web only; local files route to firecrawl-parse. + Any live-web task via the Firecrawl CLI — including ordinary web research: searching the web, reading or extracting pages, gathering sources, discovering site URLs, bulk extraction, downloading a site, change alerts, or pages needing clicks/login — web only; local files route to firecrawl-parse. For papers use firecrawl-research-index; for library, API, error, or bug questions use firecrawl-developer-index. allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl-cli *) @@ -34,6 +34,7 @@ Follow this escalation pattern: | --------------------------- | --------------------- | --------------------------------------------------------------- | | Find pages on a topic | `search` | No specific URL yet | | Find research papers | `research` | Biomedical/clinical/scientific literature — use the paper index | +| Answer a coding question | `developer` | Issues, merged PRs, READMEs, and docs — not a general web page | | Get a page's content | `scrape` | Have a URL, page is static or JS-rendered | | Find URLs within a site | `map` | Need to locate a specific subpage | | Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | @@ -63,7 +64,8 @@ For detailed command reference, run `firecrawl --help`. ## When to Load References - **Searching the web or finding sources first** -> [firecrawl-search](../firecrawl-search/SKILL.md) -- **Finding research papers (biomedical, clinical, or scientific literature; PubMed, bioRxiv, medRxiv, arXiv)** -> `firecrawl research search-papers`, documented in [firecrawl-search](../firecrawl-search/SKILL.md). Use the paper index instead of scraping PubMed or Google Scholar by hand; `search --categories research` is a website filter, not the paper index. +- **Finding research papers (biomedical, clinical, or scientific literature; PubMed, bioRxiv, medRxiv, arXiv)** -> [firecrawl-research-index](../firecrawl-research-index/SKILL.md). Use the paper index instead of scraping PubMed or Google Scholar by hand; `search --categories research` is a website filter, not the paper index. +- **Answering a library, API, error, or known-bug question from issues, merged PRs, READMEs, or docs** -> [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) - **Scraping a known URL** -> [firecrawl-scrape](../firecrawl-scrape/SKILL.md) - **Finding URLs on a known site** -> [firecrawl-map](../firecrawl-map/SKILL.md) - **Bulk extraction from a docs section or site** -> [firecrawl-crawl](../firecrawl-crawl/SKILL.md) diff --git a/src/index.ts b/src/index.ts index 87ecfee452..5a00c1e958 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1054,7 +1054,7 @@ function createSearchCommand(): Command { function createDeveloperCommand(): Command { const developerCmd = new Command('developer') .description( - 'Search an index built for coding agents: GitHub issues, merged PRs, repository READMEs, and curated documentation sites. Use it for a programming question: code behaviour, a library or framework, an API contract, an error message, or a known bug. Returns ranked results with id, type, url, title, and the matched passages in markdown.' + 'Search an index built for coding agents: issues, merged PRs, repository READMEs, and curated documentation sites. Use it for a programming question: code behaviour, a library or framework, an API contract, an error message, or a known bug. Returns ranked results with id, url, title, and the matched passages in markdown. Kind is the id prefix (doc:, issue:, pull_request:, readme:).' ) .argument('', 'Natural-language developer question or search phrase') .option(