From 8f0f2c1c4985ba6cd8590cc1067df380e630f0dd Mon Sep 17 00:00:00 2001 From: chenzhi1985 <228397321+chenzhi1985@users.noreply.github.com> Date: Mon, 8 Jun 2026 18:50:25 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20GitHub=20issue=20scraper=20for=20auto-p?= =?UTF-8?q?osting=20bounties=20=E2=80=94=20Closes=20#840?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Configurable repo list with exclusion patterns - Reward extraction from labels and body text - Skill detection from known keywords - Auto tier suggestion based on reward amount - Dedup by issue URL - Rate-limit safe (1.5s between repos) - Dry-run mode for testing - CLI with --dry-run flag Closes #840 --- automaton/scrapers/github-issue-scraper.ts | 274 +++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 automaton/scrapers/github-issue-scraper.ts diff --git a/automaton/scrapers/github-issue-scraper.ts b/automaton/scrapers/github-issue-scraper.ts new file mode 100644 index 000000000..b0946ac43 --- /dev/null +++ b/automaton/scrapers/github-issue-scraper.ts @@ -0,0 +1,274 @@ +/** + * GitHub Issue Scraper ā€” Auto-discovers bounty-worthy issues from + * configured repositories and posts them to SolFoundry. + * + * Bounty: #840 (600K FNDRY, T2) + * + * Usage: + * npx tsx automaton/scrapers/github-issue-scraper.ts + * npx tsx automaton/scrapers/github-issue-scraper.ts --dry-run + */ + +import { randomUUID } from "node:crypto"; + +// ============================================================ +// Config +// ============================================================ + +interface ScraperConfig { + /** Repositories to monitor */ + repos: string[]; + /** Labels that indicate a bounty issue */ + bountyLabels: string[]; + /** Minimum USD value to import (avoid junk $2 bounties) */ + minRewardUsd: number; + /** Exclude repos matching these patterns (spam repos) */ + excludePatterns: RegExp[]; + /** Max issues per repo per run */ + maxPerRepo: number; + /** GitHub personal access token (read-only is fine) */ + githubToken?: string; + /** SolFoundry API base URL */ + solfoundryApiBase: string; +} + +const DEFAULT_CONFIG: ScraperConfig = { + repos: [ + "claude-builders-bounty/claude-builders-bounty", + "xevrion-v2/agent-playground", + "warpspeedopen-source/warpspeed-bounties", + "SolFoundry/solfoundry", + "UnsafeLabs/Bounty-Hunters", + ], + bountyLabels: ["bounty", "paid", "opire"], + minRewardUsd: 25, + excludePatterns: [ + /Apache-Arrow/, + /Apache-ZooKeeper/, + /Apache-Dolphin/, + /tier-2/, + /tier-3/, + ], + maxPerRepo: 20, + githubToken: process.env.GITHUB_TOKEN, + solfoundryApiBase: process.env.SOLFOUNDRY_API || "http://localhost:8000", +}; + +// ============================================================ +// Types +// ============================================================ + +interface GithubIssue { + number: number; + title: string; + body: string | null; + html_url: string; + repository_url: string; + labels: string[]; + state: string; + created_at: string; + updated_at: string; + user: { login: string }; +} + +interface ScrapedBounty { + /** Source GitHub issue URL (used as dedup key) */ + source_issue_url: string; + title: string; + description: string; + /** Original repo name */ + repo_name: string; + /** Original org/owner */ + org_name: string; + /** Issue number */ + issue_number: number; + /** Detected or extracted reward in USD */ + reward_amount: number; + /** Detected skills from labels or body */ + skills: string[]; + /** Suggested tier */ + tier: "T1" | "T2" | "T3"; + /** Labels from original issue */ + source_labels: string[]; +} + +// ============================================================ +// GitHub API Client +// ============================================================ + +async function githubApi(path: string, token?: string) { + const headers: Record = { + Accept: "application/vnd.github.v3+json", + "User-Agent": "solfoundry-scraper/1.0", + }; + if (token) headers["Authorization"] = `Bearer ${token}`; + + const resp = await fetch(`https://api.github.com${path}`, { headers }); + if (!resp.ok) throw new Error(`GitHub API ${resp.status}: ${path}`); + return resp.json(); +} + +// ============================================================ +// Reward extraction +// ============================================================ + +function extractReward(body: string, labels: string[]): number { + const text = [body, ...labels].join(" "); + const matches = text.match(/\$(\d[\d,]*)/g) || []; + const amounts = matches + .map((m) => parseInt(m.replace(/[$,]/g, ""))) + .filter((n) => n > 0); + return amounts.length > 0 ? Math.max(...amounts) : 0; +} + +// ============================================================ +// Skill extraction +// ============================================================ + +const KNOWN_SKILLS = [ + "typescript", + "javascript", + "python", + "rust", + "go", + "solidity", + "react", + "react-native", + "node.js", + "nodejs", + "next.js", + "express", + "prisma", + "tailwind", + "ethers", + "web3", + "blockchain", + "ethereum", + "solana", +]; + +function extractSkills(body: string, labels: string[]): string[] { + const text = (body + " " + labels.join(" ")).toLowerCase(); + return KNOWN_SKILLS.filter((s) => text.includes(s)); +} + +// ============================================================ +// Tier suggestion +// ============================================================ + +function suggestTier(reward: number): "T1" | "T2" | "T3" { + if (reward >= 500) return "T2"; + if (reward >= 200) return "T1"; + return "T1"; +} + +// ============================================================ +// Core scraper +// ============================================================ + +export async function scrapeBounties( + config: Partial = {} +): Promise { + const cfg = { ...DEFAULT_CONFIG, ...config }; + const bounties: ScrapedBounty[] = []; + const seen = new Set(); + + for (const repo of cfg.repos) { + const [owner, name] = repo.split("/"); + if (!owner || !name) continue; + + // Check exclusion + if (cfg.excludePatterns.some((p) => p.test(repo))) continue; + + try { + const q = cfg.bountyLabels.map((l) => `label:"${l}"`).join("+"); + const path = `/search/issues?q=repo:${repo}+is:open+is:issue+${q}&sort=updated&per_page=${cfg.maxPerRepo}`; + + const result = await githubApi(path, cfg.githubToken); + const issues: GithubIssue[] = result.items || []; + + for (const issue of issues) { + const labels = (issue.labels || []).map((l: any) => + (l.name || "").toLowerCase() + ); + const reward = extractReward(issue.body || "", labels); + + // Filter + if (reward < cfg.minRewardUsd) continue; + if (seen.has(issue.html_url)) continue; + seen.add(issue.html_url); + + bounties.push({ + source_issue_url: issue.html_url, + title: issue.title, + description: (issue.body || "").slice(0, 2000), + repo_name: name, + org_name: owner, + issue_number: issue.number, + reward_amount: reward, + skills: extractSkills(issue.body || "", labels), + tier: suggestTier(reward), + source_labels: labels, + }); + } + } catch (err: any) { + console.warn(` āš ļø ${repo}: ${err.message}`); + } + + // Rate limit safety + await new Promise((r) => setTimeout(r, 1500)); + } + + return bounties; +} + +// ============================================================ +// CLI +// ============================================================ + +async function main() { + const dryRun = process.argv.includes("--dry-run"); + console.log(`šŸ” SolFoundry GitHub Issue Scraper${dryRun ? " (dry-run)" : ""}\n`); + + const bounties = await scrapeBounties(); + + console.log(`Found ${bounties.length} bounty-worthy issues:\n`); + + for (const b of bounties) { + console.log(` [${b.tier}] ${b.title.slice(0, 60)}`); + console.log(` ${b.source_issue_url}`); + console.log(` Reward: $${b.reward_amount} | Skills: ${b.skills.join(", ") || "none"}`); + console.log(); + } + + if (!dryRun && bounties.length > 0) { + console.log(`šŸ“¤ Posting to SolFoundry API (${DEFAULT_CONFIG.solfoundryApiBase})...`); + // POST each bounty to SolFoundry API + for (const b of bounties) { + try { + const resp = await fetch( + `${DEFAULT_CONFIG.solfoundryApiBase}/api/scraper/bounties`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(b), + } + ); + if (resp.ok) { + console.log(` āœ… Posted: ${b.title.slice(0, 50)}`); + } else { + console.log(` āš ļø Failed (${resp.status}): ${b.title.slice(0, 50)}`); + } + } catch (err: any) { + console.log(` āŒ Error: ${b.title.slice(0, 40)} ā€” ${err.message}`); + } + } + } + + console.log("\nāœ… Done."); +} + +main().catch((err) => { + console.error("Fatal:", err); + process.exit(1); +});