feat(security): Add comprehensive secret detection system

.github/workflows/secret-scanning.yaml

@@ -0,0 +1,131 @@
+name: Secret Scanning
+
+on:
+  push:
+    branches: [ main, master, develop ]
+  pull_request:
+    branches: [ main, master, develop ]
+  workflow_dispatch:
+
+jobs:
+  gitleaks:
+    name: Scan for Secrets with Gitleaks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history for complete scanning
+
+      - name: Check if Gitleaks license is available
+        id: check-license
+        run: |
+          if [ -n "${{ secrets.GITLEAKS_LICENSE }}" ]; then
+            echo "has_license=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_license=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run Gitleaks
+        if: steps.check-license.outputs.has_license == 'true'
+        uses: gitleaks/gitleaks-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }}
+        with:
+          config-path: .gitleaks.toml
+
+      - name: Skip Gitleaks (no license)
+        if: steps.check-license.outputs.has_license == 'false'
+        run: |
+          echo "WARNING: Gitleaks scan skipped - GITLEAKS_LICENSE secret not configured"
+          echo "To enable Gitleaks scanning, add a license key as a repository secret"
+          echo "Visit https://gitleaks.io for licensing information"
+
+  detect-secrets:
+    name: Scan for Secrets with detect-secrets
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install detect-secrets
+        run: |
+          pip install detect-secrets
+
+      - name: Run detect-secrets scan
+        run: |
+          detect-secrets scan --all-files \
+            --exclude-files '\.git/.*' \
+            --exclude-files '\.secrets\.baseline' \
+            --exclude-files 'package-lock\.json' \
+            --exclude-files '.*\.lock' \
+            > .secrets.baseline.new
+
+      - name: Compare with baseline
+        run: |
+          if [ -f .secrets.baseline ]; then
+            # Check if there are any results in the new scan by counting secrets
+            NEW_COUNT=$(jq '.results | to_entries | length' .secrets.baseline.new)
+            OLD_COUNT=$(jq '.results | to_entries | length' .secrets.baseline)
+
+            echo "Baseline secrets count: $OLD_COUNT"
+            echo "New scan secrets count: $NEW_COUNT"
+
+            if [ "$NEW_COUNT" -gt "$OLD_COUNT" ]; then
+              echo "ERROR: New secrets detected ($NEW_COUNT vs $OLD_COUNT)"
+              echo "New secrets found:"
+              jq '.results' .secrets.baseline.new
+              exit 1
+            else
+              echo "No new secrets detected"
+            fi
+            rm .secrets.baseline.new
+          else
+            echo "No baseline found, creating new baseline"
+            mv .secrets.baseline.new .secrets.baseline
+          fi
+
+  trufflehog:
+    name: Scan for Secrets with TruffleHog
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: TruffleHog OSS
+        uses: trufflesecurity/trufflehog@main
+        with:
+          path: ./
+          base: ${{ github.event.repository.default_branch }}
+          head: HEAD
+          extra_args: --debug --only-verified
+
+  summary:
+    name: Secret Scanning Summary
+    runs-on: ubuntu-latest
+    needs: [gitleaks, detect-secrets, trufflehog]
+    if: always()
+    steps:
+      - name: Check results
+        run: |
+          echo "Secret scanning completed"
+          echo "Gitleaks: ${{ needs.gitleaks.result }}"
+          echo "detect-secrets: ${{ needs.detect-secrets.result }}"
+          echo "TruffleHog: ${{ needs.trufflehog.result }}"
+
+      - name: Fail if secrets detected
+        if: |
+          (needs.gitleaks.result == 'failure') ||
+          (needs.detect-secrets.result == 'failure') ||
+          (needs.trufflehog.result == 'failure')
+        run: |
+          echo "ERROR: Secrets detected! Please remove them before merging."
+          exit 1

.gitleaks.toml

@@ -0,0 +1,69 @@
+# Gitleaks configuration for detecting secrets in commits
+# https://github.com/gitleaks/gitleaks
+
+title = "Gitleaks Configuration for Teal Agents"
+
+[extend]
+# Use the default gitleaks rules as a base
+useDefault = true
+
+[allowlist]
+description = "Allowlist for known false positives"
+paths = [
+    '''\.secrets\.baseline$''',
+    '''\.pre-commit-config\.yaml$''',
+    '''tests/.*fixtures.*''',
+    '''examples/.*test.*''',
+]
+
+# Example patterns to ignore (update as needed)
+regexes = [
+    '''(example|sample|test|mock|fake|dummy)(_)?key''',
+]
+
+# Custom rules for organization-specific patterns
+[[rules]]
+id = "organization-api-key"
+description = "Organization API Key"
+regex = '''(?i)(org|internal)[_-]?api[_-]?key['"]?\s*[:=]\s*['"]?[a-zA-Z0-9]{20,}'''
+tags = ["key", "organization", "api"]
+
+[[rules]]
+id = "openai-api-key"
+description = "OpenAI API Key"
+regex = '''sk-[a-zA-Z0-9]{20,}'''
+tags = ["key", "openai"]
+
+[[rules]]
+id = "anthropic-api-key"
+description = "Anthropic API Key"
+regex = '''sk-ant-[a-zA-Z0-9-_]{95,}'''
+tags = ["key", "anthropic"]
+
+[[rules]]
+id = "azure-key"
+description = "Azure API Key"
+regex = '''(?i)azure[_-]?key['"]?\s*[:=]\s*['"]?[a-zA-Z0-9]{32,}'''
+tags = ["key", "azure"]
+
+[[rules]]
+id = "generic-api-key"
+description = "Generic API Key"
+regex = '''(?i)api[_-]?key['"]?\s*[:=]\s*['"]?[a-zA-Z0-9]{20,}'''
+tags = ["key", "api"]
+[rules.allowlist]
+regexes = [
+    '''(example|test|sample|mock|dummy|placeholder)''',
+]
+
+[[rules]]
+id = "bearer-token"
+description = "Bearer Token"
+regex = '''(?i)bearer\s+[a-zA-Z0-9\-_=]+\.[a-zA-Z0-9\-_=]+\.?[a-zA-Z0-9\-_=]*'''
+tags = ["token", "bearer"]
+
+[[rules]]
+id = "private-key"
+description = "Private Key"
+regex = '''-----BEGIN\s+(RSA|EC|DSA|OPENSSH)?\s?PRIVATE KEY-----'''
+tags = ["key", "private"]

.pre-commit-config.yaml

@@ -1,26 +1,52 @@
 repos:
+  # Secret Detection - Prevent API keys and secrets from being committed
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
+    hooks:
+      - id: detect-secrets
+        args: ['--baseline', '.secrets.baseline']
+        exclude: package.lock.json
+
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.21.2
+    hooks:
+      - id: gitleaks
+
+  # Standard pre-commit hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: check-merge-conflict
+      - id: check-yaml
+        exclude: ^\.github/workflows/
+      - id: detect-private-key
+      - id: no-commit-to-branch
+        args: ['--branch', 'main', '--branch', 'master']
+
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.33.0
     hooks:
       - id: check-github-workflows
         args: ["--verbose"]
+
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.11.8"
     hooks:
       - id: ruff-format
       - id: ruff
         args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"]
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:
       - id: check-toml
       - id: trailing-whitespace
         exclude: ^docs.*\.md$
+
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.29.0
     hooks: