Repository navigation
Build Notion Index #124
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Notion Index | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| full_rebuild: | |
| description: 'Ignore the previous index and re-fetch + re-embed everything' | |
| type: boolean | |
| default: false | |
| schedule: | |
| - cron: '0 4 * * *' # 4 AM UTC daily | |
| permissions: | |
| contents: write # required to create/update releases | |
| jobs: | |
| build-index: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: PERBot | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '24' | |
| cache: 'npm' | |
| cache-dependency-path: PERBot/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| # Incremental build: the indexer downloads the previous release's index, embeddings and | |
| # raw-markdown cache, re-fetches only pages whose last_edited_time changed, and re-embeds | |
| # only chunks whose text changed. A full rebuild (input above) skips that. | |
| - name: Build Notion index | |
| env: | |
| NOTION_TOKEN: ${{ secrets.NOTION_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # Google Shared Drive source (service account key JSON, raw or base64). Unset ⇒ Notion only. | |
| GDRIVE_SERVICE_ACCOUNT_JSON: ${{ secrets.GDRIVE_SERVICE_ACCOUNT_JSON }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| # Slack tokens are required by config.ts at load time but unused by the indexer | |
| SLACK_BOT_TOKEN: unused | |
| SLACK_APP_TOKEN: unused | |
| CURRENT_REV: REV12 | |
| INDEX_INCREMENTAL: ${{ inputs.full_rebuild && 'false' || 'true' }} | |
| INDEX_PATH: ./data/index.json | |
| EMBEDDINGS_PATH: ./data/embeddings.i8 | |
| INDEX_BUILD_CACHE_PATH: ./data/build-cache.json | |
| INDEX_STATUS_PATH: ./data/index-status.json | |
| NODE_OPTIONS: --max-old-space-size=4096 | |
| run: npm run index | |
| - name: Compress artifacts | |
| run: | | |
| gzip -9 -f data/index.json | |
| gzip -9 -f data/embeddings.i8 | |
| gzip -9 -f data/build-cache.json | |
| - name: Ensure release exists | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh release view notion-index-latest 2>/dev/null || \ | |
| gh release create notion-index-latest \ | |
| --title "Notion Search Index" \ | |
| --notes "Pre-built Notion search index, updated nightly by CI." \ | |
| --prerelease | |
| - name: Upload index to release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh release upload notion-index-latest data/index.json.gz data/embeddings.i8.gz data/build-cache.json.gz --clobber | |
| gh release delete-asset notion-index-latest embeddings.f32.gz --yes 2>/dev/null || true | |
| # The legacy single-file asset (notion-index.json.gz) is left in place until the bot | |
| # that reads the v2 assets is deployed; delete it from the release by hand after that. |