"Initial dj mcp server code changes"

.vscode/settings.json

@@ -12,15 +12,21 @@
   },
   "json.schemas": [
     {
-      "fileMatch": ["*.model.json"],
+      "fileMatch": [
+        "*.model.json"
+      ],
       "url": ".dj/schemas/model.schema.json"
     },
     {
-      "fileMatch": ["*.source.json"],
+      "fileMatch": [
+        "*.source.json"
+      ],
       "url": ".dj/schemas/source.schema.json"
     },
     {
-      "fileMatch": ["*.python.json"],
+      "fileMatch": [
+        "*.python.json"
+      ],
       "url": ".dj/schemas/python-model.schema.json"
     }
   ],

mcp/README.md

@@ -0,0 +1,158 @@
+# DJ MCP Server
+
+Agent-agnostic stdio MCP server for DJ (Data JSON) model workflows.
+
+Two modes:
+
+1. **Production catalog** — operator configures 2–3 dbt repos; users pick `project-a` / `project-b`, give requirements, review an isolated preview, then approve a PR.
+2. **Independent local** — user selects their own local dbt path with `dj_use_local_project`.
+
+No demo projects are bundled. Create/update runs in an isolated git worktree until `dj_publish_change`.
+
+Optional **Trino** connection enables live source sampling and model data preview (`dbt compile` / `dbt run` + `SELECT … LIMIT`).
+
+## Tools
+
+| Tool | Purpose |
+|------|---------|
+| `dj_list_projects` | List catalog projects or discover under a local path |
+| `dj_use_project` | Select catalog `projectId` (Mode A) |
+| `dj_use_local_project` | Select local checkout path (Mode B) |
+| `dj_describe_structure` | DJ folder rules + existing groups/topics |
+| `dj_list_models` | List models |
+| `dj_get_model` | Read model + sql/yml |
+| `dj_validate_model` | Schema validate |
+| `dj_preview_model` | Artifact SQL/YAML only (no warehouse) |
+| `dj_trino_status` | Trino configured + reachable |
+| `dj_list_trino_tables` | Browse catalogs → schemas → tables → columns |
+| `dj_preview_source` | Sample source table rows via Trino |
+| `dj_preview_data` | `dbt compile` or `dbt run`, then sample model rows via Trino |
+| `dj_create_model` | Create model in isolated change set |
+| `dj_update_model` | Update model (+ regenerate sql/yml) |
+| `dj_create_source` | Create source in isolated change set |
+| `dj_create_e2e` | Requirement → create + preview + lineage (+ optional live data) |
+| `dj_get_lineage` | Upstream/downstream lineage |
+| `dj_get_change` | Inspect change set |
+| `dj_discard_change` | Discard change set |
+| `dj_publish_change` | Approve → commit, push, open PR |
+
+## Setup
+
+```bash
+cd dj
+npm install
+npm run mcp:build
+```
+
+Copy [`config.example.json`](config.example.json) to `~/.dj-mcp/config.json` (or set `DJ_MCP_CONFIG`).
+
+### Production catalog config
+
+```json
+{
+  "productionMode": true,
+  "allowLocalProjectMode": true,
+  "exposeFilesystemPaths": false,
+  "trino": {
+    "enabled": true,
+    "host": "trino.example.com",
+    "port": 443,
+    "httpScheme": "https",
+    "catalog": "hive",
+    "schema": "default",
+    "user": "mcp-bot",
+    "passwordEnv": "TRINO_PASSWORD",
+    "defaultLimit": 100,
+    "previewMode": "compile"
+  },
+  "projects": [
+    {
+      "id": "project-a",
+      "label": "AWS Billing",
+      "type": "git",
+      "url": "git@ghe.example.com/finance/dbt-billing.git",
+      "ref": "main",
+      "projectName": "billing",
+      "pr": { "provider": "github", "baseBranch": "main" },
+      "trino": { "catalog": "finance", "schema": "billing" }
+    }
+  ]
+}
+```
+
+Never put passwords in JSON. Prefer `passwordEnv: "TRINO_PASSWORD"` and export the var in `~/.zshrc`. Use [`start.sh`](start.sh) as the MCP `command` so Cursor loads that shell env.
+
+### Independent local (no catalog)
+
+Omit `projects` (or leave empty) and call:
+
+```json
+{ "localPath": "/Users/me/my-dbt-project" }
+```
+
+via `dj_use_local_project`. Requires `allowLocalProjectMode: true`.
+
+## Live data preview (Trino)
+
+Requires:
+
+- `trino.enabled` + `trino.host` in config (or `TRINO_HOST`)
+- Password via `TRINO_PASSWORD` / `passwordEnv`
+- `dbt` on PATH (and `DBT_PROFILES_DIR` / `DJ_DBT_PROFILES_DIR` for compile/run)
+- Network access to the Trino coordinator
+
+By default DJ uses the **HTTP** `/v1/statement` API. Set `trino.cliPath` (or `DJ_TRINO_PATH`) to use the Trino CLI instead.
+
+### Full agent flow
+
+1. `dj_use_project({ "projectId": "project-c" })`
+2. `dj_trino_status` — confirm connectivity
+3. `dj_list_trino_tables` → `dj_preview_source({ "table": "…", "limit": 20 })`
+4. `dj_create_e2e({ "requirement": "…", "model": { … } })` — isolated change set
+5. `dj_preview_data({ "changeSetId": "…", "modelName": "…", "mode": "compile" })`  
+   - `compile` — `dbt compile` then run compiled SELECT with LIMIT  
+   - `run` — `dbt run --select model` then `SELECT * FROM catalog.schema.model LIMIT N`  
+   - `includeUpstream: true` with `run` uses `+model`
+6. Approve → `dj_publish_change`
+
+Or pass `includeData: true` on `dj_create_e2e` to chain step 5 automatically.
+
+## Agent prompt recipe
+
+**Mode A**
+
+1. `dj_list_projects` → user picks `project-a`
+2. `dj_use_project({ "projectId": "project-a" })`
+3. `dj_describe_structure({ "suggestion": "aws billing" })`
+4. Interpret requirement into DJ model JSON (`group` / `topic` / `type` / `select`)
+5. `dj_create_e2e({ "requirement": "...", "model": { ... } })`
+6. Optional: `dj_preview_data` for live rows
+7. On approval: `dj_publish_change({ "changeSetId": "...", "approval": true, "commitMessage": "..." })`
+
+**Mode B**
+
+1. `dj_use_local_project({ "localPath": "/path/to/dbt" })`
+2. Same describe → create_e2e → preview_data → publish flow
+
+## Publish prerequisites
+
+- Selected project must be a git repo (catalog git mirrors, or local checkout with `.git`)
+- `git` on PATH
+- Authenticated `gh` CLI for GitHub/GHE PRs
+
+## Environment
+
+| Variable | Description |
+|----------|-------------|
+| `DJ_MCP_CONFIG` | Path to config JSON (default `~/.dj-mcp/config.json`) |
+| `DJ_WORKSPACE_ROOT` | Optional self-hosted local root when not using catalog |
+| `TRINO_HOST` / `TRINO_PORT` / `TRINO_USER` / `TRINO_PASSWORD` / `TRINO_CATALOG` / `TRINO_SCHEMA` | Override Trino connection |
+| `DJ_TRINO_PATH` | Optional Trino CLI binary (forces CLI mode) |
+| `DBT_PROFILES_DIR` / `DJ_DBT_PROFILES_DIR` | dbt profiles for compile/run |
+| `DJ_DBT_PATH` | Optional path to `dbt` binary |
+
+## Cursor / Claude Desktop
+
+See [`mcp.example.json`](mcp.example.json). Prefer `command: .../mcp/start.sh` so `TRINO_PASSWORD` from `~/.zshrc` is available. Set `DJ_MCP_CONFIG` (and optionally `DBT_PROFILES_DIR`) in `env`.
+
+After rebuild, rename the MCP server entry or restart so the host refreshes tool schemas.

mcp/__tests__/response.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, test } from '@jest/globals';
+
+import { failure, success, toToolContent } from '../src/response';
+
+describe('response helpers', () => {
+  test('success wraps data', () => {
+    const result = success({ foo: 'bar' });
+    expect(result.ok).toBe(true);
+    expect(result.data).toEqual({ foo: 'bar' });
+  });
+
+  test('failure wraps errors', () => {
+    const result = failure(['bad']);
+    expect(result.ok).toBe(false);
+    expect(result.errors).toEqual(['bad']);
+  });
+
+  test('toToolContent marks errors', () => {
+    const content = toToolContent(failure(['x']));
+    expect(content.isError).toBe(true);
+    expect(content.content[0].text).toContain('"ok": false');
+  });
+});

mcp/config.example.json

@@ -0,0 +1,60 @@
+{
+  "productionMode": true,
+  "allowLocalProjectMode": true,
+  "exposeFilesystemPaths": false,
+  "defaultProjectId": "project-a",
+  "trino": {
+    "enabled": true,
+    "host": "trino.example.com",
+    "port": 443,
+    "httpScheme": "https",
+    "catalog": "hive",
+    "schema": "default",
+    "user": "mcp-bot",
+    "passwordEnv": "TRINO_PASSWORD",
+    "defaultLimit": 100,
+    "timeoutMs": 120000,
+    "previewMode": "compile"
+  },
+  "projects": [
+    {
+      "id": "project-a",
+      "label": "AWS Billing",
+      "type": "git",
+      "url": "git@ghe.example.com/finance/dbt-billing.git",
+      "ref": "main",
+      "projectName": "billing",
+      "pr": {
+        "provider": "github",
+        "baseBranch": "main"
+      },
+      "trino": {
+        "catalog": "finance",
+        "schema": "billing"
+      }
+    },
+    {
+      "id": "project-b",
+      "label": "Customer Analytics",
+      "type": "git",
+      "url": "git@ghe.example.com/analytics/dbt-customers.git",
+      "ref": "main",
+      "projectName": "customer_analytics",
+      "pr": {
+        "provider": "github",
+        "baseBranch": "main"
+      }
+    },
+    {
+      "id": "project-c",
+      "label": "Supply Chain",
+      "type": "local",
+      "path": "/srv/dj-projects/supply-chain",
+      "projectName": "supply_chain",
+      "pr": {
+        "provider": "github",
+        "baseBranch": "main"
+      }
+    }
+  ]
+}

mcp/esbuild.mjs

@@ -0,0 +1,24 @@
+import * as esbuild from 'esbuild';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+const root = path.dirname(fileURLToPath(import.meta.url));
+
+await esbuild.build({
+  entryPoints: [path.join(root, 'src/server.ts')],
+  bundle: true,
+  platform: 'node',
+  target: 'node20',
+  outfile: path.join(root, 'dist/server.js'),
+  format: 'cjs',
+  // jsonc-parser's `main` is a UMD bundle whose inner require() can't be
+  // statically resolved; prefer the ESM build like the extension bundle does.
+  mainFields: ['module', 'main'],
+  sourcemap: true,
+  alias: {
+    admin: path.join(root, 'src/stubs/admin.ts'),
+    '@services': path.join(root, '../src/services'),
+    '@shared': path.join(root, '../src/shared'),
+  },
+  external: [],
+});

mcp/eslint.config.mjs

@@ -0,0 +1,16 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+  root: true,
+  rules: {
+    'no-restricted-imports': [
+      'error',
+      {
+        paths: [
+          { name: 'vscode', message: 'MCP server must not import vscode' },
+          { name: '@services/config', message: 'Use headless DJ config instead' },
+          { name: '@services/dbt', message: 'Use @services/framework/headless loadDbtProject' },
+        ],
+      },
+    ],
+  },
+};

mcp/jest.config.js

@@ -0,0 +1,13 @@
+/** @type {import('jest').Config} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  moduleNameMapper: {
+    admin: ['<rootDir>/src/stubs/admin.ts'],
+    '@services/(.*)': ['<rootDir>/../src/services/$1'],
+    '@services': ['<rootDir>/../src/services'],
+    '@shared/(.*)': ['<rootDir>/../src/shared/$1'],
+    '@shared': ['<rootDir>/../src/shared'],
+  },
+  testPathIgnorePatterns: ['<rootDir>/dist/'],
+};

mcp/mcp.example.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "dj": {
+      "command": "/Users/dinesh.prakash/perftool-repos/dj-mcp/dj/mcp/start.sh",
+      "args": [],
+      "env": {
+        "DJ_MCP_CONFIG": "/Users/dinesh.prakash/perftool-repos/dj-mcp/dj/mcp/config.example1.json"
+      }
+    }
+  }
+}

mcp/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@workday/dj-mcp",
+  "version": "0.1.0",
+  "description": "MCP server for DJ (Data JSON) model CRUD and SQL/YML generation",
+  "license": "Apache-2.0",
+  "private": true,
+  "type": "commonjs",
+  "main": "dist/server.js",
+  "bin": {
+    "dj-mcp": "dist/server.js"
+  },
+  "scripts": {
+    "build": "node esbuild.mjs",
+    "dev": "npx tsx src/server.ts",
+    "start": "node dist/server.js",
+    "test": "npx jest"
+  },
+  "dependencies": {
+    "ajv": "^8.17.1",
+    "glob": "^11.0.3",
+    "jsonc-parser": "^3.3.1",
+    "lodash": "^4.17.21",
+    "sql-formatter": "^15.3.1",
+    "yaml": "^2.4.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "@types/node": "20.x",
+    "esbuild": "^0.25.9",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "tsx": "^4.7.1",
+    "typescript": "^5.4.2"
+  }
+}