Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions .github/scripts/copilot-workflows.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,12 @@ const fs = require('node:fs');
const path = require('node:path');

const ROOT = path.resolve(__dirname, '..', '..');
const AI_ACTION = 'actions/ai-inference@2c43c91ae16266ca159d311430343c67a5ffa222';
const CLI_INSTALL = 'npm install --global @github/copilot@1.0.74';
const COPILOT_RUNNER = 'node .github/scripts/run-copilot-inference.cjs';
const CLI_INSTALL = 'bash .github/scripts/install-copilot-cli.sh';
const SETUP_NODE = 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e';
const TOKEN_FALLBACK = 'COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN || github.token }}';
const COPILOT_VERSION = 'COPILOT_VERSION="v1.0.74"';
const COPILOT_SHA256 = 'COPILOT_SHA256="4a708b0a1cbaef4c2ca5c546a622f887a3b70e8a0432bc3cee0d386704816650"';

function readWorkflow(name) {
return fs.readFileSync(path.join(ROOT, '.github', 'workflows', name), 'utf8');
Expand All @@ -16,24 +19,30 @@ function count(text, fragment) {
return text.split(fragment).length - 1;
}

test('issue automation uses pinned Copilot inference without tool access', () => {
test('issue automation streams prompts through the digest-pinned Copilot CLI without tool access', () => {
const quality = readWorkflow('enforce-issue-quality.yml');
const triage = readWorkflow('issue-triage.yml');
const combined = quality + '\n' + triage;
const installer = fs.readFileSync(path.join(ROOT, '.github', 'scripts', 'install-copilot-cli.sh'), 'utf8');

assert.equal(count(quality, AI_ACTION), 2);
assert.equal(count(triage, AI_ACTION), 1);
assert.equal(count(quality, COPILOT_RUNNER), 2);
assert.equal(count(triage, COPILOT_RUNNER), 1);
assert.equal(count(quality, SETUP_NODE), 2);
assert.equal(count(triage, SETUP_NODE), 1);
assert.equal(count(quality, CLI_INSTALL), 2);
assert.equal(count(triage, CLI_INSTALL), 1);
assert.equal(count(quality, 'copilot-requests: write'), 2);
assert.equal(count(triage, 'copilot-requests: write'), 1);
assert.equal(count(quality, 'GITHUB_TOKEN: ${{ github.token }}'), 2);
assert.equal(count(triage, 'GITHUB_TOKEN: ${{ github.token }}'), 1);
assert.equal(count(quality, 'model: ""'), 2);
assert.equal(count(triage, 'model: ""'), 1);
assert.equal(count(quality, TOKEN_FALLBACK), 2);
assert.equal(count(triage, TOKEN_FALLBACK), 1);

assert.match(installer, new RegExp(COPILOT_VERSION.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
assert.match(installer, new RegExp(COPILOT_SHA256.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
assert.match(installer, /sha256sum --check --status/);
assert.match(installer, /releases\/download\/\$\{COPILOT_VERSION\}\/\$\{COPILOT_ASSET\}/);

assert.doesNotMatch(combined, /npm install --global @github\/copilot/);
assert.doesNotMatch(combined, /actions\/ai-inference@/);
assert.doesNotMatch(combined, /\bmodels:\s*read\b/);
assert.doesNotMatch(combined, /max-tokens:/);
assert.doesNotMatch(combined, /copilot-allow-tools:/);
Expand Down
33 changes: 33 additions & 0 deletions .github/scripts/install-copilot-cli.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -euo pipefail

COPILOT_VERSION="v1.0.74"
COPILOT_ASSET="copilot-linux-x64.tar.gz"
COPILOT_SHA256="4a708b0a1cbaef4c2ca5c546a622f887a3b70e8a0432bc3cee0d386704816650"
COPILOT_URL="https://github.com/github/copilot-cli/releases/download/${COPILOT_VERSION}/${COPILOT_ASSET}"

install_root="${RUNNER_TEMP:?RUNNER_TEMP is required}/copilot-cli-${COPILOT_VERSION}"
archive="${install_root}/${COPILOT_ASSET}"
bin_dir="${install_root}/bin"

rm -rf -- "$install_root"
mkdir -p "$bin_dir"

curl \
--proto '=https' \
--tlsv1.2 \
--fail \
--silent \
--show-error \
--location \
--retry 3 \
"$COPILOT_URL" \
--output "$archive"

printf '%s %s\n' "$COPILOT_SHA256" "$archive" | sha256sum --check --status

tar -xzf "$archive" -C "$bin_dir"
chmod +x "$bin_dir/copilot"
"$bin_dir/copilot" --version

printf '%s\n' "$bin_dir" >> "${GITHUB_PATH:?GITHUB_PATH is required}"
73 changes: 73 additions & 0 deletions .github/scripts/run-copilot-inference.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
const fs = require('node:fs');
const crypto = require('node:crypto');
const { spawnSync } = require('node:child_process');

function fail(message, code = 1) {
process.stderr.write(`${message}\n`);
process.exit(code);
}

const promptPath = process.argv[2];
let userPrompt;
try {
userPrompt = promptPath
? fs.readFileSync(promptPath, 'utf8')
: fs.readFileSync(0, 'utf8');
} catch (error) {
fail(`Unable to read Copilot prompt: ${error instanceof Error ? error.message : String(error)}`);
}

const systemPrompt = String(process.env.COPILOT_SYSTEM_PROMPT || '').trim();
const prompt = systemPrompt
? `${systemPrompt}\n\n${userPrompt}`
: userPrompt;

const rawTimeout = Number(process.env.COPILOT_TIMEOUT_MS || 120_000);
const timeout = Number.isFinite(rawTimeout) && rawTimeout > 0
? Math.floor(rawTimeout)
: 120_000;

const args = [
'-s',
'--no-ask-user',
'--no-custom-instructions',
'--no-auto-update',
];

const copilotEnv = { ...process.env };
if (copilotEnv.COPILOT_GITHUB_TOKEN) {
// Copilot CLI v1.0.74 authenticates from GH_TOKEN or GITHUB_TOKEN.
copilotEnv.GITHUB_TOKEN = copilotEnv.COPILOT_GITHUB_TOKEN;
}

const result = spawnSync('copilot', args, {
input: prompt,
encoding: 'utf8',
env: copilotEnv,
maxBuffer: 16 * 1024 * 1024,
timeout,
killSignal: 'SIGKILL',
});

if (result.stderr) {
process.stderr.write(result.stderr);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

if (result.error) {
const errorCode = result.error.code || 'spawn_error';
const signal = result.signal || 'none';
fail(`Copilot CLI execution failed (${errorCode}; signal=${signal}): ${result.error.message}`);
}

if (result.status !== 0) {
process.exit(Number.isInteger(result.status) ? result.status : 1);
}

const outputFile = process.env.GITHUB_OUTPUT;
if (!outputFile) {
fail('GITHUB_OUTPUT is not set.');
}

const response = String(result.stdout || '').trimEnd();
const delimiter = `COPILOT_RESPONSE_${crypto.randomBytes(12).toString('hex')}`;
fs.appendFileSync(outputFile, `response<<${delimiter}\n${response}\n${delimiter}\n`);
123 changes: 123 additions & 0 deletions .github/scripts/run-copilot-inference.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');

const RUNNER = path.join(__dirname, 'run-copilot-inference.cjs');

function makeFakeCopilot(source) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fake-copilot-'));
const file = path.join(dir, 'copilot');
fs.writeFileSync(file, `#!/usr/bin/env node\n${source}\n`, { mode: 0o755 });
return { dir, file };
}

function outputValue(file, key) {
const text = fs.readFileSync(file, 'utf8');
const match = text.match(new RegExp(`${key}<<([^\\n]+)\\n([\\s\\S]*?)\\n\\1(?:\\n|$)`));
assert.ok(match, `missing ${key} output in ${text}`);
return match[2];
}

test('streams a large prompt over stdin and maps the Copilot token to GITHUB_TOKEN', () => {
const fake = makeFakeCopilot(`
const fs = require('node:fs');
const input = fs.readFileSync(0, 'utf8');
const argvBytes = Buffer.byteLength(process.argv.slice(2).join(' '));
if (argvBytes > 8192) {
console.error('prompt leaked into argv');
process.exit(91);
}
process.stdout.write(JSON.stringify({
inputBytes: Buffer.byteLength(input),
argv: process.argv.slice(2),
githubToken: process.env.GITHUB_TOKEN || '',
}));
`);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'copilot-runner-test-'));
const promptFile = path.join(dir, 'prompt.txt');
const outputFile = path.join(dir, 'output.txt');
const prompt = 'x'.repeat(512 * 1024);
fs.writeFileSync(promptFile, prompt);
fs.writeFileSync(outputFile, '');

const result = spawnSync(process.execPath, [RUNNER, promptFile], {
encoding: 'utf8',
env: {
...process.env,
PATH: `${fake.dir}${path.delimiter}${process.env.PATH}`,
GITHUB_OUTPUT: outputFile,
COPILOT_SYSTEM_PROMPT: 'system instruction',
COPILOT_GITHUB_TOKEN: 'test-token',
GITHUB_TOKEN: '',
},
});

assert.equal(result.status, 0, result.stderr);
const response = JSON.parse(outputValue(outputFile, 'response'));
assert.ok(response.inputBytes > Buffer.byteLength(prompt));
assert.deepEqual(response.argv, ['-s', '--no-ask-user', '--no-custom-instructions', '--no-auto-update']);
assert.equal(response.githubToken, 'test-token');
});

test('surfaces Copilot stderr and preserves a non-zero exit code', () => {
const fake = makeFakeCopilot(`
process.stdin.resume();
process.stdin.on('end', () => {
console.error('copilot auth failed: test diagnostic');
process.exit(7);
});
`);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'copilot-runner-test-'));
const promptFile = path.join(dir, 'prompt.txt');
const outputFile = path.join(dir, 'output.txt');
fs.writeFileSync(promptFile, 'hello');
fs.writeFileSync(outputFile, '');

const result = spawnSync(process.execPath, [RUNNER, promptFile], {
encoding: 'utf8',
env: {
...process.env,
PATH: `${fake.dir}${path.delimiter}${process.env.PATH}`,
GITHUB_OUTPUT: outputFile,
COPILOT_SYSTEM_PROMPT: 'system instruction',
COPILOT_GITHUB_TOKEN: 'test-token',
},
});

assert.equal(result.status, 7);
assert.match(result.stderr, /copilot auth failed: test diagnostic/);
assert.equal(fs.readFileSync(outputFile, 'utf8'), '');
});

test('kills a hung Copilot process at the configured timeout', () => {
const fake = makeFakeCopilot(`
process.stderr.write('copilot started\\n');
setInterval(() => {}, 1000);
`);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'copilot-runner-test-'));
const promptFile = path.join(dir, 'prompt.txt');
const outputFile = path.join(dir, 'output.txt');
fs.writeFileSync(promptFile, 'hello');
fs.writeFileSync(outputFile, '');

const result = spawnSync(process.execPath, [RUNNER, promptFile], {
encoding: 'utf8',
timeout: 5000,
env: {
...process.env,
PATH: `${fake.dir}${path.delimiter}${process.env.PATH}`,
GITHUB_OUTPUT: outputFile,
COPILOT_SYSTEM_PROMPT: 'system instruction',
COPILOT_GITHUB_TOKEN: 'test-token',
COPILOT_TIMEOUT_MS: '75',
},
});

assert.notEqual(result.status, 0);
assert.match(result.stderr, /ETIMEDOUT/);
assert.match(result.stderr, /SIGKILL/);
assert.equal(fs.readFileSync(outputFile, 'utf8'), '');
});
Loading
Loading