From 936fc638d80a12adde522a98c2a08c5174239bfa Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Wed, 11 Nov 2026 19:00:37 +0530 Subject: [PATCH 1/2] test-companion: say why signing in as the GitHub App failed Any failure to look up the App's installation was reported as "the GitHub App is not installed on this repository", hiding GitHub's real answer. A wrong App ID or a key from another App looked exactly like a missing installation. Now only a 404 is reported as "not installed". Any other failure is a warning that carries GitHub's own reason, e.g. "Bad credentials (HTTP 401)", and says which inputs to check. A failed token request also carries GitHub's reason. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/app-auth.mjs | 26 +++++++++++++++++++++ test-companion/src/main.mjs | 13 ++++++++--- test-companion/test/app-auth.test.mjs | 33 ++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index 725c524..8a0e527 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -23,6 +23,32 @@ export function jwtClaims(appId, nowSeconds) { return { iat, exp: iat + JWT_LIFETIME_SECONDS, iss: String(appId) } } +/** + * GitHub's reason a `gh api` call failed, e.g. "Bad credentials (HTTP 401)", for the log. + * gh's error output holds only the response message, never the token it sent. + */ +export function ghFailureReason(result) { + const line = String(result?.stderr ?? "") + .split("\n") + .map((l) => l.trim()) + .find(Boolean) + return line ? line.replace(/^gh:\s*/, "") : "no detail from GitHub" +} + +/** What to log when looking up the App's installation on this repository fails. */ +export function installationLookupMessage(result) { + const reason = ghFailureReason(result) + if (/\bHTTP 404\b/.test(reason)) { + return { level: "notice", text: "Test Companion: the GitHub App is not installed on this repository; using the workflow token." } + } + return { + level: "warning", + text: + `Test Companion: could not sign in as the GitHub App (${reason}); using the workflow token. ` + + "Check that app-id is the App's numeric ID and app-private-key is a key generated for that App.", + } +} + export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })) const payload = b64url(JSON.stringify(jwtClaims(appId, now))) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 4d87881..08a99d0 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -14,7 +14,13 @@ import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, re import { agentEnvFrom, asAgentCommand, checkPrHandoff, readClaimResult } from "./sandbox-policy.mjs" import { startRelay } from "./relay.mjs" import { runCapturingAnswer } from "./answer.mjs" -import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" +import { + appCredentials, + signJwt, + installationTokenRequest, + ghFailureReason, + installationLookupMessage, +} from "./app-auth.mjs" import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs" import { renderJobSummary, @@ -181,7 +187,8 @@ function appInstallationToken() { const found = asApp(["api", `repos/${REPO}/installation`]) if (found.status !== 0) { - notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") + const { level, text } = installationLookupMessage(found) + ;(level === "warning" ? warn : notice)(text) return undefined } let installationId @@ -213,7 +220,7 @@ function appInstallationToken() { const minted = mint(scope) if (minted.status !== 0) { warn( - "Could not get a GitHub App token with the permissions Test Companion needs; using the workflow token instead. " + + `Could not get a GitHub App token with the permissions Test Companion needs (${ghFailureReason(minted)}); using the workflow token instead. ` + "If the App's permissions were changed, an organisation owner may need to accept them.", ) return undefined diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index 872d454..96732ea 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -1,7 +1,14 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { generateKeyPairSync, createVerify } from "node:crypto" -import { appCredentials, jwtClaims, signJwt, installationTokenRequest } from "../src/app-auth.mjs" +import { + appCredentials, + jwtClaims, + signJwt, + installationTokenRequest, + ghFailureReason, + installationLookupMessage, +} from "../src/app-auth.mjs" const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) const pem = privateKey.export({ type: "pkcs1", format: "pem" }) @@ -88,3 +95,27 @@ describe("installationTokenRequest", () => { assert.equal(installationTokenRequest(undefined), undefined) }) }) + +describe("App sign-in failures say why", () => { + test("GitHub's reason is taken from gh's error output", () => { + assert.equal(ghFailureReason({ stderr: "gh: Bad credentials (HTTP 401)\n" }), "Bad credentials (HTTP 401)") + }) + + test("an empty error output still gives a reason", () => { + assert.equal(ghFailureReason({ stderr: "" }), "no detail from GitHub") + assert.equal(ghFailureReason(undefined), "no detail from GitHub") + }) + + test("only a 404 means the App is not installed", () => { + const m = installationLookupMessage({ stderr: "gh: Not Found (HTTP 404)" }) + assert.equal(m.level, "notice") + assert.match(m.text, /not installed on this repository/) + }) + + test("any other failure is a warning that carries GitHub's reason", () => { + const m = installationLookupMessage({ stderr: "gh: A JSON web token could not be decoded (HTTP 401)" }) + assert.equal(m.level, "warning") + assert.match(m.text, /A JSON web token could not be decoded \(HTTP 401\)/) + assert.doesNotMatch(m.text, /not installed/) + }) +}) From f81d5cc6faa83aee7a2b9e2e58c3f9f04e826c38 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Wed, 11 Nov 2026 20:41:19 +0530 Subject: [PATCH 2/2] test-companion: sign in as the GitHub App with Bearer, not through gh gh sends its token as "Authorization: token ...", and GitHub accepts an App's JWT only as "Authorization: Bearer ...". So every App sign-in failed with "A JSON web token could not be decoded (HTTP 401)" and the Action fell back to github-actions[bot], even with the right App ID and key. Checked against the real App: through gh, 401; with Bearer, 200. The two calls made as the App (find the installation, get the token) now use fetch with Bearer. Everything after uses the installation token through gh as before. The two remaining silent fallbacks now say why. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/app-auth.mjs | 35 ++++++++++++------ test-companion/src/main.mjs | 49 ++++++++++--------------- test-companion/test/app-auth.test.mjs | 52 ++++++++++++++++++++++----- 3 files changed, 87 insertions(+), 49 deletions(-) diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index 8a0e527..ed62421 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -24,21 +24,36 @@ export function jwtClaims(appId, nowSeconds) { } /** - * GitHub's reason a `gh api` call failed, e.g. "Bad credentials (HTTP 401)", for the log. - * gh's error output holds only the response message, never the token it sent. + * Call the GitHub API as the App. Not through gh: gh sends `Authorization: token`, + * and GitHub accepts an App's JWT only as `Authorization: Bearer`. + * Returns { ok, status, data, reason }; `reason` is GitHub's message, never the JWT. */ -export function ghFailureReason(result) { - const line = String(result?.stderr ?? "") - .split("\n") - .map((l) => l.trim()) - .find(Boolean) - return line ? line.replace(/^gh:\s*/, "") : "no detail from GitHub" +export async function appApi(path, { jwt, method = "GET", body, apiUrl, fetchImpl = fetch } = {}) { + const base = String(apiUrl || "https://api.github.com").replace(/\/+$/, "") + let response + try { + response = await fetchImpl(`${base}/${path}`, { + method, + headers: { + Authorization: `Bearer ${jwt}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }) + } catch (e) { + return { ok: false, status: 0, data: undefined, reason: `could not reach GitHub: ${e?.message ?? e}` } + } + const data = await response.json().catch(() => undefined) + const reason = response.ok ? "" : `${data?.message || response.statusText || "no detail from GitHub"} (HTTP ${response.status})` + return { ok: response.ok, status: response.status, data, reason } } /** What to log when looking up the App's installation on this repository fails. */ export function installationLookupMessage(result) { - const reason = ghFailureReason(result) - if (/\bHTTP 404\b/.test(reason)) { + const reason = result?.reason || "no detail from GitHub" + if (result?.status === 404) { return { level: "notice", text: "Test Companion: the GitHub App is not installed on this repository; using the workflow token." } } return { diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 08a99d0..a93827c 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -18,7 +18,7 @@ import { appCredentials, signJwt, installationTokenRequest, - ghFailureReason, + appApi, installationLookupMessage, } from "./app-auth.mjs" import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs" @@ -168,7 +168,7 @@ function fetchAndCheckout(sha, branch) { } /** A repo-scoped App installation token, or undefined to fall back to the workflow token. */ -function appInstallationToken() { +async function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined // Remove the key from the environment so nothing started later, the agent included, inherits it. @@ -183,25 +183,22 @@ function appInstallationToken() { } // The JWT authenticates the App; only the installation token can touch a repository. - const asApp = (args) => runGh(args, { token: jwt }) + const asApp = (path, opts) => appApi(path, { jwt, apiUrl: env.GITHUB_API_URL, ...opts }) - const found = asApp(["api", `repos/${REPO}/installation`]) - if (found.status !== 0) { + const found = await asApp(`repos/${REPO}/installation`) + if (!found.ok) { const { level, text } = installationLookupMessage(found) - ;(level === "warning" ? warn : notice)(text) + if (level === "warning") warn(text) + else notice(text) return undefined } - let installationId - let appLogin - try { - const installation = JSON.parse(found.stdout) - installationId = installation?.id - // Adopted only once a token is minted, so a failed mint can't leave us looking under the wrong login. - appLogin = installation?.app_slug ? `${installation.app_slug}[bot]` : undefined - } catch { + const installationId = found.data?.id + // Adopted only once a token is minted, so a failed mint can't leave us looking under the wrong login. + const appLogin = found.data?.app_slug ? `${found.data.app_slug}[bot]` : undefined + if (!installationId) { + warn("Test Companion: GitHub's reply about the App's installation had no ID; using the workflow token.") return undefined } - if (!installationId) return undefined // Scoped to this repository; an empty request gets every repository the installation covers. const scope = installationTokenRequest(REPO) @@ -210,29 +207,21 @@ function appInstallationToken() { return undefined } - const mint = (body) => - runGh(["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { - token: jwt, - input: JSON.stringify(body), - }) - // No retry with a smaller request: dropping the permission list asks for everything the installation holds. - const minted = mint(scope) - if (minted.status !== 0) { + const minted = await asApp(`app/installations/${installationId}/access_tokens`, { method: "POST", body: scope }) + if (!minted.ok) { warn( - `Could not get a GitHub App token with the permissions Test Companion needs (${ghFailureReason(minted)}); using the workflow token instead. ` + + `Test Companion: could not get a GitHub App token with the permissions it needs (${minted.reason}); using the workflow token instead. ` + "If the App's permissions were changed, an organisation owner may need to accept them.", ) return undefined } - let token - try { - token = JSON.parse(minted.stdout)?.token - } catch { + const token = minted.data?.token + if (!token) { + warn("Test Companion: GitHub's reply had no App token; using the workflow token.") return undefined } - if (!token) return undefined // Mask before first use. console.log(`::add-mask::${token}`) @@ -388,7 +377,7 @@ async function main() { } // Act as the App when installed, before anything else talks to GitHub. - const appToken = appInstallationToken() + const appToken = await appInstallationToken() if (appToken) { env.GH_TOKEN = appToken notice("Test Companion: acting as the GitHub App.") diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index 96732ea..4bd6305 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -6,7 +6,7 @@ import { jwtClaims, signJwt, installationTokenRequest, - ghFailureReason, + appApi, installationLookupMessage, } from "../src/app-auth.mjs" @@ -96,24 +96,58 @@ describe("installationTokenRequest", () => { }) }) -describe("App sign-in failures say why", () => { - test("GitHub's reason is taken from gh's error output", () => { - assert.equal(ghFailureReason({ stderr: "gh: Bad credentials (HTTP 401)\n" }), "Bad credentials (HTTP 401)") +describe("appApi", () => { + const fakeFetch = (status, json, seen) => async (url, init) => { + Object.assign(seen, { url, init }) + return { ok: status >= 200 && status < 300, status, statusText: "", json: async () => json } + } + + test("sends the JWT as Bearer, which GitHub requires for an App", async () => { + const seen = {} + const r = await appApi("repos/o/r/installation", { jwt: "J", fetchImpl: fakeFetch(200, { id: 7 }, seen) }) + assert.equal(seen.init.headers.Authorization, "Bearer J") + assert.equal(seen.url, "https://api.github.com/repos/o/r/installation") + assert.deepEqual(r, { ok: true, status: 200, data: { id: 7 }, reason: "" }) + }) + + test("uses the server's API address when given", async () => { + const seen = {} + await appApi("x", { jwt: "J", apiUrl: "https://ghe.example/api/v3/", fetchImpl: fakeFetch(200, {}, seen) }) + assert.equal(seen.url, "https://ghe.example/api/v3/x") + }) + + test("sends a JSON body on POST", async () => { + const seen = {} + await appApi("x", { jwt: "J", method: "POST", body: { a: 1 }, fetchImpl: fakeFetch(201, {}, seen) }) + assert.equal(seen.init.method, "POST") + assert.equal(seen.init.body, '{"a":1}') + assert.equal(seen.init.headers["Content-Type"], "application/json") + }) + + test("a failure carries GitHub's message and status, never the JWT", async () => { + const r = await appApi("x", { jwt: "SECRET-JWT", fetchImpl: fakeFetch(401, { message: "Bad credentials" }, {}) }) + assert.equal(r.ok, false) + assert.equal(r.reason, "Bad credentials (HTTP 401)") + assert.doesNotMatch(r.reason, /SECRET-JWT/) }) - test("an empty error output still gives a reason", () => { - assert.equal(ghFailureReason({ stderr: "" }), "no detail from GitHub") - assert.equal(ghFailureReason(undefined), "no detail from GitHub") + test("a network failure is reported, not thrown", async () => { + const r = await appApi("x", { jwt: "J", fetchImpl: async () => { throw new Error("ECONNRESET") } }) + assert.equal(r.ok, false) + assert.equal(r.status, 0) + assert.match(r.reason, /ECONNRESET/) }) +}) +describe("installationLookupMessage", () => { test("only a 404 means the App is not installed", () => { - const m = installationLookupMessage({ stderr: "gh: Not Found (HTTP 404)" }) + const m = installationLookupMessage({ status: 404, reason: "Not Found (HTTP 404)" }) assert.equal(m.level, "notice") assert.match(m.text, /not installed on this repository/) }) test("any other failure is a warning that carries GitHub's reason", () => { - const m = installationLookupMessage({ stderr: "gh: A JSON web token could not be decoded (HTTP 401)" }) + const m = installationLookupMessage({ status: 401, reason: "A JSON web token could not be decoded (HTTP 401)" }) assert.equal(m.level, "warning") assert.match(m.text, /A JSON web token could not be decoded \(HTTP 401\)/) assert.doesNotMatch(m.text, /not installed/)