From 7e7a4bc651fef85d441fcf57515c7a7c1b0d1d6f Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Wed, 11 Nov 2026 18:28:24 +0530 Subject: [PATCH] test-companion: trim comments to what the code can't say Most comments restated the code or told the history of a change. They're cut to short notes on reasons that aren't obvious: the security reasoning, GitHub and git quirks, and ordering constraints. Also tightens four action.yml input descriptions and the README, and replaces a personal email and GitHub handle in test data with someone@example.com and octocat. No behaviour change: code is identical once comments are removed, and all 693 tests pass. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-companion-tests.yml | 22 +- CODEOWNERS | 2 - test-companion/README.md | 100 +-- test-companion/action.yml | 200 +---- test-companion/src/access.mjs | 31 +- test-companion/src/annotations.mjs | 50 +- test-companion/src/answer.mjs | 56 +- test-companion/src/app-auth.mjs | 54 +- test-companion/src/child-env.mjs | 28 +- test-companion/src/framework.mjs | 16 +- test-companion/src/gate.mjs | 67 +- test-companion/src/intent.mjs | 180 +--- test-companion/src/main.mjs | 850 ++++-------------- test-companion/src/outcome.mjs | 129 +-- test-companion/src/plan-run.mjs | 180 +--- test-companion/src/push-work.mjs | 324 +------ test-companion/src/relay.mjs | 28 +- test-companion/src/report.mjs | 490 +--------- test-companion/src/runs.mjs | 58 +- test-companion/src/sandbox-policy.mjs | 142 +-- test-companion/src/sandbox.mjs | 74 +- test-companion/src/starter.mjs | 45 +- test-companion/src/ticket.mjs | 38 +- test-companion/starter-workflow.yml | 102 +-- test-companion/test/access.test.mjs | 16 - test-companion/test/action-yml.test.mjs | 54 +- test-companion/test/annotations.test.mjs | 15 +- test-companion/test/answer.test.mjs | 25 +- test-companion/test/app-auth.test.mjs | 22 +- .../test/approval-targeting.test.mjs | 56 +- test-companion/test/build-effect.test.mjs | 36 +- test-companion/test/child-env.test.mjs | 9 - test-companion/test/framework.test.mjs | 4 - test-companion/test/gate.test.mjs | 39 - test-companion/test/imports.test.mjs | 67 +- test-companion/test/intent.test.mjs | 83 +- test-companion/test/outcome-emoji.test.mjs | 9 - test-companion/test/outcome.test.mjs | 37 +- test-companion/test/plan-run.test.mjs | 132 +-- test-companion/test/push-work.test.mjs | 204 +---- test-companion/test/report.test.mjs | 295 +----- test-companion/test/runs.test.mjs | 43 +- test-companion/test/sandbox-policy.test.mjs | 25 +- test-companion/test/starter.test.mjs | 78 -- test-companion/test/stop-removed.test.mjs | 34 +- test-companion/test/ticket.test.mjs | 8 - 46 files changed, 547 insertions(+), 4010 deletions(-) diff --git a/.github/workflows/test-companion-tests.yml b/.github/workflows/test-companion-tests.yml index 2a36496..56424df 100644 --- a/.github/workflows/test-companion-tests.yml +++ b/.github/workflows/test-companion-tests.yml @@ -1,12 +1,5 @@ -# Runs the Test Companion Action's own unit tests. -# -# The Action runs on customers' machines with their credentials, so its tests -# have to run on every change to it. -# -# The `paths` filter is safe only while this check is NOT required. A required -# check with a paths filter never reports on a pull request that misses the -# filter, and then blocks that pull request from merging for ever. If this is -# ever made required, remove the filter. +# If this check is ever made required, remove the `paths` filter: a required +# check that never reports blocks the pull request. name: Test Companion tests on: @@ -23,8 +16,6 @@ on: permissions: contents: read -# One run per pull request or branch. A newer push makes the older run's -# result irrelevant, so it is cancelled rather than left to finish. concurrency: group: test-companion-tests-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -37,19 +28,14 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: - # The tests need no git credentials, so none are left on disk. persist-credentials: false - # The Action has no dependencies: Node's standard library plus `gh`. - # There is nothing to install. - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "22" - # The exit code alone is not enough. When a test file throws while it - # loads (a missing import, say), `node --test` prints "fail 0" and - # exits 0, even though none of that file's tests ran. The only sign - # is a `not ok` line, so fail on that as well. + # `node --test` exits 0 when a test file fails to load; only a + # `not ok` line shows it. - name: Run the unit tests working-directory: test-companion run: | diff --git a/CODEOWNERS b/CODEOWNERS index 78b1c33..49fd81e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,6 +1,4 @@ * @browserstack/automate-public-repos -# The Test Companion action. Changes to it run an AI agent in customers' jobs, -# so they also need a review from its team. /test-companion/ @browserstack/automate-public-repos @browserstack/test-companion /.github/workflows/test-companion-tests.yml @browserstack/automate-public-repos @browserstack/test-companion diff --git a/test-companion/README.md b/test-companion/README.md index f8f2f34..beb25d5 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -12,42 +12,33 @@ comes back as one comment and one check. 2. Copy [`starter-workflow.yml`](starter-workflow.yml) to `.github/workflows/test-companion.yml` in your repository, as-is. -What runs is decided by `testcompanion-pipeline.yml` in your repository, not by -the workflow file. - -**Runner requirements:** a Linux runner (GitHub's `ubuntu-latest` works as is) -with Node 20 or later, the `gh` command installed system-wide (in `/usr/bin` or -`/usr/local/bin`), and password-free `sudo` for the job. - -`sudo` is what keeps your tokens away from the AI agent. Test Companion runs -the agent as a separate user with no admin rights, which cannot read this -job's tokens, secrets or temp files. Its `gh` can only read from GitHub, -through a relay; every write (saving tests, opening a pull request) is made by -the Action itself after the run. On a runner without password-free `sudo` the -run stops and says why, unless you set `allow-unisolated: true`. - -Use the starter file itself rather than writing your own. Besides the triggers, -it carries safeguards that are easy to leave out: - -- a filter that skips comments not meant for Test Companion, so they don't start - a job; -- `persist-credentials: false` on checkout, so the workflow token is not left in - `.git/config` where commands the agent runs could read it; -- the concurrency rules that keep one run per pull request, and let closing a - pull request cancel its run. +Which jobs run, on which events and with which limits is set in +`testcompanion-pipeline.yml` in your repository, not in the workflow file. -## Pin the CLI version +**Runner requirements:** Linux (GitHub's `ubuntu-latest` works as is), Node 20 +or later, `gh` installed system-wide (`/usr/bin` or `/usr/local/bin`), and +password-free `sudo` for the job. + +`sudo` keeps your tokens away from the AI agent. The agent runs as a separate +user with no admin rights that cannot read this job's tokens, secrets or temp +files. Its `gh` can only read from GitHub, through a relay; every write is made +by the Action after the run. Without password-free `sudo` the run stops and +says why, unless you set `allow-unisolated: true`. + +Use the starter file rather than writing your own. It includes safeguards that +are easy to miss: -`cli-version` is optional and you should set it once you know which version you -want. +- a filter so comments not meant for Test Companion don't start a job; +- `persist-credentials: false` on checkout, so the workflow token is not left + in `.git/config` where the agent could read it; +- concurrency rules that keep one run per pull request and let closing a pull + request cancel its run. -Without a pin the CLI upgrades itself the first time it runs on a release day. -That spends your pipeline's time on a download, and it means two runs a day -apart are not the same software. +## Pin the CLI version -**Both lines or neither.** A version with no checksum is refused — "refusing an -unverified pinned install" — and the job stops before it reports anything. Take -both from the release you tested with. +Without a pin the CLI upgrades itself on a release day, so two runs a day apart +may not run the same software. Set `cli-version` and `cli-sha256` together, +from the release you tested; a version without a checksum is refused. ```yaml - uses: browserstack/github-actions/test-companion@v1.1.0 @@ -64,46 +55,37 @@ both from the release you tested with. |---|---|---| | `username` | yes | BrowserStack username, from your secret store | | `access-key` | yes | BrowserStack access key, from your secret store | -| `cli-version` | no | Pin the CLI. Strongly recommended — see above | +| `cli-version` | no | Pin the CLI. Recommended, see above | | `cli-sha256` | no | Checksum for the pinned version | -| `build-effect` | no | Whether this step goes red. `fail-on-break` (the default) only when the run itself broke, could not start or hit a limit, not when tests failed. `report` never. `fail` on anything that is not a success, test failures included. The Test Companion check always reports truthfully either way | +| `build-effect` | no | When this step fails. `fail-on-break` (default): the run broke, could not start or hit a limit, not test failures. `report`: never. `fail`: anything but success. The check reports the real outcome either way | | `workflow` | no | Manual runs only; event-driven runs read the pipeline file | | `preview` | no | Report what would happen and change nothing | -| `allow-unisolated` | no | `"true"` runs the agent as the job's own user when it cannot be locked down. It can then reach this job's tokens and secrets. Off by default | -| `app-id` / `app-private-key` | no | Use the GitHub App's identity. Keep the workflow's `permissions:` block anyway: the workflow token is still the fallback | +| `allow-unisolated` | no | `"true"` runs the agent as the job's own user when it cannot be locked down, where it can reach this job's tokens and secrets. Off by default | +| `app-id` / `app-private-key` | no | Post as the GitHub App. Keep the workflow's `permissions:` block: the workflow token is the fallback | ## Approving a plan Before a job changes anything, Test Companion posts its plan and waits for someone with write access to reply `go`. Read the plan before you reply. -Without the GitHub App, Test Companion posts as `github-actions[bot]`, the same -account every other workflow in your repository uses. It can only recognise its -own plans by that account and a hidden marker. So a different workflow that -copies outside text into a comment could imitate a plan. A reply of `go` only -ever approves a plan for the pull request's current commit. With the App, Test -Companion posts under its own account, which no other workflow can use. +Without the GitHub App, Test Companion posts as `github-actions[bot]`, the +account every workflow in your repository shares, and recognises its own plans +by that account and a hidden marker. Another workflow that copies outside text +into a comment could imitate a plan. A `go` only ever approves a plan for the +pull request's current commit. With the App, Test Companion posts under its own +account, which no other workflow can use. ## How the CLI is installed -Each run downloads the CLI installer from -`test-companion.browserstack.com/cli/install.sh`. The installer script itself -is not checked against a fixed checksum, so a run trusts that address. Pinning -`cli-version` with `cli-sha256` (above) makes the installer refuse any CLI -build other than the one you tested. - -## What runs is not decided here - -This file is awkward to change once it is in many repositories, so it stays -near-empty on purpose. Which jobs run, on which events, with which spending -limits, lives in `testcompanion-pipeline.yml` in your own repository, which you -can edit freely without touching the workflow above. +Each run downloads the installer from +`test-companion.browserstack.com/cli/install.sh`. The installer itself is not +checked against a fixed checksum, so a run trusts that address. Pinning +`cli-version` with `cli-sha256` makes it refuse any CLI build but the one you +tested. ## Notes for maintainers -This is a **composite** action with **no dependencies** and no `dist/` bundle to -rebuild — unlike the JavaScript actions elsewhere in this repository. The -sources in `src/` are what run. +A composite action with no dependencies and no `dist/` bundle: the sources in +`src/` are what run. -Tests: `cd test-companion && node --test`. They need no network and no -credentials. +Tests: `cd test-companion && node --test`. No network or credentials needed. diff --git a/test-companion/action.yml b/test-companion/action.yml index 6f1dfe1..225966f 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -41,10 +41,8 @@ inputs: cli-version: description: > - Exact Test Companion CLI version to install, e.g. 1.39.8. Pinning keeps - the same pipeline behaving the same next month. LEAVE IT UNSET AND THE - LATEST RELEASE IS INSTALLED, which can change between runs — so pin it - if you want a pipeline that does not move under you. + Exact Test Companion CLI version to install, e.g. 1.39.8. Unset installs + the latest release, which can change between runs. required: false cli-sha256: description: > @@ -57,22 +55,19 @@ inputs: required: false build-effect: description: > - Whether a bad outcome reddens THIS step. fail-on-break (the default) - reddens it when the run itself did not work — it broke, it could not - start, or it hit a limit — and leaves it green when the run worked - and the tests failed. report never reddens it. fail reddens it on - anything that is not a success, test failures included. The Test - Companion check always reports truthfully regardless. + Whether a bad outcome fails this step. fail-on-break (default) fails it + when the run broke, could not start or hit a limit, but not for test + failures. report never fails it. fail fails it on anything but success. + The Test Companion check reports the real outcome either way. required: false default: "fail-on-break" allow-unisolated: description: > - Run even when the agent cannot be locked down. Test Companion runs its - AI agent as a separate user with no admin rights, so it cannot reach - this job's tokens and secrets. That needs a Linux runner where the job - has password-free sudo, which GitHub's own runners have. On a runner - without it, the run stops unless this is "true", in which case the - agent runs as the job's own user and CAN reach them. + Run even when the agent cannot be locked down. The agent normally runs + as a separate user that cannot reach this job's tokens and secrets, + which needs Linux with password-free sudo (GitHub-hosted runners have + it). Without that the run stops, unless this is "true": then the agent + runs as the job's own user and can reach them. required: false default: "false" github-token: @@ -94,10 +89,8 @@ outputs: value: ${{ steps.run.outputs.outcome }} result-file: description: >- - Path to the run's RESULT — the test cases it generated, with their steps. - Written whether or not they reached Test Management, which is the only - record a preview run leaves behind. Distinct from summary-file, which - records how the run went rather than what it produced. + Path to the test cases the run generated, with their steps, whether or + not they reached Test Management. summary-file says how the run went. value: ${{ steps.run.outputs.result-file }} summary-file: description: "Path to the run's report file." @@ -109,8 +102,7 @@ outputs: runs: using: composite steps: - # A fresh runner has no git identity, and several jobs commit internally. - # Scoped to this repository rather than --global so nothing outlives the run. + # Several jobs commit. Repo-scoped, not --global, so nothing outlives the run. - name: Give this run a git identity shell: bash run: | @@ -126,27 +118,13 @@ runs: TC_SHA256: ${{ inputs.cli-sha256 }} run: | set -euo pipefail - # $RUNNER_TEMP, not /tmp: it is per-job and cleaned up, where a - # fixed /tmp path is shared with everything else on the runner - # and can be written before this step runs. The Action's own - # report files already take this care; this step did not. + # $RUNNER_TEMP is per-job; a fixed /tmp path could be written by something else first. installer="${RUNNER_TEMP:-/tmp}/tc-install.sh" curl -fsSL -o "$installer" https://test-companion.browserstack.com/cli/install.sh sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" - # pinning only means something if the pin actually took. The - # installer refuses an unverified pinned install on its own, but a - # silent fallback to latest would leave the run reporting a version - # nobody asked for — and "the same pipeline behaves the same next - # month" is the entire point of the input. - # EXACT, not a substring. `${version#*$TC_VERSION}` accepted a pin - # of 1.2.0 when 11.2.0 was installed, and a pin of 1.1 when 1.1.9 - # was — so the check passed for a version nobody asked for, which - # is the one thing it exists to catch. - # The first version-shaped token. Stripping non-digits instead - # would fold the architecture into the number, turning a version - # line ending "linux/amd64" into 1.1.964. + # The pin must have taken: exact match on the first version-shaped token. installed="$(printf '%s' "$version" | grep -oE '[0-9]+(\.[0-9]+)+' | head -1 || true)" if [ -n "${TC_VERSION:-}" ] && [ "$installed" != "$TC_VERSION" ]; then echo "::error::Asked for Test Companion CLI $TC_VERSION but got '$version'." >&2 @@ -155,119 +133,54 @@ runs: echo "cli-version=$version" >> "$GITHUB_OUTPUT" echo "::notice::Test Companion $version" - # The CLI only picks up ambient BROWSERSTACK_* credentials during `auth - # login`; other commands just find no stored session and exit 3. Assuming - # the environment variables alone were enough cost a live run to discover. + # The CLI only reads BROWSERSTACK_* credentials during `auth login`. - name: Sign in shell: bash env: BROWSERSTACK_USERNAME: ${{ inputs.username }} BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} - # This is the FIRST tc command of the job, and without the - # pin here the CLI self-upgrades during `auth login` — before the - # Run step below ever applies it. The pin then correctly reports - # a mismatch against the binary the pin was meant to preserve, - # so ASKING FOR A PINNED VERSION MADE THE RUN FAIL. Seen live on - # 2026-10-29 when tc 1.1.9 shipped: "pinned to version 1.1.8 but - # the installed binary is 1.1.9". - # - # Every step that runs tc needs this, not just the one that does - # the work. + # Every step that runs tc needs the pin, or the CLI self-upgrades on its first command. TC_PIN_VERSION: ${{ inputs.cli-version }} TC_SKIP_AGENT_SKILL_INSTALL: "1" TC_ACTION_CLI_VERSION: ${{ steps.install.outputs.cli-version }} run: | set -euo pipefail - # Bounded: a surface that cannot draw a browser must never sit waiting - # on an interactive sign-in. + # Bounded: never wait on an interactive sign-in. timeout 120 tc auth login - # Jobs that read a failed build — rca, verdict, fix — fetch its logs - # with `gh`, and on a runner `gh` has no credentials. Three live runs - # on 2 Nov all got as far as finding the right build and then stopped - # with "The GitHub CLI is installed but not signed in… run `gh auth - # login` in your terminal". Nobody can follow that here: there is no - # terminal, no browser, and nobody watching. - # - # The token is already in this job for our own API calls, so sign gh - # in with it. Deliberately NOT exported into the environment the CLI - # inherits: childEnvFor keeps only an allowlist, and the agent's own - # shell commands must not be able to read the token out of env. - # - # This puts the token in gh's config, inside this job's temp folder. - # The agent runs as a separate, locked-down user that cannot reach - # that folder: its own gh talks to a read-only relay instead (see - # sandbox-policy.mjs). This sign-in is for the Action's own use, such - # as planting the "already running" marker before the run. Only an - # unisolated run (allow-unisolated) leaves it within the agent's reach. + # rca, verdict and fix read build logs with gh, which has no credentials on a runner. + # The token goes in a job-scoped gh config, not the CLI's environment. The locked-down + # agent can't reach it; it reads GitHub through a read-only relay instead. - name: Sign gh in, so a failed build can be read shell: bash env: GH_TOKEN: ${{ inputs.github-token }} run: | - # NOT `set -e`. Most jobs never read a build, and a problem - # signing gh in must not take down a generate or test-pr run. - # The first version did exactly that: the login failed, the step - # exited 1, and the whole Run step was skipped (live run - # 41445939531). Best effort, always exit 0. + # Not `set -e`: most jobs never read a build, so this must not fail the job. set -uo pipefail if [ -z "${GH_TOKEN:-}" ]; then echo "::notice::No GitHub token, so jobs that read a failed build will not be able to fetch its logs." exit 0 fi - # Already signed in is not a failure — a self-hosted runner may - # have its own credentials, and overwriting them is not ours to do. - # - # Checked with GH_TOKEN REMOVED, because this step has the token - # in its own environment and `gh auth status` reports that as - # being signed in. The first version of this check passed for - # that reason, skipped the login, and left the CLI — which does - # NOT get the token — still unauthenticated. The run then failed - # exactly as before. What matters here is - # whether a credential is STORED, not whether this shell happens - # to hold one. + # Leave a runner's own stored credentials alone. GH_TOKEN is removed so + # `gh auth status` reports stored credentials, not this shell's variable. if env -u GH_TOKEN -u GITHUB_TOKEN gh auth status >/dev/null 2>&1; then echo "gh already has stored credentials on this runner; leaving them alone." exit 0 fi - # A JOB-SCOPED config dir, never the runner's shared one. A - # self-hosted runner outlives the job, so signing in to - # ~/.config/gh would leave the NEXT job — possibly another - # repository's — holding this repository's token. Cleaned up - # below whatever happens. + # Job-scoped, so a self-hosted runner's next job doesn't inherit this token. export GH_CONFIG_DIR="${RUNNER_TEMP}/tc-gh-config" mkdir -p "$GH_CONFIG_DIR" chmod 700 "$GH_CONFIG_DIR" echo "GH_CONFIG_DIR=$GH_CONFIG_DIR" >> "$GITHUB_ENV" - # OURS, so the teardown knows it may delete it. The check above - # leaves a runner's own credentials alone — but a self-hosted - # runner can export GH_CONFIG_DIR machine-wide, and the teardown - # then saw a variable it had not set and removed somebody else's - # config. The sign-in respected them; the - # cleanup has to as well. + # Marks the dir as ours, so cleanup never removes a runner-wide GH_CONFIG_DIR. echo "TC_OWNS_GH_CONFIG_DIR=1" >> "$GITHUB_ENV" - # GH_TOKEN REMOVED for the login itself. gh refuses to store a - # credential while that variable is set — "The value of the - # GH_TOKEN environment variable is being used for - # authentication. To have GitHub CLI store credentials instead, - # first clear the value from the environment." The token still - # reaches gh on stdin; it just must not also be in the env. + # gh won't store a credential while GH_TOKEN is set; the token goes in on stdin. if printf '%s' "$GH_TOKEN" | env -u GH_TOKEN -u GITHUB_TOKEN gh auth login --with-token; then echo "gh signed in for this job." - # Let git use that credential too, for this host only. - # - # A live run's dedupe claim is a real `git push` to origin - # . With `persist-credentials: false` in the workflow - # which is the right setting, and what the starter ships - # there is no credential for it, so a live `fix` or `generate` - # died in under a second at the claim, before doing any work - # Preview never hit it, which is why - # this went unnoticed. - # - # No token is added to any environment variable: git asks gh, - # and gh reads the job-scoped config dir set above. Undone in - # the cleanup step, which matters on a self-hosted runner. + # Let git use it for github.com: the CLI's dedupe claim pushes, and the starter sets + # persist-credentials: false. Undone in the cleanup step. if env -u GH_TOKEN -u GITHUB_TOKEN gh auth setup-git --hostname github.com 2>/dev/null; then echo "TC_ADDED_GIT_HELPER=1" >> "$GITHUB_ENV" echo "git will use gh's credential for github.com." @@ -286,14 +199,11 @@ runs: # The CLI picks these up on a non-interactive shell with no extra flags. BROWSERSTACK_USERNAME: ${{ inputs.username }} BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} - # Stops the CLI upgrading itself mid-pipeline, so a pinned version stays - # pinned and the 90-second first-response budget is not spent on a download. + # Stops the CLI upgrading itself mid-run. TC_PIN_VERSION: ${{ inputs.cli-version }} - # Only this step may talk to the GitHub API. Deliberately not exported to - # the environment the agent's own shell commands run in. + # For the Action's own API calls; not passed to the agent. GH_TOKEN: ${{ inputs.github-token }} - # A CI runner has no Claude Code to install a skill into; it just adds - # noise and a write to the home directory. + # No Claude Code on a runner to install a skill into. TC_SKIP_AGENT_SKILL_INSTALL: "1" TC_ACTION_WORKFLOW: ${{ inputs.workflow }} TC_ACTION_PREVIEW: ${{ inputs.preview }} @@ -307,58 +217,32 @@ runs: - name: Attach the report if: always() && steps.run.outputs.summary-file != '' - # v4.6.2, pinned to a commit: a tag can be repointed by its owner, - # and this step runs in the customer's job with the customer's token. + # v4.6.2, pinned to a commit: a tag can be repointed. uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: test-companion-report path: ${{ steps.run.outputs.summary-file }} if-no-files-found: ignore - # Separate artifact from the report, because it answers a different - # question and has a different reader: the report says how the run went, - # this is the work itself. For a preview run it is the only place that - # work survives, so it is attached even when the run is about to go red. + # The generated work itself, attached even when the run is about to go red. - name: Attach the generated test cases if: always() && steps.run.outputs.result-file != '' - # v4.6.2, pinned to a commit: a tag can be repointed by its owner, - # and this step runs in the customer's job with the customer's token. + # v4.6.2, pinned to a commit: a tag can be repointed. uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: test-companion-result path: ${{ steps.run.outputs.result-file }} if-no-files-found: ignore - # The token must not outlive the job. On a GitHub-hosted runner the - # machine is destroyed anyway; on a self-hosted one it is not, and - # this is the difference between a token that lives for one job and - # one that waits around for the next. + # Matters on a self-hosted runner, which outlives the job. - name: Take gh's credentials back off the runner if: always() shell: bash run: | set -uo pipefail [ -n "${GH_CONFIG_DIR:-}" ] || exit 0 - # The credential helper first. It lives in the GLOBAL git config, - # which outlives the job on a self-hosted runner, and it points - # at the config dir removed below — leaving it would hand the - # next job a helper that answers nothing, and a git command that - # stops to ask for a password rather than failing. - # - # ONLY THE ONE WE ADDED. `--unset-all` removed EVERY helper for - # that URL, including ones the runner's owner set up long before - # us — manager-core, a PAT helper — and it ran even when our own - # setup-git had failed and we had added nothing. The first Test - # Companion job on a self-hosted runner permanently broke that - # runner's git auth. `--unset` with a - # value pattern removes our entry and leaves the rest. - # - # BOTH entries, not just ours-looking-one. `gh auth setup-git` - # writes TWO values for each host: an EMPTY helper, which clears - # any general helper inherited from a broader scope, and then its - # own. Removing only the second left the empty one behind, so the - # runner's manager-core (or whatever it had) stayed disabled for - # github.com — the same breakage by a quieter route. + # Remove only the helper entries setup-git added (it writes an empty one and its own), + # leaving any the runner's owner configured. if [ "${TC_ADDED_GIT_HELPER:-}" = "1" ]; then for host in github.com gist.github.com; do git config --global --unset "credential.https://$host.helper" '^!.*gh auth git-credential' 2>/dev/null || true @@ -373,13 +257,11 @@ runs: echo "GH_CONFIG_DIR was not set by this job; leaving it alone." fi - # Separate from the run step so the report is attached even when the - # build effect says this run should redden the build. + # Separate step, so the report is attached even when the build goes red. - name: Apply the build effect if: always() && steps.run.outputs.should-fail == 'true' shell: bash - # Through env, never pasted into the script: a step output written - # straight into `run:` becomes part of the shell code. + # Through env: a step output pasted into `run:` becomes shell code. env: OUTCOME: ${{ steps.run.outputs.outcome }} run: | diff --git a/test-companion/src/access.mjs b/test-companion/src/access.mjs index 6095d08..b762037 100644 --- a/test-companion/src/access.mjs +++ b/test-companion/src/access.mjs @@ -1,20 +1,6 @@ /** - * Who may ask Test Companion for a run. - * - * Write access is the floor and it is not negotiable. On a public repository - * anyone can comment, and a run executes an agent with --yolo against the - * customer's own machine and credentials. - * - * The pipeline file may NARROW that to a named team. It may never widen it: - * being on the team is not a way in for someone who cannot write to the - * repository. That asymmetry is why this is a function rather than two - * booleans combined at the call site. - */ - -/** - * `inTeam` is tri-state on purpose. `undefined` means we could not find out — - * no token for the organisation, API down, rate limited — and an unanswerable - * question is a no. + * Write access is always required: a run executes an agent on the runner. + * `askTeam` can only narrow that. `inTeam` undefined (lookup failed) counts as no. */ export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { if (!hasWriteAccess) return false @@ -22,18 +8,7 @@ export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { return inTeam === true } -/** - * Can this run write test code where the settings say it should? - * - * Tests go in the pull request's own repository by default. A repository that - * points somewhere else needs either the GitHub App installed on both, or a - * token with write access stored as a secret. - * - * The message names WHICH of the two is missing: a bare "permission denied" - * halfway through a run leaves someone guessing, after the credits are spent. - * - * `canPush` is tri-state, same rule as above — undefined is a no. - */ +/** Whether this run can write to the configured tests repository. `canPush` undefined counts as no. */ export function describeTestRepoAccess({ testsRepo, thisRepo, canPush } = {}) { const target = String(testsRepo ?? "").trim() if (!target) return { ok: true } diff --git a/test-companion/src/annotations.mjs b/test-companion/src/annotations.mjs index 2d77dc9..1a64849 100644 --- a/test-companion/src/annotations.mjs +++ b/test-companion/src/annotations.mjs @@ -1,43 +1,13 @@ /** - * Putting each failing test on the diff. - * - * An annotation puts the failure on the line itself, in the files-changed - * view the reviewer is already in, instead of sending them to the run logs. - * - * Two limitations, both easy to misread: - * - * The line is where the test is DECLARED, not where the assertion failed. - * Playwright prints `file:line:col` for the declaration and nothing - * machine-readable for the failing statement. - * - * The path comes from the reporter, relative to the test runner's root - * (Playwright's `testDir`), while GitHub wants it relative to the repository - * root. Those match in the common layout and differ when the suite lives in a - * subdirectory; GitHub silently drops an annotation whose path it cannot - * place. - * - * Reporters that print no locator at all (mocha, wdio) produce no - * annotations. Every failure is named in the report either way. + * Check-run annotations for failing tests. The line is where the test is + * declared, not the failing assertion. Paths are relative to the runner's root, + * so GitHub drops them when the suite lives in a subdirectory. */ -/** - * GitHub rejects a check-run request carrying more than 50 annotations. A - * suite with 300 failures must not lose the whole set to one rejected call, so - * the list is cut rather than the request risked. - */ +// GitHub rejects a check-run request with more than 50 annotations. export const MAX_ANNOTATIONS = 50 -/** - * Read the failing tests back out of the report. - * - * Needs its own reader because parseSummary builds an object, where repeated - * keys collapse and only the last failing test would survive. - * - * The writer separates the name from the location with a TAB (see - * RunSummary.Render in cli/pkg/cli/workflow/summary.go) because a test name is - * whatever the reporter printed and may legitimately contain a colon, a path, - * or something that looks exactly like a locator. - */ +// The CLI separates name and location with a tab, since a test name may contain colons. export function parseFailedTests(reportText) { const out = [] for (const line of String(reportText ?? "").split("\n")) { @@ -59,11 +29,6 @@ export function parseFailedTests(reportText) { return out } -/** - * A path GitHub can actually annotate: inside the repository, relative to its - * root. An absolute path annotates nothing, and one climbing out of the - * checkout would point at a file that is not in the pull request at all. - */ const insideTheRepo = (file) => typeof file === "string" && file.length > 0 && @@ -71,7 +36,6 @@ const insideTheRepo = (file) => !/^[A-Za-z]:[\\/]/.test(file) && !file.split(/[\\/]/).includes("..") -/** Turn the failing tests into check-run annotations. */ export function buildAnnotations(failedTests) { const out = [] for (const t of failedTests ?? []) { @@ -79,9 +43,7 @@ export function buildAnnotations(failedTests) { const known = Number.isInteger(t.line) && t.line > 0 out.push({ path: t.file, - // GitHub requires a line. Without one the top of the file is the - // only honest place left, and the message says why it is there so - // nobody reads line 1 as the failing statement. + // GitHub requires a line; fall back to 1 and say so in the message. start_line: known ? t.line : 1, end_line: known ? t.line : 1, annotation_level: "failure", diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs index 1aff69d..95f4f53 100644 --- a/test-companion/src/answer.mjs +++ b/test-companion/src/answer.mjs @@ -1,59 +1,29 @@ -/** - * Catch the agent's own closing words out of the run's output. - * - * Every job announces what it concluded. Under `--json` the CLI prints one - * event per line and `{"event":"answer","text":...}` carries the prose, for - * every verb. What varies is whether the CLI also saves it to the result - * file, which it does for `rca` and `verdict` alone. - * - * Reading the stream here means every other job can show its answer too — - * it matters most for `fix` in preview, where the prose IS the product. - */ +// Captures the agent's final `{"event":"answer"}` line from the CLI's --json stream. import { spawn } from "node:child_process" import { StringDecoder } from "node:string_decoder" -/** - * How much of an unterminated line to hold before giving up on it. - * - * The agent runs with --yolo and picks its own shell commands, so one of them - * printing megabytes with no newline is a thing that happens. Buffering that - * to the end would take the runner down over output we are only scanning. - */ +// Cap on an unterminated line; the agent's commands can print megabytes without a newline. export const MAX_PARTIAL_LINE = 1 << 20 -/** - * A scanner for the CLI's event stream. - * - * Feed it `push(chunk)` as the bytes arrive and `end` when the process - * closes; `answer` is the last thing the agent said. - */ export function createAnswerCapture() { - // Chunk boundaries fall mid-character: agent prose is full of em dashes and - // arrows, and decoding each chunk on its own splits one in half, breaks the - // JSON and silently drops the answer — on exactly the runs with most to say. + // Chunks can split a multi-byte character. const decoder = new StringDecoder("utf8") let pending = "" - // True while discarding the tail of a line that overran MAX_PARTIAL_LINE, - // so its remainder is not mistaken for the start of a fresh line. + // Discarding the rest of a line that overran MAX_PARTIAL_LINE. let skipping = false let answer - /** Keep this line's text if it is an answer event. Last one wins. */ const read = (line) => { const t = line.trim() - // Cheap gate first: most of this stream is not an object, and parsing - // every log line the agent's own commands print would be wasteful. if (!t.startsWith("{") || !t.endsWith("}")) return let event try { event = JSON.parse(t) } catch { - // Not an event. The stream carries whatever the run printed. return } if (!event || typeof event !== "object" || event.event !== "answer") return const text = typeof event.text === "string" ? event.text.trim() : "" - // An empty answer must not erase a real one that came before it. if (text) answer = text } @@ -75,28 +45,17 @@ export function createAnswerCapture() { skipping = true } }, - /** The last line often has no newline, because the process ended. */ end() { pending += decoder.end() if (!skipping && pending) read(pending) pending = "" }, answer: () => answer, - /** Bytes held for an unfinished line — for the test that pins the cap. */ pending: () => pending.length, } } -/** - * Run a command, showing its output live AND reading it on the way past. - * - * stdout is piped so it can be scanned and echoed on; stdin and stderr stay - * inherited, so a prompt or a crash reaches the log unchanged. The echo - * happens first — someone watching a ten-minute run must not wait on our - * parsing. - * - * Never rejects. Losing the answer is not a reason to lose the report too. - */ +/** Run a command, echoing stdout live while scanning it for the answer. Never rejects. */ export function runCapturingAnswer(command, args, childEnv) { return new Promise((resolve) => { const capture = createAnswerCapture() @@ -106,11 +65,8 @@ export function runCapturingAnswer(command, args, childEnv) { capture.push(chunk) }) child.stdout.on("error", () => {}) - // The command is missing or not executable. Null status is what - // spawnSync gave here too, and the caller turns it into 1. child.on("error", () => resolve({ status: null, answer: undefined })) - // `close`, not `exit`: exit can fire with stdout still draining, and the - // closing message is the last thing through. + // `close`, not `exit`: stdout may still be draining at exit. child.on("close", (status) => { capture.end() resolve({ status, answer: capture.answer() }) diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index 193e0f7..725c524 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -1,29 +1,12 @@ /** - * Signing in as the Test Companion GitHub App. - * - * The comment, the check and the commits come from the App's own account - * rather than github-actions[bot]. The workflow's permissions block stays: - * the workflow token is still the fallback when the App cannot be reached. - * - * It does not run anything — a run still happens on the customer's machine - * with their credentials. This only changes who GitHub thinks is talking. - * - * Zero dependencies: Node's crypto signs the JWT and `gh` exchanges it. The - * private key never leaves this process and never reaches the agent's - * environment — childEnvFor is an allowlist and these names are not on it. + * Signing in as the GitHub App, so comments, checks and commits come from the + * App rather than github-actions[bot]. The workflow token is the fallback. */ import { createSign } from "node:crypto" -/** base64url, which is what JWT uses — base64 with three characters swapped. */ const b64url = (input) => Buffer.from(input).toString("base64url") -/** - * The App's credentials, or nothing. - * - * Both halves are required: an id with no key cannot sign, a key with no id - * cannot say who it is. Either alone falls back to the workflow token - * silently, because a repository that never set this up is the normal case. - */ +/** The App's id and key, or undefined unless both are set. */ export function appCredentials(env = {}) { const appId = String(env.TC_ACTION_APP_ID ?? "").trim() const privateKey = String(env.TC_ACTION_APP_PRIVATE_KEY ?? "").trim() @@ -31,23 +14,15 @@ export function appCredentials(env = {}) { return { appId, privateKey } } -/** How long a JWT lives. GitHub refuses anything over ten minutes. */ +// GitHub refuses a JWT that lives over ten minutes. const JWT_LIFETIME_SECONDS = 540 -/** - * The claims GitHub expects on an App JWT. - * - * `iat` is backdated a minute on purpose. GitHub rejects a token whose issued-at - * is in the future, runner clocks drift, and the failure reads "issued at is in - * the future" on an otherwise perfectly configured workflow. Backdating is the - * documented workaround. - */ +/** `iat` is backdated a minute because runner clocks drift and GitHub rejects a future `iat`. */ export function jwtClaims(appId, nowSeconds) { const iat = Math.floor(nowSeconds) - 60 return { iat, exp: iat + JWT_LIFETIME_SECONDS, iss: String(appId) } } -/** A signed App JWT. RS256 is the only algorithm GitHub accepts here. */ export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })) const payload = b64url(JSON.stringify(jwtClaims(appId, now))) @@ -56,27 +31,14 @@ export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { } /** - * What to ask for when minting an installation token: this repository only. - * - * A token minted with no body carries every permission the installation - * holds across every repository it covers, so a run in one repository would - * hold write access to all of an organisation's. - * - * The four permissions the starter asks for and this Action uses: - * contents — branches and commits for the fix and reconcile jobs - * pull_requests — the one comment, and opening a pull request - * checks — the "Test Companion" check - * actions — reading the red run, and cancelling one when a PR closes - * - * Returns undefined for a repository we cannot parse, rather than falling - * back to an unscoped request. + * Scope the installation token to this repository: without a body it covers + * every repository the App is installed on. */ export function installationTokenRequest(repo) { const parts = String(repo ?? "").split("/") if (parts.length !== 2 || !parts[0] || !parts[1]) return undefined return { - // Bare names: the API rejects "owner/name" here as not found, which - // would silently fall back to the workflow token. + // Bare name: the API rejects "owner/name" here. repositories: [parts[1]], permissions: { contents: "write", diff --git a/test-companion/src/child-env.mjs b/test-companion/src/child-env.mjs index e765dfe..ceaeafe 100644 --- a/test-companion/src/child-env.mjs +++ b/test-companion/src/child-env.mjs @@ -1,17 +1,6 @@ /** - * The environment the agent's subprocess gets — an allowlist, not a denylist. - * - * The run executes shell commands the model chooses, so everything here is - * readable by them. This is the one place in the Action where a mistake hands - * away a credential, which is why it lives in its own testable module. - * - * A denylist is not enough: a composite action's own inputs can surface as - * INPUT_* variables, reintroducing the very token a denylist removed. Listing - * what the CLI needs is the only version that stays correct as the - * surroundings change. - * - * Known gap: `actions/checkout` leaves a credential in .git/config, which - * this cannot reach. + * The agent's environment, as an allowlist: a composite action's inputs can + * reappear as INPUT_* variables, which a denylist would miss. */ export function childEnvFor(source) { const KEEP_EXACT = new Set([ @@ -29,20 +18,11 @@ export function childEnvFor(source) { "RUNNER_OS", "RUNNER_ARCH", "RUNNER_TEMP", - // Where this job's gh credentials live. A PATH, not a secret — but - // without it the CLI's gh looks in the runner's shared config, finds - // nothing, and jobs that read a failed build fail with "not signed - // in". The token itself still never enters this environment. + // A path, not a secret; the CLI's gh needs it to find its config. "GH_CONFIG_DIR", ]) - // The exception to the TC_ prefix below, and the reason this is an explicit - // list rather than a rule. The App's private key is TC_ACTION_APP_* — it - // matches the kept prefix, and it can mint installation tokens for every - // repository the App is installed on. That is a far worse thing to hand a - // shell than the workflow token this allowlist was written to keep out. + // The App credentials match the TC_ prefix but must never reach the agent. const NEVER = new Set(["TC_ACTION_APP_ID", "TC_ACTION_APP_PRIVATE_KEY"]) - // TC_* is the CLI's own configuration; BROWSERSTACK_* is how it signs in; - // the proxy variables matter inside corporate networks. const KEEP_PREFIX = ["TC_", "BROWSERSTACK_", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] const out = {} for (const [k, v] of Object.entries(source)) { diff --git a/test-companion/src/framework.mjs b/test-companion/src/framework.mjs index b5d63e3..1a0dd0f 100644 --- a/test-companion/src/framework.mjs +++ b/test-companion/src/framework.mjs @@ -1,18 +1,6 @@ -/** - * Which test framework does this repository use? - * - * Pure: the caller does the directory walk and the package.json read, this - * decides. Guessing WRONG is worse than not guessing — a starter file naming - * Cypress at a Playwright shop tells the team Test Companion understood their - * setup when it did not — so anything unrecognised returns undefined and the - * starter stays generic. - */ +/** Detect the repository's test framework, or undefined when unsure. */ -/** - * A config file is stronger evidence than a dependency: it is what actually - * runs. A repository can carry `@playwright/test` from an experiment while - * its suite is Cypress. - */ +// A config file is stronger evidence than a dependency. const BY_CONFIG = [ [/(^|\/)playwright\.config\.[cm]?[jt]s$/i, "Playwright"], [/(^|\/)cypress\.config\.[cm]?[jt]s$/i, "Cypress"], diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs index 46f588f..cf9dd11 100644 --- a/test-companion/src/gate.mjs +++ b/test-companion/src/gate.mjs @@ -1,49 +1,28 @@ /** - * Should this request run, or wait for someone to approve it? - * - * A developer asks, Test Companion says what it plans to do, a reviewer - * replies "go", and only then is anything written. - * - * The mechanism is deliberately boring: a held request runs in PREVIEW and - * its report becomes the proposal. Preview is enforced in the agent's tool - * dispatch, so "nothing is written before go" is a property of how the run - * executes rather than a promise made in a prompt. - * - * Pure, so every branch is testable without a runner. + * Should this request run now, or be proposed and wait for "go"? A held + * request runs in preview, which the agent's tool dispatch enforces. */ import { INTENT } from "./intent.mjs" -/** What to do with the request. */ export const GATE = Object.freeze({ - /** Run it. `preview` says whether for real. */ RUN: "run", - /** Run it in preview and post the report as a proposal, then end. */ PROPOSE: "propose", - /** A "go" arrived with no proposal waiting. */ NOTHING_TO_APPROVE: "nothing-to-approve", - /** A "go" arrived on a proposal older than the window. */ EXPIRED: "expired", }) const STRAIGHT_THROUGH = "straight-through" /** - * `approval` is the RESOLVED value from `tc workflow bindings --json`. - * - * Anything else — unvalidated, or a bindings read that failed — holds for - * review. Fails closed: a needless proposal costs one comment, the other - * mistake is an unreviewed run that writes. + * `approval` is the resolved value from `tc workflow bindings --json`. + * Anything unrecognised holds for review. */ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) { if (intent?.kind === INTENT.GO) { if (!proposal) return { action: GATE.NOTHING_TO_APPROVE } if (proposal.expired) return { action: GATE.EXPIRED } - // Approving is the whole point, so this is explicitly not a preview. - // - // selfApproved is reported, not refused. Asking and approving both need - // write access, so this is not a privilege boundary — it is a - // workflow-integrity signal. Refusing it outright would strand a solo - // maintainer, who could then never approve anything. + // Self-approval is reported, not refused: both need write access anyway, + // and refusing it would strand a solo maintainer. return { action: GATE.RUN, preview: false, @@ -52,49 +31,25 @@ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) } } - // They already asked for the safe thing. Holding a preview for approval - // would be asking permission to write nothing. if (intent?.preview) return { action: GATE.RUN, preview: true } - // The same reasoning for a job that CANNOT write. `rca` and `verdict` - // produce an explanation and nothing else, so there is nothing to approve - // and the proposal would be empty. - // - // PREVIEW, not just unheld. Skipping the hold is only safe because the run - // cannot write, and that has to be made true rather than assumed: these - // jobs run with --yolo and a shell, and a binding set to `mode: live` - // would otherwise hand an unapproved run write tools. + // Answer-only jobs skip the hold, so force preview: a `mode: live` binding + // would otherwise give an unapproved run write tools. if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: true } if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } - // "review-before-run", and anything unrecognised, which fails closed here. return { action: GATE.PROPOSE, preview: true } } -/** - * Jobs that produce an explanation and no artefact. - * - * Used by the gate above and by whether the Action shows the answer in the - * comment. Mirrored on the CLI side, which decides whether to write the - * answer to the result file at all. - */ +/** Jobs that produce an explanation and no artefact. Mirrored in the CLI. */ export const ANSWER_JOBS = new Set(["rca", "verdict"]) -/** Where a change stops being reviewable in one go. Both are assumptions. */ +// Rough thresholds for a change too big to review in one go. const MANY_FILES = 20 const MANY_LINES = 1000 -/** - * Is this change too big to test in one go? - * - * An unreadable pull request is NOT large. Getting that backwards would tell - * every reviewer their change is too big whenever the API call failed. - * - * Reports the size only. Choosing which part of a thirty-file change to test - * is the agent's judgement, and there is no input for handing it that - * instruction, so the reviewer narrows the proposal before approving it. - */ +/** An unreadable pull request counts as not large. */ export function describeChangeSize(pr) { const files = Number(pr?.changed_files ?? 0) const lines = Number(pr?.additions ?? 0) + Number(pr?.deletions ?? 0) diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index 9af9313..82352a7 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -1,219 +1,89 @@ /** - * What did the comment ask for? - * - * "@browserstack fix the failing tests" becomes a job name here, in the - * Action, before the CLI is spawned. Handing the sentence to the agent - * instead would spend the customer's credits discovering that "make coffee" - * was never a job. - * - * Pure — text in, intent out, no I/O — so the whole vocabulary is testable in - * milliseconds. - * - * The phrasing people use is not a closed set, so UNKNOWN is a real answer - * rather than a failure: an unmatched comment gets the list of what Test - * Companion can do. + * Maps a comment to a job before the CLI starts, so an unrecognised request + * costs no agent run. UNKNOWN replies with the list of what we can do. */ -/** What a comment turned out to be asking for. */ export const INTENT = Object.freeze({ - /** Run a Test Companion job. `workflow` names it, or is null for "whatever is bound". */ + /** `workflow` is null for "whatever is bound". */ WORKFLOW: "workflow", - /** Asked us to stop. Nothing stops any more, but the word is still answered. */ + /** No longer stops anything; still answered with what to do instead. */ STOP: "stop", - /** Approve a waiting proposal, which starts the real run. */ GO: "go", - /** Open the pull request that adds a starter pipeline file. */ SETUP: "setup", - /** We could not place it: reply with the list, run nothing. */ UNKNOWN: "unknown", }) -/** - * GitHub's "Quote reply" button copies the comment being replied to in as `>` - * lines, mention and all. Without dropping those first, replying to Test - * Companion's own "comment stop to cancel" message would cancel the run: the - * quoted words match before the person's own ever get read. - */ +/** Drop `>` quote-reply lines, which would otherwise match before the person's own words. */ const withoutQuotes = (text) => text .split("\n") .filter((line) => !/^\s*>/.test(line)) .join("\n") -/** - * A leading politeness, which says nothing about what is being asked for. - * "please stop" is a stop; dropping it lets the patterns below anchor. - */ const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+/i -/** - * Checked before any job word, so "stop, don't fix anything" cannot start a - * fix run. - * - * ANCHORED, because matching the word anywhere made "don't stop until the - * tests pass" cancel the run. What the comment opens with is what it is - * asking for; the same word later in a sentence is usually about something - * else. - */ -/** - * STILL RECOGNISED, but it no longer does anything. - * - * Proposals already sitting in live threads tell people to say it, and - * answering those with "that did not name anything I can do" would be true and - * useless. main.mjs says what to do instead. - */ +// STOP, SETUP and GO are anchored to the start: "don't stop until the tests +// pass" or "the test setup is broken" must not match. const STOP = /^stop\b/i -/** - * "set up this repo" opens the pull request that adds the starter file. - * Anchored for the same reason — "the test setup is broken, fix it" was - * opening a settings pull request instead of running fix. - */ const SETUP = /^set\s?-?\s?up\b/i -/** "Any comment MAY ask for a single preview run." A modifier, not a job. */ const PREVIEW = /\bpreview\b/i -/** Run the bound jobs again on this commit — which is what a re-check is. */ const RECHECK = /\bre-?\s?check\b/i -/** - * Approve a waiting proposal. - * - * ANCHORED, like STOP and SETUP: only a comment that OPENS with the word (after - * any politeness) approves. Matched anywhere, a comment that merely mentions it - * approved a waiting plan and started a live run that writes to the branch: - * "don't go ahead yet, explain this failure" and "no, do not approve that" - * both did. Opening with it still allows edits after it: "go, and fix case 3's - * title". `\b` keeps "going" and "gopher" out. - */ +// "don't go ahead yet" must not approve; "go, and fix case 3's title" should. const GO = /^(?:go|approved?)\b/i -/** - * An approval and nothing else. - * - * ONE definition, exported, because three readers have to agree on it: this - * file (does the reply name a plan?), plan-run.mjs (may an UNADDRESSED - * comment reach us at all?) and report.mjs (did the approver edit the plan?). - * Each past disagreement between them was a bug. - */ +/** Shared with plan-run.mjs and report.mjs, which must agree on it. */ export const BARE_APPROVAL = /^\s*(?:go|approved?)\s*[.!]*\s*$/i -/** - * An approval that names WHICH plan — `go #`, or `go `. - * - * ANCHORED to the approval word and to the end of the comment. Matching a - * `#id` anywhere would turn "go — and pull the steps from - * #issuecomment-…" into an approval of a plan that does not exist. - * - * Six digits at least: a GitHub comment id is ten digits, and `#25` is - * somebody pointing at a pull request. - */ +/** `go #` or `go `. Six digits minimum so `#25` (a PR) is not read as a plan. */ export const APPROVAL_WITH_PLAN = /^\s*(?:go|approved?)\s+\S*#(?:issuecomment-)?(\d{6,})\s*[.!]*\s*$/i -/** Is this reply an approval and nothing else, with or without a plan named? */ export const isApprovalOnly = (text) => BARE_APPROVAL.test(String(text ?? "")) || APPROVAL_WITH_PLAN.test(String(text ?? "")) -/** - * WHICH plan a reply is approving. - * - * `go #`, or the comment link GitHub gives you when you copy a comment's - * address. A bare `go` names nothing and the newest waiting plan wins, which - * is right when there is one and a coin toss when there are two. - */ const namedProposal = (text) => APPROVAL_WITH_PLAN.exec(String(text ?? ""))?.[1] -/** - * Plain words to job names, most specific first. - * - * The names on the right are the CLI's: `tc workflow run` only accepts its - * own registered jobs, so "explain" has to become `rca` and "find the bad PR" - * has to become `verdict` here, or the run fails with "unknown workflow". - */ +/** Phrases to CLI job names, most specific first; the first job the repo has bound wins. */ const PHRASES = [ - // Before the `fix` rule: "find the bad PR" shares no word with it, but - // keeping the multi-word jobs first means a later single-word rule can never - // steal one of them. [/\bfind\s+(the\s+)?bad\s+(pr|prs|pull requests?)\b/i, ["verdict"]], - // Each phrase maps to a PREFERENCE LIST: the first one this repository has - // actually bound wins, and the first overall is the fallback. - // - // "test this PR" is why the list exists. `test` runs a project's EXISTING - // suite and never reads the diff, which is what test-pr replaced. A - // repository still bound to the older job keeps working; one that has - // moved gets the job that reads the change. + // Falls back to `test` for repositories still bound to it. [/\btest\s+(this\s+)?(pr|pull request|change|diff)\b/i, ["test-pr", "test"]], [/\bexplain\b/i, ["rca"]], [/\bfix\b/i, ["fix"]], ] -/** The first candidate this repository has bound, or the first overall. */ const pickBound = (candidates, workflows) => candidates.find((c) => (workflows ?? []).includes(c)) ?? candidates[0] -/** - * A job asked for by its own name. - * - * PHRASES covers four of the CLI's fourteen jobs, so the names a repository - * has actually bound are matched literally as a fallback. - * - * Longest first, so "mobile run" is never read as "run". Word boundaries, so - * "regenerated the lockfile" is not a request to run `generate`. - */ +/** A bound job named literally, longest first so "mobile run" is not read as "run". */ function namedWorkflow(text, workflows) { const said = wordsOf(text) const byLength = [...(workflows ?? [])].filter(Boolean).sort((a, b) => b.length - a.length) return byLength.find((name) => saysAll(said, wordsOf(name))) } -/** - * The words in a string, lowercased. - * - * Underscore counts as part of a word, matching what `\b` does, so `test_pr` - * is one word and is not read as the job `test-pr`. - */ +// Underscore is a word character, so `test_pr` is not read as `test-pr`. const wordsOf = (text) => String(text ?? "") .toLowerCase() .match(/[a-z0-9_]+/g) ?? [] -/** - * Does `said` contain `name` as a consecutive run of WHOLE words? - * - * Word comparison rather than a regex built from the name, so there is - * nothing to escape. - * - * One consequence: punctuation inside a name is a word break, so a job called - * `a.c` is also found in the words "a c". Names come from the repository's own - * pipeline file, and being slightly more willing to recognise one is the safe - * direction — the alternative is telling someone their own job name is not a - * job. - */ const saysAll = (said, name) => name.length > 0 && said.some((_, i) => name.every((w, j) => said[i + j] === w)) -/** - * Read a comment's instruction — the body with the @mention already stripped by - * plan-run.mjs — and say what it asked for. - * - * `workflows` is what the repository has bound, so a job can be asked for by - * name. Without it, only the written-out phrases are understood. - */ +/** `instructions` is the comment with the @mention already stripped. */ export function readIntent(instructions, { workflows = [], proposalWaiting = false } = {}) { const text = withoutQuotes(String(instructions ?? "")).trim() if (!text) return { kind: INTENT.UNKNOWN } - // Before the job words, so "stop, don't fix anything" cannot start a fix - // run. It no longer stops anything; it answers with what to use instead. + // Before job words, so "stop, don't fix anything" cannot start a fix. const asked = text.replace(POLITENESS, "") if (STOP.test(asked)) return { kind: INTENT.STOP } if (SETUP.test(asked)) return { kind: INTENT.SETUP } const preview = PREVIEW.test(text) - // With a plan waiting, a reply most likely IS an approval — and edits - // alongside it are invited ("you can edit the list first"). Job words - // match anywhere, so testing them first would read "go, and fix case 3's - // title" as a fresh request for `fix` and drop the approval. Still after - // STOP: approving and cancelling in one breath has to cancel. + // Before job words, so "go, and fix case 3's title" stays an approval. if (proposalWaiting && GO.test(asked)) return { kind: INTENT.GO, proposalId: namedProposal(text) } for (const [pattern, candidates] of PHRASES) { @@ -222,31 +92,19 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal } } - // A bound job asked for by its own name. const named = namedWorkflow(text, workflows) if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } - // a reply of "go", with or without edits to the case list. if (GO.test(asked)) return { kind: INTENT.GO, proposalId: namedProposal(text) } - // No job named, but a clear ask: run whatever the pipeline file binds to this - // event — the same thing an event-driven run does — and carry the preview - // flag if they asked for one. + // No job named: run whatever is bound to this event. if (preview || RECHECK.test(text)) { return { kind: INTENT.WORKFLOW, workflow: null, preview } } return { kind: INTENT.UNKNOWN } } -/** - * What to offer when we could not place a request. - * - * Only what actually works today: advertising a job that does not exist - * invites a request we would have to refuse a second time. - * - * Kept next to PHRASES on purpose — a phrase people can type and a phrase we - * advertise should never drift apart. - */ +/** Keep in step with PHRASES. */ export const CAPABILITIES = Object.freeze([ { ask: "test this PR", does: "write and run tests for this change" }, { ask: "explain this failure", does: "say why a red build failed, test by test" }, diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index c2179ac..4d87881 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -1,15 +1,4 @@ -/** - * The Action's entry point. - * - * Deliberately thin. Every decision worth testing lives in plan-run.mjs, - * outcome.mjs and report.mjs, which are pure and run under `node --test` in - * milliseconds. What is left here is I/O: read the event, spawn the CLI, call - * GitHub, write the step's outputs. That split exists because the only other - * way to test this is to push a commit and wait several minutes for a runner. - * - * Zero dependencies — Node's standard library plus `gh`, which is preinstalled - * on every GitHub runner and is already how this product talks to GitHub. - */ +/** The Action's entry point: I/O only. Decisions live in the pure modules beside it. */ import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, mkdtempSync, lstatSync, constants } from "node:fs" import { spawnSync } from "node:child_process" import { join } from "node:path" @@ -79,21 +68,13 @@ import { const env = process.env const REPO = env.GITHUB_REPOSITORY ?? "" const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` -// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while -// /tmp survives on a self-hosted or reused runner. With fixed names in a shared -// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run -// would report, and upload, test cases it never produced. Falls back to tmpdir -// so the Action still runs outside a GitHub runner. +// RUNNER_TEMP is cleared per job; /tmp may hold an earlier job's files on a reused runner. const REPORT_DIR = env.RUNNER_TEMP || tmpdir() const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") -// What the run PRODUCED, as opposed to how it went. Kept separate because the -// two have separate readers: a CI step branches on the summary, a person reads -// the result. For a preview run — which refuses to save by design — this file -// is the only place the generated cases survive. +// What the run produced. For a preview run this is the only copy of the generated cases. const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") -// The artifact the result is uploaded under, named here so the report can -// point a reader at it by name. Must match the upload step in action.yml. +// Must match the upload step in action.yml. const RESULT_ARTIFACT = "test-companion-result" const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) @@ -101,11 +82,7 @@ const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GI const notice = (m) => console.log(`::notice::${m}`) const warn = (m) => console.log(`::warning::${m}`) -/** - * The gh we run, found once, at load, BEFORE the agent runs. Looked up by - * name instead, a `gh` the agent planted on PATH would be handed our token. - * See trustedGhPath. - */ +// Resolved at load, before the agent runs, so a gh it plants on PATH is never used. const GH_BIN = trustedGhPath({ exists: (p) => existsSync(p), canWrite: (p) => { @@ -118,14 +95,7 @@ const GH_BIN = trustedGhPath({ }, }) -/** - * Every gh call in the Action goes through here, never through a bare - * `spawnSync("gh", ...)`. A test checks that. - * - * Each call gets its own empty settings folder, made just before and removed - * just after (see ghEnvFrom for why). The token is passed per call, so the - * App's signed key never sits in any environment. - */ +// Every gh call goes through here (a test checks). Each call gets a fresh config dir and its own token. function runGh(args, { token, input } = {}) { if (!GH_BIN) return { status: 1, stdout: "", stderr: "no trusted gh binary" } const configDir = mkdtempSync(join(env.RUNNER_TEMP || tmpdir(), "tc-gh-call-")) @@ -140,17 +110,15 @@ function runGh(args, { token, input } = {}) { } } -/** The raw `gh` call, so a caller that cares WHY it failed can look. */ function ghRaw(args, body) { return runGh(args, { token: env.GH_TOKEN, input: body ? JSON.stringify(body) : undefined }) } -/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ +/** `gh api`, returning parsed JSON or null. Never throws. */ function gh(args, body) { const r = ghRaw(args, body) if (r.status !== 0) { - // A missing permission is a normal, documented outcome: skip that - // write, name it, carry on. It is never a reason to fail the run. + // A missing permission skips that write; it never fails the run. warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) return null } @@ -161,27 +129,8 @@ function gh(args, body) { } } -/** - * Sign in as the Test Companion GitHub App, when it is installed. - * - * The comment and the tick then come from "Test Companion" rather than - * github-actions[bot], and the workflow needs no permissions block because - * the installation token carries the App's own. - * - * It does not run anything — the run still happens on the customer's machine - * with their BrowserStack credentials. - * - * Every failure here falls back to the workflow token rather than stopping. - * A repository without the App installed is the normal case, not an error. - */ -/** - * The login this Action's comments appear under. - * - * `github-actions[bot]` with the workflow token; `[bot]` when the - * App is installed. Used to tell OUR comments from any other bot's — "any - * bot" would let another App post a proposal marker that a maintainer's `go` - * then runs live. - */ +// The login our comments appear under: github-actions[bot], or [bot] with the App. +// Used to tell our proposals from another bot's. let botLogin = "github-actions[bot]" /** HEAD before the agent ran, so a commit it made can be told from ours. */ @@ -193,38 +142,17 @@ let consumedMarker = "" /** What happened to a pull request the CLI handed back to open, for the comment. */ let handoffNote -/** - * The CLI this job installed. Ours to run: it lives in the runner's home, - * which the locked-down agent cannot reach. The agent gets its own copy. - */ +// The CLI this job installed, in the runner's home, out of the agent's reach. const TC_PATH = join(env.HOME ?? "", ".local", "bin", "tc") -/** - * Whether to run the agent unisolated when it cannot be locked down. Off by - * default: unisolated, the agent can reach every token and secret in the job. - */ +// Off by default: unisolated, the agent can reach every token and secret in the job. const ALLOW_UNISOLATED = /^true$/i.test(String(env.TC_ACTION_ALLOW_UNISOLATED ?? "").trim()) -/** - * The proposal this run is carrying out, so the report can link it. - * - * Approving one job no longer finishes off the - * others on that commit, so two plans really can be waiting and "approved" on - * its own no longer says which one this is. - */ +/** The proposal this run is carrying out, so the report can link it. */ let approvedPlan -/** - * Put the working tree on one commit. - * - * Never `--depth 1`: that SHALLOWS a clone the starter deliberately made with - * `fetch-depth: 0`, so anything looking at history afterwards sees one - * commit. The checkout's own result is checked, not just the fetch's. - * - * The URL is explicit and carries its own credential, because the starter - * sets `persist-credentials: false` — the workflow token must not sit in - * `.git/config` where a shell command the agent chose can read it. - */ +// No --depth: it would make the full clone shallow. The URL carries its own credential +// because the starter sets persist-credentials: false. function fetchAndCheckout(sha, branch) { const url = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!url || !sha) return false @@ -233,32 +161,22 @@ function fetchAndCheckout(sha, branch) { return branch ? git(["checkout", "-B", branch, sha]).ok : git(["checkout", "--detach", sha]).ok } +/** A repo-scoped App installation token, or undefined to fall back to the workflow token. */ function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined - // `creds` holds its own copy, so the key can leave - // the environment the moment it has been read. Everything this process - // starts from here on inherits that environment — including the agent, - // which runs with --yolo and chooses its own commands — and a signing key - // is not something to leave lying where they can read it. - // - // childEnvFor strips it for the agent already; this closes the same hole - // for everything else, and does not rely on remembering to list it. + // Remove the key from the environment so nothing started later, the agent included, inherits it. delete env.TC_ACTION_APP_PRIVATE_KEY let jwt try { jwt = signJwt(creds) } catch (e) { - // A malformed key is worth saying out loud: someone configured this and - // it silently did nothing. warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) return undefined } - // The JWT authenticates the APP; only the installation token can touch a - // repository. Passed per-call rather than exported, so it never lands in - // the environment at all. + // The JWT authenticates the App; only the installation token can touch a repository. const asApp = (args) => runGh(args, { token: jwt }) const found = asApp(["api", `repos/${REPO}/installation`]) @@ -271,21 +189,14 @@ function appInstallationToken() { try { const installation = JSON.parse(found.stdout) installationId = installation?.id - // What WE post as. An App comments as - // `[bot]`, and knowing that is what lets us tell our own - // proposal from another App's. - // Remembered here, but only adopted once a token is actually minted - // (below). Adopted early, a failed mint left us posting as - // github-actions[bot] while looking for our plans under the App's name. + // Adopted only once a token is minted, so a failed mint can't leave us looking under the wrong login. appLogin = installation?.app_slug ? `${installation.app_slug}[bot]` : undefined } catch { return undefined } if (!installationId) return undefined - // "for that repo only". A token minted with no body carries every - // permission the installation holds across every repository it covers — a - // run in one repository would hold write access to all of an organisation's. + // Scoped to this repository; an empty request gets every repository the installation covers. const scope = installationTokenRequest(REPO) if (!scope) { warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) @@ -298,12 +209,7 @@ function appInstallationToken() { input: JSON.stringify(body), }) - // No retry with a smaller request. An installation granted fewer - // permissions than we ask for refuses the whole request, and the old - // retry dropped the permission list, which asks for EVERYTHING the - // installation holds. If the App ever gained a permission, that retry - // would hand it to a run that never needed it. The workflow token is - // the safer fallback. + // No retry with a smaller request: dropping the permission list asks for everything the installation holds. const minted = mint(scope) if (minted.status !== 0) { warn( @@ -321,27 +227,15 @@ function appInstallationToken() { } if (!token) return undefined - // credentials are masked in the log. Before the token is used for - // anything, so there is no window where it could be echoed unmasked. + // Mask before first use. console.log(`::add-mask::${token}`) if (appLogin) botLogin = appLogin return token } /** - * The settings file as it stands on the DEFAULT BRANCH, not the checked-out - * tree. - * - * Read from the branch under test, a pull request could rewrite the rules for - * its own run — switch itself from preview to live, delete its spending - * limit, point its writes elsewhere. - * - * Returns the local path the content was written to, or undefined when there - * is no settings file there. Fails SAFE: a fetch that errors is treated as - * "no settings", a clean skip rather than a run governed by whatever happens - * to be checked out. - * - * `--raw` because this is YAML, not the JSON envelope gh parses. + * The settings file from the default branch, so a pull request cannot rewrite the rules for its own run. + * Returns the local path, or undefined (treated as no settings). */ function fetchSettings(path, ref) { const r = runGh( @@ -350,8 +244,7 @@ function fetchSettings(path, ref) { ) if (r.status !== 0) { const first = (r.stderr ?? "").trim().split("\n")[0] - // A 404 is the ordinary "this repo has no settings file yet" case. - // Anything else is worth saying out loud, because the run is about to skip. + // 404 just means no settings file yet. if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) return undefined } @@ -360,24 +253,14 @@ function fetchSettings(path, ref) { return local } -/** - * Whether `login` may ask Test Companion for a run. - * - * admin, maintain and write can; triage and read cannot. Any failure to get a - * clear answer counts as "no". - */ +/** admin, maintain and write may ask for a run. Any failure counts as no. */ function hasWriteAccess(login) { if (!login) return false const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) return ["admin", "maintain", "write"].includes(res?.permission) } -/** - * Is `login` on `team` in this repository's organisation? - * - * Tri-state on purpose: undefined means we could not find out (no token for - * the organisation, API down, rate limited), which mayAsk treats as a no. - */ +/** Undefined when we cannot find out; mayAsk treats that as no. */ function isOnTeam(login, team) { if (!login || !team) return undefined const org = REPO.split("/")[0] @@ -386,14 +269,7 @@ function isOnTeam(login, team) { return res.state === "active" } -/** - * Run git in the checkout. Never throws: every caller here treats a git - * failure as "we could not save the work", which is reported, not fatal. - */ -/** - * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on - * PATH later cannot be the one we find. Absent means we do not commit at all. - */ +// Resolved at load, before the agent runs. Absent means we do not commit. const GIT_BIN = trustedGitPath({ exists: (p) => existsSync(p), canWrite: (p) => { @@ -407,35 +283,23 @@ const GIT_BIN = trustedGitPath({ }) function git(args, extraEnv = {}) { - // A near-empty environment, and no hooks. These run AFTER the agent, which - // has had --yolo write access to the tree: a planted .git/hooks/pre-commit - // would otherwise be executed by our own `git commit` with the App's - // private key and the customer's credentials in its environment. + // Near-empty environment and no hooks: the agent has had write access to the tree. if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: { ...gitEnvFrom(env), ...extraEnv }, }) - // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first - // column for an unstaged modification, and trimming the whole output eats - // it on the first line, which once lost ten passing tests. - // Anything parsing columns reads `raw`; everything else wants `out`. + // `raw` keeps porcelain's leading space; trimming would lose the first line's status column. return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } } -/** - * The repository's default branch, or undefined when we cannot find out. - * Undefined only relaxes the default-branch refusal, so an unanswered call - * never widens anything else. - */ +/** The repository's default branch, or undefined. */ function defaultBranchName() { return gh(["api", `repos/${REPO}`])?.default_branch } let cachedEvent function readEvent() { - // Parsed once. This is called seven times in a run, and the file cannot - // change underneath us — GitHub writes it before the job starts. if (cachedEvent !== undefined) return cachedEvent cachedEvent = readEventUncached() return cachedEvent @@ -469,11 +333,6 @@ function finishSkipped(reason, detail) { "no-workflow-named": "A manual run needs a `workflow` input naming the job.", "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", - // name the permission and the line to add. The commonest cause by - // far is a workflow without `pull-requests: read` — the lookup 403s, the - // head commit is unknown, and the run fails closed. Said plainly here - // because "nothing safe to run against" sends a reader looking at their - // pull request rather than at their permissions block. "no-pr-head": "Could not read that pull request, so there is no commit it is safe to run against. " + "If this workflow has no `pull-requests: read` permission, add it.", @@ -491,14 +350,7 @@ function finishSkipped(reason, detail) { }[reason] const msg = human ?? `Nothing to do (${reason}).` notice(`Test Companion: ${msg}`) - // The "in progress" check goes up before the CLI - // runs, so the run is cancellable while it is going. Skipping out from - // under it left "Test Companion" pending for ever — and `no-binding` is - // most events, so for most repositories that was most runs. - // - // Completing it here rather than posting the check later is what keeps - // that true. Nothing deliberately leaves a check pending through this - // function: the proposal path posts its own and exits directly. + // Complete any in-progress check we posted, so it isn't left pending. for (const sha of checksMade.keys()) { postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) } @@ -508,14 +360,7 @@ function finishSkipped(reason, detail) { process.exit(0) } -/** - * no settings file means a clean pass and the file they need. - * - * Checked before anything else that could cost money or time. Left to the CLI - * this is a red build reading `open testcompanion-pipeline.yml: no such file or - * directory` — the person did nothing wrong, and the message names a file they - * have never heard of. - */ +/** No settings file: pass and show a starter. */ function finishWithStarter(path) { notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) addToJobSummary(renderNoPipelineReport(path)) @@ -525,8 +370,7 @@ function finishWithStarter(path) { } async function main() { - // Without a gh we trust, nothing can be reported, so a run would finish - // green and silent. Say so, and stop before anything is spent. + // Without a trusted gh nothing can be reported. if (!GH_BIN) { console.log( "::error::Test Companion could not find GitHub's gh command in a system folder this job cannot write to " + @@ -536,33 +380,24 @@ async function main() { process.exit(1) } - // Release 4: act as the App when it is installed, so the comment and the - // tick carry Test Companion's name. Before anything else talks to GitHub. + // Act as the App when installed, before anything else talks to GitHub. const appToken = appInstallationToken() if (appToken) { env.GH_TOKEN = appToken notice("Test Companion: acting as the GitHub App.") } - // a run without credentials does nothing, says so, and passes. This is - // the normal shape of a fork pull request, not an error. + // No credentials is the normal case for a fork pull request: skip, don't fail. if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { finishSkipped("no-credentials") } - // Before the plan, so a repository with no pipeline file gets the starter - // rather than whatever the planner happens to say about this event. const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) - // from the default branch, never from the branch under test. - // May be undefined — a repository with no settings file is a normal, and - // very common, first run. It is NOT an early exit any more: a - // comment asking us to add that file is the one thing worth doing in a - // repository that has none, and the exit used to sit in front of it. - // Nothing below costs money before the settings check further down. + // From the default branch, never the branch under test. May be absent; the setup request + // below still has to work then. const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) - // before anything else this run does, clear out proposals that ran - // out of time, so none of them leaves a check stuck on "in progress". + // Clear expired proposals first so none leaves a check stuck in progress. if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() let plan = planRun({ @@ -578,41 +413,25 @@ async function main() { }, }) if (plan.skip) { - // closing a pull request without merging stops whatever it started. + // Closing a pull request without merging stops what it started. if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) finishSkipped(plan.skip.reason, plan.skip.detail) } - // Only someone who can write to the repository may ask for a run by - // commenting. On a public repository anyone can comment, and a run runs an - // agent with --yolo against the customer's own machine and credentials. - // - // Fails CLOSED: if the permission lookup itself fails — no token, API - // down, rate limited — the run does not start. - // - // Keyed on the TRIGGER, not only on a flag the planner sets. A planner - // change that forgot the flag would delete this gate silently, and runs - // would simply start working for people who cannot start them. - // - // A comment names an issue, not a commit, so look the pull request up: - // the run works on ITS head and the check lands on that commit. Fails - // closed on a fork, like every other untrusted trigger. + // Comment triggers need write access. Keyed on the trigger, so a planner change can't drop + // the check. Fails closed. A comment names an issue, so resolve the pull request's head. let pull if (!plan.context?.sha && plan.context?.pr) { pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) plan = withPullRequestHead(plan, pull, REPO) if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) - // Work on the commit that was asked about. The starter checks out the - // default branch for a comment event, since the payload names no ref. + // The starter checks out the default branch for a comment event. if (!fetchAndCheckout(plan.context.sha)) { warn(`Could not check out ${plan.context.sha}; running against the tree this job checked out.`) } } - // the ticket this change is for, so the run tests against what the - // ticket asks for and not only what the diff does. The pull request object - // is already in hand for a comment event; for a pull_request event it is on - // the payload. + // The ticket this change is for, so the run tests what it asks for. const prForTicket = pull ?? readEvent().pull_request const ticket = findTicket({ title: prForTicket?.title, @@ -621,54 +440,35 @@ async function main() { }) if (ticket && plan.context) plan.context.ticket = ticket - // Read once, before the access check: team narrowing lives in the - // same document as approval gate. + // Read before the access check: ask_team lives in the same file. const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined const askTeam = bindingsDoc?.settings?.ask_team - // Only issue_comment. `pull_request_review_comment` was listed here too, - // but route has no case for it, so such an event skips as - // "unsupported-event" long before this line — and the starter does not - // subscribe to it either. - // Jobs that understand --propose-only and --plan-file. For anything else - // the proposal IS the preview run's report. + // Jobs that understand --propose-only and --plan-file. const CARRIES_A_PLAN = new Set(["test-pr"]) // Jobs that cannot run without knowing which build failed. const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) const COMMENT_TRIGGERS = new Set(["issue_comment"]) if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { const who = plan.commenter ?? readEvent().comment?.user?.login - // write access is the floor, and the pipeline file may narrow it - // to a team. It may never widen it — see mayAsk. + // Write access is the floor; ask_team may narrow it, never widen it. const allowed = mayAsk({ hasWriteAccess: hasWriteAccess(who), askTeam, inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, }) if (!allowed) { - // A refusal wants a one-line reply, not a line in a log the person - // who asked will never open. - // - // Unless they never addressed us. An unaddressed comment reaches - // here only because it was a bare "go", and answering a bystander's - // one-word comment with a public refusal is noise in someone else's - // conversation. + // Don't reply to a bystander's bare "go". if (!plan.approvalOnly) replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) finishSkipped("not-authorized", who) } } - // work out what the comment actually asked for, here, before the CLI - // is spawned — a request we cannot place has to cost nothing. + // Place the request before spawning the CLI, so one we can't place costs nothing. let proposing = false let approvalNote = "" if (plan.event === TC_EVENTS.PR_COMMENT) { - // pass what this repository has actually bound, so a job can be - // asked for by name and not only by the phrases written into PHRASES. - // Fetched BEFORE the intent is read, because what a reply means depends - // on whether we asked a question: with a plan waiting, "go, and fix - // case 3's title" is an approval with an edit, not a request to run - // `fix`. + // Fetched before reading intent: with a plan waiting, "go, and fix case 3" is an approval with an edit. const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) let waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) @@ -682,13 +482,7 @@ async function main() { replyOnPr(plan.context.pr, renderUnknownRequest()) finishSkipped("unmapped-request") } - // `stop` is gone. The word is still answered, because proposals already - // in live threads tell people to say it; it just says what to do - // instead and starts nothing. - // - // Still handled BEFORE the job words, for the reason it always was: - // falling through would be the worst possible reading of "stop", where - // the person asks for the run to end and a new one begins. + // `stop` is retired but still answered, ahead of job words, so it never starts a run. if (intent.kind === INTENT.STOP) { replyOnPr(plan.context.pr, renderStopRemoved()) finishSkipped("stopped") @@ -698,15 +492,7 @@ async function main() { finishSkipped("setup-requested") } - // `go #` names WHICH plan. Resolved here because the intent is - // what carries the number, and before - // anything reads `waiting`, because from this point on it IS the plan - // being approved. - // - // A number that matches nothing gets its own answer rather than the - // generic "nothing is waiting": the person was specific, and telling - // them the thread is empty when it is not would send them looking in - // the wrong place. + // `go #` names which plan. if (intent.kind === INTENT.GO && intent.proposalId) { waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin, { id: intent.proposalId }) if (!waiting) { @@ -720,16 +506,13 @@ async function main() { } } - // A named job narrows what the binding would otherwise choose. The event - // is kept, so the binding still governs result mode and limits — asking - // by comment must not be a way around the repository's own settings. + // A named job narrows the binding; the binding still governs mode and limits. if (intent.workflow) plan.workflow = intent.workflow const bindings = bindingsDoc const binding = pickBinding(bindings, intent.workflow) - // A comment may only ask for a BOUND workflow. Saying so now beats a preview run, - // a proposal, a "go", and only then "no binding for pr_comment". + // A comment may only ask for a bound workflow. if (intent.workflow && bindings && !binding) { replyOnPr( plan.context.pr, @@ -738,20 +521,9 @@ async function main() { finishSkipped("not-bound", intent.workflow) } - // An unaddressed comment only ever reaches us as a possible approval. - // With nothing waiting, it is what it looked like: a person talking to - // each other, not to us. - // - // Unless a plan we put up has been left behind by a newer commit. Then - // they were talking to us, they used a word we accept, and telling them - // "no @browserstack mention" is both wrong and unhelpful — the reason - // is sitting in the comment history. + // An unaddressed comment only reaches here as a possible approval. if (plan.approvalOnly && !waiting) { - // Already approved, or already stopped. Checked BEFORE the - // superseded reading, which otherwise reaches for any older plan - // still in the thread and tells the person their branch moved on - // when it did not, on the very commit the plan had just been - // approved and run against. + // Checked before the superseded case, which would otherwise match an older plan. const used = proposalAlreadyUsed(comments, plan.context.sha, botLogin, plan.workflow) if (used) { replyOnPr( @@ -780,8 +552,7 @@ async function main() { approval: binding?.approval, proposal: waiting, actor: plan.commenter ?? plan.context.actor, - // Named by the comment when it named one, otherwise whatever the - // binding would run — the gate needs the job that will ACTUALLY run. + // The job that will actually run. workflow: plan.workflow ?? binding?.workflow, }) @@ -793,8 +564,7 @@ async function main() { finishSkipped("nothing-to-approve") } if (gate.action === GATE.EXPIRED) { - // dropped after 24 hours. Enforced on the approving side, so a - // stale proposal cannot be approved however long it sits there. + // Proposals expire after 24 hours. replyOnPr( plan.context.pr, "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", @@ -803,10 +573,7 @@ async function main() { } if (gate.approvedProposal) { - // Pin the run to the job that was actually proposed and read. The - // GO intent names no workflow, so without this the CLI resolved one - // from the event alone — whichever binding came first in the file - // and a `fix` proposal could run something else, live, with --yolo. + // Run the job that was proposed, not whatever the event would pick. if (waiting?.workflow) { if (!isApprovableJob(waiting.workflow, enabledJobs)) { replyOnPr( @@ -818,40 +585,25 @@ async function main() { plan.workflow = waiting.workflow } - // Adopt the check the PROPOSAL run left in progress. This run is a - // different process with an empty memo, so without the lookup it - // posts a SECOND check and the proposal's one stays pending beside - // it for the life of the branch. The sweep already does this; - // the approval path did not. + // Adopt the proposal run's in-progress check instead of posting a second one. const pending = inFlightCheckId( gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs, ) if (pending) rememberCheck(plan.context.sha, pending) - // Named with the JOB it finished. Without that, approving `fix` - // leaves a waiting `generate` plan unapprovable until somebody - // pushes. + // Per job, so approving one doesn't spend another job's plan. consumedMarker = proposalConsumedMarkerFor(plan.context.sha, plan.workflow) approvedPlan = waiting?.url ? { id: waiting.id, url: waiting.url } : undefined - // the run carries out the approved plan — the proposal as the - // approver left it, edits and all. Written to a file rather than - // squeezed onto a command line: a case list is prose, and --context - // strips the punctuation it is written with. + // The approved plan, edits included, goes in a file: --context strips punctuation. const replyText = plan.context.instructions const planFile = join(REPORT_DIR, "approved-plan.md") writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile - // And the approving comment's own text stops here. The edits are in - // the plan file now; passing the raw reply as well would put free - // text back on the path to the agent, which is what made the approval - // meaningless in the first place. + // The raw reply stops here; its edits are already in the plan file. delete plan.context.instructions - // Reported, not refused. Both asking and approving need write access, - // so one person doing both is not an escalation — but edits made at - // approval time were reviewed by nobody but the approver, and on a - // self-approved run by nobody at all. A reviewer should see that. + // Self-approval is reported, not refused: nobody else reviewed the plan or its edits. if (gate.selfApproved) { approvalNote = hasApproverEdits(replyText) ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` @@ -859,8 +611,7 @@ async function main() { notice(`Test Companion:${approvalNote}`) } } else { - // too big to cover in one go — say so to the run, which then - // covers what matters and lists what it left out. + // A large change: the run covers what matters and lists what it skipped. const size = describeChangeSize(pull) if (size.large) { plan.context.focus = `${size.files} files and ${size.lines} changed lines` @@ -868,32 +619,12 @@ async function main() { } proposing = gate.action === GATE.PROPOSE - // a planning pass has to KNOW it is one. Without this it attempts - // the work, is refused by preview, and reports "Could not run" after - // spending real credits — observed on the first live run. - // Only `test-pr` has --propose-only and --plan-file; sending either to - // any other job is an unknown-flag exit before the run starts. - // - // The gate still works for them, because a proposal run is a PREVIEW - // run and preview is enforced in tool dispatch: it does the work, - // writes nothing, and its report is what the reviewer approves. + // Only test-pr has --propose-only; other jobs propose through a preview run. if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" if (gate.preview || intent.preview) plan.preview = true - // THIS RUNS LAST, and it has to. - // - // `rca`, `fix` and `verdict` need to know WHICH build failed. A - // build_failed trigger is handed the run; a comment is not. - // - // Last, because the workflow is not final until the approval handling - // above has run: "go" names no job, so the job comes from the proposal - // being approved. Higher up, asking for an rca would work and - // APPROVING it would not — the CLI gets spawned with no build and - // refuses with `rca requires --context build=`. - // - // The newest red run on this commit is what a person means by "the - // failure". Only looked up when the job needs it and the caller has - // not already said. + // Last, because the job isn't final until approval is handled ("go" names none). + // rca, fix and verdict need a failed build: default to the newest red run on this commit. if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) const failed = latestFailedRun(runs?.workflow_runs, { @@ -901,7 +632,6 @@ async function main() { selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, }) if (!failed) { - // Nothing red is an answer, not an error. replyOnPr( plan.context.pr, `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, @@ -913,28 +643,14 @@ async function main() { } } - // no settings file means a clean pass and the file they need. Checked - // before anything that costs money or time — the comment handling above - // spends only API calls, and "set up this repo" has to be reachable from here. + // After comment handling, which costs only API calls and must reach the setup request. if (!settingsFile) { finishWithStarter(configPath) } - // Put the tree on the commit under review, when nothing else already has. - // - // An `issue_comment` event checks out the DEFAULT branch, and a - // `pull_request` event checks out the MERGE commit. Neither is the commit - // the run is about, and the starter workflow cannot be amended once - // customers have merged it, so the Action corrects its own checkout. - // - // No --depth: on a clone made with fetch-depth 0 a shallow fetch makes a - // COMPLETE clone shallow, taking history away from the agent and from - // tc's dedupe claim. - // - // Advisory. A checkout we cannot do is reported and the run continues - // against whatever is here, because a proposal built from the diff is - // still worth having. shouldPushWork re-checks before anything is pushed. + // Comment events check out the default branch and pull_request events the merge commit, so + // move to the commit under review. Advisory: shouldPushWork re-checks before pushing. if ( plan.context.branch && plan.context.sha && @@ -947,19 +663,11 @@ async function main() { const saving = shouldPushWork({ preview: plan.preview, - // From whichever source THIS trigger actually has. A comment names an - // issue, so `pull` was fetched to resolve a head sha; a pull_request - // event carries the head on its own payload and nothing is fetched. - // Reading only `pull` refused every pull_request-triggered push. + // A comment event fetched `pull`; a pull_request event carries the head on its payload. sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), branch: plan.context.branch, headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, - // The tree is put on the pull request's commit much earlier, where the - // comment's issue number is resolved to a head sha. That fetch is - // ADVISORY — it warns and carries on, so the proposal still gets built - // from the diff — which makes this the check that the advisory step - // actually worked. Pushing from a tree that is not the reviewed commit - // would send the default branch's content to the pull request's branch. + // The earlier checkout is advisory, so confirm we are on the reviewed commit before pushing. onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), defaultBranch: defaultBranchName(), }) @@ -967,27 +675,14 @@ async function main() { notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) } - // Lock the agent down before it starts: a separate user, no admin rights, - // no way into this Action's process or the runner's files. See - // sandbox-policy.mjs for what that closes. Set up AFTER the checkout, so - // the tree it is given is the one under review. + // Lock the agent down (see sandbox-policy.mjs). After the checkout, so it gets the reviewed tree. const sandbox = isolateAgent(plan) - // A manual run names its job and carries no event, which is how the CLI is - // told to ignore the bindings. An event-driven run does NOT name one: - // which job an event runs is the pipeline file's decision, so the Action - // passes only --event. Guessing a name here was the first live run's - // failure — `no binding for "pr_opened" (workflow "test")`, because the - // repository had bound `generate`. + // A manual run names its job; an event run passes only --event and the pipeline file chooses. const args = ["workflow", "run"] if (plan.workflow) args.push(plan.workflow) if (plan.event) args.push("--event", plan.event) - // One flag per pair. A single joined value is parsed by pflag with - // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` - // fails the whole flag and the run reports "Could not run". - // The run's files. When the agent is locked down, the ones it reads are - // copies in its `in` folder and the ones it writes land in its `out` - // folder; ours are in the runner's home, which it cannot reach. + // Locked down, the agent reads copies in its `in` folder and writes to its `out` folder. const runFiles = sandbox ? { config: handToAgent(sandbox, settingsFile, "testcompanion-pipeline.yml"), @@ -1003,19 +698,12 @@ async function main() { args.push("--config", runFiles.config) args.push("--json", "--yolo", "--summary-file", runFiles.summary, "--result-file", runFiles.result) - // The "already running" marker. The locked-down agent has nothing to - // write to GitHub with, so the Action plants the marker itself, before - // the run, and tells the CLI it is held. A marker already there means - // this job already ran for this commit. + // The agent can't write to GitHub, so the Action plants the "already running" marker itself. const claim = sandbox ? holdClaim(plan, settingsFile) : { held: false } if (claim.collision) finishSkipped("already-done", claim.workflow) if (claim.passFlag) args.push("--claim-held") - // The handle anything cancelling this run needs — a pull request closed - // without merging is the one left. Posted BEFORE the run, because - // a check written at the end cannot help cancel the run that is writing - // it. It also makes "the check shows while it runs" true for every - // run rather than only a proposal. + // Posted before the run so a closed pull request can find and cancel it. const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) if (checkSubject) { postCheck( @@ -1031,29 +719,20 @@ async function main() { // Where HEAD was before the agent touched anything. headBeforeRun = git(["rev-parse", "HEAD"]).out notice(`tc ${args.join(" ")}`) - // Both files are written by `tc` itself, and the result file only when the - // run generates cases. Remove any stale copy first so a run that produces - // nothing cannot report an earlier run's work as its own. + // Remove stale files so a run that produces nothing can't report an earlier run's work. for (const f of [SUMMARY_FILE, RESULT_FILE]) { try { rmSync(f, { force: true }) } catch { - // A temp file we cannot remove is not worth failing the run over; the - // sha check on the summary is the backstop. + // Not worth failing over. } } - // The GitHub token belongs to the Action's own reporting, never to the agent. - // The run executes shell commands the model chooses; anything in this - // environment is readable by them. Stripped rather than trusted, because the - // product only scrubs these on BrowserStack-hosted runs, not a customer's - // own machine. + // The agent never gets the GitHub token. const run = sandbox ? await runLockedDown(sandbox, args, runFiles) : await runCapturingAnswer("tc", args, childEnvFor(env)) const exitCode = run.status ?? 1 if (sandbox) { - // Stop anything the agent left running before reading its files or - // saving its work, then bring its files over to where the rest of this - // Action (and the upload steps) expect them. + // Stop the agent's processes before reading its files or saving its work. stopSandbox(sandbox) for (const [from, to] of [ [runFiles.summary, SUMMARY_FILE], @@ -1066,8 +745,7 @@ async function main() { if (!plan.preview) handoffNote = openHandedOffPr(runFiles.handoff, plan) } - // The report is absent whenever the process was killed, so treat it as a - // real case rather than assuming it is there. + // Absent if the process was killed. let summaryText try { summaryText = readFileSync(SUMMARY_FILE, "utf8") @@ -1075,33 +753,25 @@ async function main() { summaryText = undefined } const summary = parseSummary(summaryText) - // Read separately from parseSummary, which builds an object: repeated - // `failed_test:` keys would collapse to the last one. + // Read separately: parseSummary would collapse repeated failed_test keys. const failedTests = parseFailedTests(summaryText) const outcome = resolveOutcome(exitCode, summary) - // Nothing was bound to this event. Finish the way every other "nothing to - // do" finishes — no check, no comment, green step — rather than reporting a - // run that never happened. The starter subscribes to every trigger, so for - // most repositories most events land here. + // Nothing bound to this event: finish like any other skip. if (outcome === OUTCOMES.SKIPPED) { finishSkipped("no-binding", summary.event) } - // Absent whenever the run generated nothing, which is most workflows. Its - // presence is what tells the report there is something to point a reader at. const hasResult = existsSync(RESULT_FILE) const readResult = () => { try { return readFileSync(RESULT_FILE, "utf8") } catch { - // Best effort, exactly as the CLI treats writing it: a report we - // cannot read back is not worth failing a finished run over. + // Best effort. return undefined } } - // a configured write that could not happen is said out loud, naming - // what is missing — a silent skip reads as "there was nothing to write". + // A configured write that can't happen is reported. const warnings = [] const testsRepo = bindingsDoc?.settings?.tests_repo if (testsRepo) { @@ -1110,23 +780,16 @@ async function main() { if (!access.ok) warnings.push(access.message) } - // silent unless the settings asked for them. const extraChecks = renderExtraChecks(summary, { wcag: bindingsDoc?.settings?.wcag, visual: bindingsDoc?.settings?.visual, }) - // the approved work must outlive the runner. Until this existed - // nothing carried it off the machine — the CLI opens a pull request only - // for `automate` and `fix`, so an approved `test-pr` run wrote eight - // passing tests and lost all eight. - // automate and fix hand back a pull request to open rather than work to - // save on this branch; when they did, that is what the comment reports. + // automate and fix hand back a pull request; other jobs save to this branch. const workNote = handoffNote ?? saveWork(saving, plan) const view = { workflow: plan.workflow ?? summary.workflow, - // Which plan this run carried out — undefined on a run nobody approved. approvedPlan, workNote, extraChecks, @@ -1134,65 +797,43 @@ async function main() { outcome, summary, runUrl: RUN_URL, - // two runs a week apart on the same pinned version must report the - // same CLI version, which means showing it. + // Shown so runs on the same pinned version are comparable. cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, - // Which request this answers. The comment is edited in place, so on a - // pull request where several people have asked for different things, - // nothing otherwise said whose question the current contents answer. + // Whose request this answers; the comment is edited in place. askedBy: plan.commenter, askedAt: plan.askedAt, - // `rca` and `verdict` leave no artefact — their result file holds the - // agent's closing words, and that IS what was asked for. Without this - // the comment is a table saying "Succeeded" with no answer in it. - // - // The closing words are read from the output stream for EVERY job, so - // `fix` does not post a "Succeeded" table with its diagnosis nowhere - // in sight. The result file stays as the fallback for the two verbs - // the CLI writes it for, so a stream we fail to parse cannot lose - // what worked. - // - // Still gated on ANSWER_JOBS: for `generate` that file holds test - // cases, not prose. + // rca and verdict leave no artefact: the answer is the agent's closing words. The result + // file is the fallback, only for ANSWER_JOBS (for generate it holds test cases). answer: run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), failedTests, } - // a held request posts what it WOULD do and ends. The run that - // produced this was a preview run, so nothing has been written. + // Proposing: post what it would do and stop. Nothing has been written. if (proposing) { const sha = checkSha(plan.context, summary, env.GITHUB_SHA) addToJobSummary(renderJobSummary(view)) - // say when the change is too big to test in one go, so the - // reviewer can narrow the list before approving rather than after. const size = describeChangeSize(pull) const sizeNote = size.large ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` : "" - // The plan itself, which the run writes to the result file. Without it - // the proposal is a report header and nothing to review. + // The plan itself, from the result file. let planBody = "" try { planBody = readFileSync(RESULT_FILE, "utf8") } catch { - // No result file means the run produced no plan; renderProposalBody - // says so rather than rendering an empty proposal. + // No plan; renderProposalBody says so. } replyOnPr( plan.context.pr, renderProposal( - // The same cap, for the same reason: this body is a comment, and - // an over-long one does not truncate — it fails to post, taking - // the whole proposal with it. + // Capped: an over-long comment fails to post. renderProposalBody({ summary: renderCheckSummary(view), plan: planBody, sizeNote }), sha, plan.commenter, - // record WHICH job this proposal is for, so approving it - // cannot run a different one. + // Record the job, so approving can't run a different one. plan.workflow ?? summary.workflow, ), ) - // the check shows in progress while the proposal waits. postCheck(outcome, view, sha, [], "in_progress") setOutput("outcome", "Proposed") setOutput("should-fail", "false") @@ -1205,14 +846,9 @@ async function main() { setOutput("outcome", outcome) setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) - // each failing test on the line it is declared on, so a reviewer sees - // it in the files-changed view instead of having to open the run's logs. - // Empty whenever the reporter printed no locator, which is every mocha run - // those failures are named in the report instead. + // Annotate each failing test on its line, when the reporter gave one. const annotations = buildAnnotations(failedTests) - // accessibility may turn the check red, but ONLY when the settings - // say so and only for violations this change introduced. The step's own - // result is untouched — that follows the build effect. + // Accessibility can turn the check red only when configured, and only for new violations. const checkOutcome = outcome === OUTCOMES.SUCCEEDED && accessibilityFailsCheck(summary, { @@ -1222,11 +858,7 @@ async function main() { ? OUTCOMES.TESTS_FAILED : outcome postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) - // Mark the plan spent only if a run actually happened. A run that broke - // before doing anything used to spend it anyway, which left the commit - // permanently unapprovable. See approvalWasSpent. - // On the PROPOSAL, before the comment: the proposal outlives every later - // run, and the shared comment does not. + // Spend the plan only if a run happened. Marked on the proposal, which later runs don't rewrite. if (approvalWasSpent(outcome)) markPlanSpent(approvedPlan?.id, consumedMarker) if (plan.context.pr) postComment(plan.context.pr, { @@ -1235,30 +867,18 @@ async function main() { consumedMarker: approvalWasSpent(outcome) ? consumedMarker : "", }) - // Always exit 0. Whether the build goes red is the build effect's call, and - // a separate step applies it — otherwise the report never gets attached. + // Always exit 0; a separate step applies the build effect. process.exit(0) } /** - * Commit what the run wrote, onto the pull request's own branch. - * - * A commit on the branch rather than a second pull request: someone who asked - * for tests on their change wants them on their change, not in a review queue - * behind it. - * - * Returns the line for the comment, or undefined when there was nothing to - * save. Never throws and never fails the run — a failed push that takes the - * whole run down loses the report too. + * Commit what the run wrote onto the pull request's branch. + * Returns the line for the comment. Never throws. */ function saveWork(saving, plan) { if (!saving.ok) return undefined const branch = plan.context.branch - // We push HEAD, so a commit the AGENT made rides - // along unfiltered — source, lockfiles, `.github/` — and a workflow-file - // change fails the whole push with an error nobody can read. Our own - // commit happens below, after this check, so a HEAD that moved during the - // run was not us. + // We push HEAD, so refuse if the agent committed: its commit would ride along unfiltered. if (agentCommitted(headBeforeRun, git(["rev-parse", "HEAD"]).out)) { return [ `**Could not save this work to \`${branch}\`:** the run committed to this branch itself,`, @@ -1270,13 +890,8 @@ function saveWork(saving, plan) { return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` } - // .git/config decides whether git runs a program for us and where a push - // ends up, and the agent has just had --yolo write access to it. An - // ordinary actions/checkout never sets any of the keys this looks for, so - // one being present means the config was changed after the checkout. - // - // This runs before `git status`, because status can run a clean filter - // itself — by the time we have a file list it is already too late. + // The agent could have edited .git/config (hooks, filters, URL rewrites). Checked before + // `git status`, which can itself run a clean filter. const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) if (unexpected.length) { return [ @@ -1287,13 +902,7 @@ function saveWork(saving, plan) { "changes are in the job's workspace only.", ].join(" ") } - // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a - // path like café.spec.ts arrives as itself. unquote decodes the escapes - // anyway, but not producing them is the better half of the fix. - // `-uall` so a NEW DIRECTORY arrives as its files. Without it git reports - // `?? tests/`, splitPushable sees one path with one segment, and - // `git add tests/` commits `tests/.env` and `tests/run.log` along with the - // spec. + // quotePath=false keeps non-ASCII paths as-is; -uall lists new directories file by file. const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) if (!status.ok) return undefined const { push, skip } = splitPushable(changedPaths(status.raw), { @@ -1301,16 +910,14 @@ function saveWork(saving, plan) { try { return lstatSync(p).isSymbolicLink() } catch { - // Gone (a deletion) or unreadable: not a link we would be adding. + // Gone or unreadable: not a link. return false } }, }) if (push.length === 0) return undefined - // Named paths, never `git add -A`: the agent ran shell commands of its own - // choosing and the tree may hold anything. Adding only what we listed is - // also what makes the comment's file list true by construction. + // Named paths only, never `git add -A`. const added = git(["add", "--", ...push]) if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) @@ -1331,22 +938,14 @@ function saveWork(saving, plan) { }) } - // Authenticate as the App when it is installed. Without this the push - // uses whatever actions/checkout left in .git/config, so a team that - // deleted the permissions block as we advise got a 403. - // - // The URL is built here from GITHUB_REPOSITORY rather than taken from the - // remote named `origin`. The agent has had write access to this tree, and - // `url..insteadOf` in .git/config silently redirects `origin` to any - // host — which would hand the App token to it. The credential header is - // scoped to this exact URL for the same reason. + // Push to a URL built from GITHUB_REPOSITORY, not `origin`: an insteadOf rewrite in + // .git/config could send the token elsewhere. const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) if (!pushed.ok) { - // The one failure worth spelling out, because it is a settings problem - // rather than a bug: the workflow's token is read-only by default. + // The workflow token is read-only by default. const hint = /403|permission|denied|protected/i.test(pushed.err) ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` : pushed.err || "git push failed" @@ -1355,13 +954,7 @@ function saveWork(saving, plan) { return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) } -/** - * Set up the locked-down user, or decide what to do when we cannot. - * - * Returns the sandbox layout, or undefined when the run goes ahead - * unisolated because the workflow asked for that. Otherwise stops the run: - * running an AI agent that can reach the job's tokens is not a default. - */ +/** Set up the locked-down user. Without it, stop unless allow-unisolated is set. */ function isolateAgent(plan) { const can = canIsolate() let why = can.why @@ -1391,10 +984,7 @@ function isolateAgent(plan) { process.exit(1) } -/** - * What the installed CLI can do, asked once. An older CLI still claims and - * opens pull requests itself, which a locked-down agent cannot. - */ +/** What the installed CLI supports, asked once. */ let cliAbilities function cliSupports() { if (cliAbilities) return cliAbilities @@ -1405,20 +995,15 @@ function cliSupports() { cliAbilities = { claimHeld: /--claim-held/.test(runHelp), prHandoff: /--pr-handoff/.test(runHelp), - // An unknown subcommand prints the parent's help and exits 0, so look - // for a flag only the claim command has. + // An unknown subcommand prints the parent's help and exits 0. claimCommand: /--release/.test(`${claim.stdout ?? ""}${claim.stderr ?? ""}`), } return cliAbilities } /** - * Plant the "already running" marker for this job and commit, as this - * Action, before the agent starts. - * - * Returns { collision, workflow } when the job already ran for this commit; - * otherwise { held, passFlag }: whether we hold a marker to release on - * failure, and whether to tell the CLI the marker is taken care of. + * Plant the "already running" marker before the agent starts. + * Returns { collision, workflow } if the job already ran, else { held, passFlag }. */ function holdClaim(plan, settingsFile) { const can = cliSupports() @@ -1428,7 +1013,7 @@ function holdClaim(plan, settingsFile) { ) return { held: false, passFlag: false } } - // Preview writes nothing, the marker included. No commit, no marker. + // Preview writes nothing, the marker included. if (plan.preview || !plan.context.sha) return { held: false, passFlag: true } const args = [ "workflow", @@ -1464,21 +1049,14 @@ function releaseClaim(claim) { } } -/** - * The node binary the agent's launcher runs with. Ours, unless it lives in - * the runner's home (the runner's own bundled node), which the agent cannot - * reach: then a copy goes in the agent's `in` folder. - */ +/** The node the launcher runs with; copied in when ours is under the runner's home. */ function nodeForAgent(sandbox) { const home = String(env.HOME ?? "") if (!home || !process.execPath.startsWith(`${home}/`)) return process.execPath return handToAgent(sandbox, process.execPath, "node") } -/** - * Run the CLI as the locked-down user, with the read-only GitHub relay up - * for the length of the run. Same result shape as runCapturingAnswer. - */ +/** Run the CLI as the locked-down user, with the read-only relay up. */ async function runLockedDown(sandbox, args, runFiles) { const extra = {} if (cliSupports().prHandoff) extra.TC_WORKFLOW_PR_HANDOFF = runFiles.handoff @@ -1493,9 +1071,7 @@ async function runLockedDown(sandbox, args, runFiles) { return undefined }) try { - // Signed in to BrowserStack as the agent's own user, the same way the - // Sign in step did it for ours. Best effort: the CLI also reads the - // credentials from its environment. + // Best effort; the CLI also reads credentials from its environment. const login = asAgentCommand({ node, launcher: sandbox.launcher, @@ -1521,13 +1097,8 @@ async function runLockedDown(sandbox, args, runFiles) { } /** - * Open the pull request automate or fix handed back, as this Action. - * - * The branch and its commits were made by the agent, so they get the same - * refusals as saving work: an untouched git config, and no `.github/` files - * or symlinks. Then the branch is pushed to a URL we build ourselves and the - * pull request is opened. Returns the line for the comment, or undefined - * when nothing was handed back. + * Open the pull request automate or fix handed back, with the same refusals as saveWork. + * Returns the line for the comment, or undefined. */ function openHandedOffPr(path, plan) { const text = readAgentFile(path, 256 * 1024) @@ -1543,8 +1114,7 @@ function openHandedOffPr(path, plan) { if (!tip.ok || !tip.out) return `**Did not open a pull request:** the run did not leave a \`${req.branch}\` branch.` const from = git(["merge-base", headBeforeRun || "HEAD", tip.out]) if (!from.ok) return `**Did not open a pull request:** \`${req.branch}\` does not share history with this commit.` - // --raw shows each file's new mode, so a symlink (120000) is caught from - // the commit itself rather than from whatever is on disk now. + // --raw gives file modes, so a symlink is caught from the commit itself. const diff = git(["-c", "core.quotePath=false", "diff", "--raw", "--no-renames", "-z", from.out, tip.out]) if (!diff.ok) return `**Could not open the pull request:** ${diff.err || "could not list its changes"}.` const fields = diff.raw.split("\0").filter(Boolean) @@ -1567,11 +1137,7 @@ function openHandedOffPr(path, plan) { ? `**Already done:** \`${req.branch}\` already exists, so this job's pull request was opened by an earlier run.` : `**Could not open the pull request:** ${pushed.err || "git push failed"}.` } - // The base is the repository's real default branch, asked of GitHub. The - // CLI's guess comes from the clone's record of the remote's HEAD, which - // actions/checkout never writes, so on a runner it is always "main", - // wrong for a repository on master or develop. The CLI's value is only - // the fallback when GitHub does not answer. + // actions/checkout doesn't record the remote HEAD, so the CLI's base is only a fallback. const opened = ghRaw(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { head: req.branch, base: defaultBranchName() || req.base, @@ -1587,12 +1153,12 @@ function openHandedOffPr(path, plan) { try { url = JSON.parse(opened.stdout)?.html_url ?? "" } catch { - // Opened; the link is a nicety. + // The link is a nicety. } return `**Opened a pull request** with this run's changes${url ? `: ${url}` : ` from \`${req.branch}\``}.` } -/** one check, never neutral.: each failing test on its own line. */ +/** Create or update this run's check on `sha`. */ function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { if (!sha) return const waiting = status === "in_progress" @@ -1600,19 +1166,12 @@ function postCheck(outcome, view, sha, annotations = [], status = "completed", t name: "Test Companion", head_sha: sha, status, - // Which run this check belongs to. The closed-PR cancel reads it back: the - // check is posted on the PULL REQUEST's head commit, which is the one - // thing that reliably ties a run to the pull request it is about. + // The closed-PR cancel reads this back to find the run. external_id: String(env.GITHUB_RUN_ID ?? ""), details_url: RUN_URL, output: { title: waiting ? (title ?? "Waiting for approval") : outcome, - // CAPPED. GitHub rejects a summary over 65,535 characters with a - // 422, gh turns that into a warning and returns null, and the - // "Running" check posted before the run is then never completed - // it sits in_progress on the head commit for ever, blocking anyone - // who made it required. The comment was capped and this was not - // + // Capped: GitHub rejects a summary over 65,535 characters. summary: renderCheckSummary(view), ...(annotations.length ? { annotations } : {}), }, @@ -1620,11 +1179,7 @@ function postCheck(outcome, view, sha, annotations = [], status = "completed", t // A check that is still running must carry no conclusion; GitHub rejects it. if (!waiting) payload.conclusion = checkConclusion(outcome) - // Update the check this run already made for this commit, rather than - // adding another. A run posts up to three times — in progress, the result, - // and the catch-all on a crash — so posting every time left two or three - // checks of the same name, and a proposal's in-progress one pending - // forever. asks for one. + // Update this run's existing check rather than adding another. const { method, path } = checkRequestFor(sha, checksMade) const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) if (res?.id) checksMade.set(sha, res.id) @@ -1638,25 +1193,12 @@ function rememberCheck(sha, id) { checksMade.set(sha, id) } -/** - * End the runs in flight on this commit. - * - * A cancel call rather than a flag the next run would notice: asking GitHub to - * cancel is the only thing that stops work already underway. The one caller - * left is a pull request closed without merging; `stop` was the other, and it - * was removed. - */ +/** Cancel the runs in flight on a commit. Used when a pull request closes unmerged. */ function cancelRunsOn(sha) { - // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run - // started by an issue_comment carries the DEFAULT BRANCH's sha, so that - // query matched nothing and a closed pull request never cancelled anything - // (verified live, 2026-10-29). + // Our own check runs, because a comment-triggered run carries the default branch's sha. const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) - // Unioned with the old head_sha query, which is still right for a run whose - // head sha genuinely IS this commit — a pull_request-triggered one — and - // whose check may not have been written yet. Wrong only for comment runs, - // which is exactly the gap the checks above close. + // Plus head_sha matches, for pull_request runs whose check isn't posted yet. const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) const fromRuns = chooseRunsToCancel(list?.workflow_runs, { selfRunId: env.GITHUB_RUN_ID, @@ -1674,23 +1216,13 @@ function cancelRunsOn(sha) { return { found: ids.length, cancelled, forbidden } } -/** - * A pull request closed without merging stops its run. - * - * Without the permission to cancel, the run finishes anyway and the comment - * says it ran on a closed pull request — so this warns and lets it be, - * rather than failing anything. - */ +/** A pull request closed without merging stops its run. Without `actions: write`, warn only. */ function cancelForClosedPr(payload) { const sha = payload?.pull_request?.head?.sha if (!sha) return const { cancelled, forbidden } = cancelRunsOn(sha) if (cancelled > 0) { - // Cancelling leaves the run's check in progress, and a check that never - // resolves blocks anyone who made it required. - // `stopRuns` used to complete it and went with `stop`; nothing replaced - // it, so a pull request closed mid-run kept a pending check that came - // back if it was reopened. + // Complete the cancelled run's check so it doesn't stay pending. const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) if (existing) rememberCheck(sha, existing) postCheck( @@ -1707,14 +1239,7 @@ function cancelForClosedPr(payload) { } } -/** - * Which test framework the checked-out repository uses. - * - * Two levels deep and skipping node_modules and dotted directories: deep - * enough for the common `e2e/playwright.config.ts` layout, shallow enough not - * to walk a monorepo. Every read is best-effort — a repository we cannot scan - * gets the generic starter, which is the honest outcome. - */ +/** The repository's test framework. Scans two levels, skipping node_modules and dot folders. */ function scanForFramework() { let deps = {} try { @@ -1744,13 +1269,7 @@ function scanForFramework() { return detectFramework({ filenames, deps }) } -/** - * Open the pull request that adds the settings file. - * - * A pull request rather than a direct commit, deliberately: the settings decide - * what Test Companion is allowed to do, so they go in the way every other - * change to that does — reviewed. - */ +/** Open a pull request adding the settings file, so it is reviewed like any other change. */ function openSetupPr(plan, configPath) { const framework = scanForFramework() const { branch, title, body } = renderSetupPr(configPath, framework) @@ -1761,8 +1280,7 @@ function openSetupPr(plan, configPath) { return } - // An existing branch from an earlier attempt is fine to reuse; the file write - // below is what actually matters. + // A branch left from an earlier attempt is fine to reuse. ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { @@ -1787,15 +1305,7 @@ function openSetupPr(plan, configPath) { ) } -/** - * Drop proposals nobody answered. - * - * This is what makes the in-progress check safe to post at all. A proposal - * never replied to would otherwise leave that check unresolved for the life - * of the pull request, blocking anyone who made "Test Companion" required. - * - * Runs on the nightly schedule, alongside whatever else is bound to it. - */ +/** Drop unanswered proposals so their in-progress checks don't stay pending. Runs on schedule. */ function sweepExpiredProposals() { const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) for (const pr of prs) { @@ -1803,30 +1313,19 @@ function sweepExpiredProposals() { if (!sha || !pr?.number) continue const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) if (!proposalNeedsDropping(comments, sha, Date.now(), botLogin)) continue - // The same question, asked again for the ANSWER rather than the yes/no: - // the dropped marker has to name the job, or dropping one stale plan - // would finish off every other plan on that commit. + // The dropped marker must name the job, or it would end every plan on the commit. const stale = findProposal(comments, sha, Date.now(), botLogin) - // A proposal written before the job was recorded has no workflow, and a - // marker with no workflow means the WHOLE COMMIT under the legacy rule - // in SPENT_MARKER — so sweeping one old plan would kill every current - // one beside it, which is the thing per-job plans set out to stop. Leave it; - // it is already expired, so it cannot be approved either way. + // No recorded job means the marker would cover the whole commit. Already expired, so leave it. if (!stale?.workflow) { notice(`Test Companion: left an untagged stale proposal on #${pr.number} rather than sweeping the whole commit.`) continue } - // Complete the check the PROPOSAL run left in progress, rather than - // adding a second one beside it. The sweep is a different process with - // an empty memo, so without this lookup the proposal's check stayed - // pending for the life of the branch — the "blocked by a check that - // never arrives" failure this sweep exists to prevent. + // Complete the proposal's own check rather than adding a second. const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) if (existing) rememberCheck(sha, existing) - // "Action required", by design — checkConclusion already maps Could not run - // to exactly that, and never to neutral, which branch protection reads as a pass. + // Could not run maps to action_required, never neutral, which branch protection treats as a pass. postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) replyOnPr( pr.number, @@ -1836,19 +1335,9 @@ function sweepExpiredProposals() { } } -/** - * What the pipeline file binds to this event, with every value resolved. - * - * Asks the CLI rather than parsing the YAML here: one parser, one schema, one - * place for it to drift. Returns undefined when the read fails, which - * decideGate treats as "hold for review" — failing closed. - */ +/** The bindings for this event, from the CLI. Undefined on failure, which decideGate treats as hold. */ function readBindings(configFile, event, branch) { - // `--branch` narrows the same way a run does. Without it this returns - // bindings a `when:` clause would exclude, and pickBinding could hand - // decideGate the approval policy of a binding that never runs — reading - // `straight-through` off one while the CLI ran another, which skipped the - // gate in the direction that writes. + // --branch so `when:` clauses apply, as they do for the run. const base = ["workflow", "bindings", "--config", configFile, "--event", event] const ask = (extra) => spawnSync("tc", [...base, ...extra, "--json"], { @@ -1857,12 +1346,7 @@ function readBindings(configFile, event, branch) { }) let r = branch ? ask(["--branch", branch]) : ask([]) - // An older CLI does not know --branch. Falling back is much better than - // losing the settings block entirely: without it there is no wcag, no - // targets, no tests_repo and no ask_team, and the run quietly behaves as - // though the repository had configured none of them. The cost of the - // fallback is only that bindings are not branch-narrowed, which is exactly - // how it behaved before the flag existed. + // Older CLIs lack --branch; read unnarrowed rather than lose the settings block. if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") r = ask([]) @@ -1884,29 +1368,13 @@ function pickBinding(doc, workflow) { return workflow ? list.find((b) => b.workflow === workflow) : list[0] } -/** - * A plain reply in the thread. - * - * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a - * later run neither overwrites a refusal nor adopts one as the place it reports. - */ +/** A plain reply. Carries no COMMENT_MARKER, so later runs never overwrite it. */ function replyOnPr(pr, body) { if (!pr) return gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) } -/** - * Mark the approved plan spent, ON THE PROPOSAL ITSELF. - * - * Put in the shared result comment it would not survive: every later run - * REPLACES that body, so the record of an approval disappears, `findProposal` - * offers the plan as waiting again, and a bare `go` re-runs a `--yolo` job. - * A proposal comment is written once and never rewritten, so the marker - * lasts there. - * - * Best effort: failing to mark it must not take down a run that has already - * done its work. - */ +/** Mark the approved plan spent on the proposal comment, which is never rewritten. Best effort. */ function markPlanSpent(planId, marker) { if (!planId || !marker) return const current = gh(["api", `repos/${REPO}/issues/comments/${planId}`]) @@ -1916,16 +1384,10 @@ function markPlanSpent(planId, marker) { } function postComment(pr, view) { - // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER - // PAGE, so the output of a pull request with more than 30 comments is - // `[...][...]`, JSON.parse throws, gh returns null and `existing` becomes - // []. Every run then posted a NEW comment instead of editing its own, which - // broke the single-comment guarantee. + // --slurp: without it gh prints one array per page and JSON.parse fails. const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) const choice = chooseCommentAction(existing, botLogin) // An approved proposal must stop looking like one. - // Without this a second `go` reran the job, and a day later the sweep told - // the person their approved plan had gone unanswered. const body = { body: renderComment(view) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } if (choice.action === "update") { gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) @@ -1934,14 +1396,7 @@ function postComment(pr, view) { } } -// A crash after the job finishes would leave no tick and no comment, which -// is worse than a red tick: anyone who made the check required is blocked by -// a check that never arrives, with nothing on the pull request saying why. -// -// finishSkipped and the normal path both exit(0) from inside main, so -// reaching this catch means something unexpected threw. Report it as "Could -// not run" and still exit 0, so the build-effect step decides whether the -// build goes red. +// Unexpected throw: report Could not run, and still exit 0 so the build-effect step decides. try { await main() } catch (e) { @@ -1954,10 +1409,7 @@ try { setOutput("outcome", OUTCOMES.COULD_NOT_RUN) setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) try { - // The commit we already posted the in-progress check on, NOT the - // event's sha: on a comment run GITHUB_SHA is the default branch's - // commit, so this used to post a second check on main and leave the - // pull request's one pending. + // The commit we posted the in-progress check on, not GITHUB_SHA. postCheck( OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index 4a9d87e..7db4142 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -1,39 +1,18 @@ /** - * What happened, in five words. - * - * The CLI distinguishes these cases across two channels, and neither is - * sufficient alone: the exit code is a stable contract but exit 4 covers a - * crash, a time ceiling AND a version-pin mismatch, while the report file - * separates them but is absent if the process was killed. - * - * Three separate questions, kept apart deliberately: - * - what happened → resolveOutcome (the five labels) - * - what the check says → checkConclusion - * - whether the step is red → stepShouldFail - * - * The check always reports truthfully. The build effect only governs the step. + * The run's outcome comes from both the exit code and the report file: exit 4 + * covers several causes, and the report is missing if the process was killed. */ - -/** The five outcomes, spelled exactly. */ export const OUTCOMES = Object.freeze({ SUCCEEDED: "Succeeded", TESTS_FAILED: "Tests failed", RUN_BROKE: "Run broke", STOPPED_AT_LIMIT: "Stopped at limit", COULD_NOT_RUN: "Could not run", - // Not one of the five. Those describe a run that HAPPENED; this says none - // did — a draft PR, a fork, an unaddressed comment, no pipeline file, or - // an event nothing is bound to. + // Nothing ran (draft PR, fork, unaddressed comment, nothing bound to the event). SKIPPED: "Skipped", }) -/** - * Parse the CLI's report: one `key: value` per line. - * - * Splits on the FIRST colon only — a value is routinely a URL, and splitting on - * every colon would truncate every build link. Unknown keys are kept rather than - * dropped, so a newer CLI adding a field does not need a change here. - */ +// One `key: value` per line, split on the first colon (values are often URLs). export function parseSummary(text) { const out = {} if (!text) return out @@ -46,21 +25,8 @@ export function parseSummary(text) { return out } -/** - * Which of the five happened. - * - * `summary` is the parsed report, possibly empty — an absent report is a real - * case, not an impossible one: the CLI only writes it when --summary-file was - * passed, and never when the process was killed outright. - */ export function resolveOutcome(exitCode, summary = {}) { - // Nothing was bound to this event, so nothing ran. Checked before the - // report-wins rule below, which would otherwise read "not succeeded" as a - // breakage and redden a commit that was never meant to run anything. - // - // The report then wins over a success exit code: a run with an expired - // BrowserStack session can exit 0 having done nothing at all, which a CI - // step checking only the exit code reads as green. + // The report wins over exit 0: a run with an expired session can exit 0 having done nothing. if (summary.outcome === "skipped") { return OUTCOMES.SKIPPED } @@ -71,39 +37,25 @@ export function resolveOutcome(exitCode, summary = {}) { switch (exitCode) { case 0: return OUTCOMES.SUCCEEDED - // ExitTestFailed: the agent finished cleanly, the tests did not pass. case 5: return OUTCOMES.TESTS_FAILED - // ExitTaskFailed covers three causes; only the report separates them. + // Exit 4 covers several causes; only the report separates them. case 4: return summary.error_type === "ceiling_exceeded" ? OUTCOMES.STOPPED_AT_LIMIT : OUTCOMES.RUN_BROKE - // ExitUsage: a bad pipeline file or bad flags, so nothing ran. case 2: return OUTCOMES.COULD_NOT_RUN - // ExitAuth: not signed in. case 3: return OUTCOMES.COULD_NOT_RUN - // ExitTaskBlocked: the turn finished but the task could not be - // performed — plan, entitlement, or a refusal. case 7: return OUTCOMES.COULD_NOT_RUN - // ExitUnansweredQuestion: headless promised to answer every ask and did - // not, so the run is broken rather than merely unfinished. case 6: return OUTCOMES.RUN_BROKE default: - // Including exit 1. An unrecognised code is never reported as success. return OUTCOMES.RUN_BROKE } } -/** - * What the "Test Companion" check says. - * - * Never `neutral`: branch protection treats a neutral check as a pass, so a - * neutral conclusion would let a run that could not finish satisfy a required - * check. That is the one thing this must not do. - */ +// Never `neutral`: branch protection treats it as a pass. export function checkConclusion(outcome) { switch (outcome) { case OUTCOMES.SUCCEEDED: @@ -111,26 +63,12 @@ export function checkConclusion(outcome) { case OUTCOMES.TESTS_FAILED: return "failure" case OUTCOMES.SKIPPED: - // Distinct from "could not finish", which the default below covers: - // nothing was bound to this event, so there was nothing to run and - // nothing to report. Defensive — main.mjs takes the skip path and - // posts no check at all — but if one is ever posted it must not - // demand a person's attention for a commit nobody asked about. return "success" default: - // Stopped at limit and Could not run both need a person: a re-run, more - // credits, or a fix. Distinct from a test failure, and still not a pass. return "action_required" } } -/** - * One mark per outcome, so the heading reads at a glance in a crowded thread. - * - * Deliberately the ones GitHub itself uses for the same ideas. An outcome a - * newer CLI reports and this Action has never heard of gets no mark rather - * than a crash: losing the mark is cosmetic, losing the report is not. - */ const OUTCOME_MARKS = Object.freeze({ [OUTCOMES.SUCCEEDED]: "✅", [OUTCOMES.TESTS_FAILED]: "❌", @@ -144,15 +82,8 @@ export function emojiFor(outcome) { return OUTCOME_MARKS[outcome] ?? "" } -/** - * A workflow annotation, because that is where somebody looking at a newly-red - * build will be. - */ const warnedBuildEffects = new Set() function warnUnrecognisedBuildEffect(value) { - // Once per distinct value, not once per process: a run only ever sees one - // setting, so this is "once" in practice, and it stays observable when - // several are exercised in the same process. if (warnedBuildEffects.has(value)) return warnedBuildEffects.add(value) const known = Object.values(BUILD_EFFECTS).join(", ") @@ -161,54 +92,26 @@ function warnUnrecognisedBuildEffect(value) { ) } -/** What a repository can ask a bad outcome to do to its build. */ export const BUILD_EFFECTS = Object.freeze({ - /** Never red. The check still reports truthfully. */ REPORT: "report", - /** Red when the RUN broke. Green when the run worked and the tests did not. */ + // Red only when the run itself broke, not on failing tests. FAIL_ON_BREAK: "fail-on-break", - /** Red on anything that is not a success, test failures included. */ FAIL: "fail", }) -/** - * The run itself did not work: it crashed, it never started, or it hit a - * ceiling. Distinct from a test failure, which is a RESULT — the run did its - * job and the answer was no. - */ const RUN_DID_NOT_WORK = new Set([OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT]) /** - * Whether the Action's own step goes red. - * - * Only ever governs the step. The check reports truthfully regardless, so a - * repository choosing `report` sees a red check and a green build by choice. - * - * The default is `fail-on-break`: a run that did not work reddens the step, a - * run that worked and found failing tests does not. Making a test RESULT - * block the build is a larger decision, and `fail` is there for anyone who - * wants it. - * - * Anything unrecognised behaves as the DEFAULT rather than as `report`, so a - * typo cannot buy a quieter setting than a fresh install gets. - * - * `off` is still honoured as a synonym for `report`, undocumented: a workflow - * already saying it must not start failing builds because we stopped - * advertising the word. + * Whether the Action's step goes red. The check is unaffected. Unknown values + * act as the default, not `report`; `off` is a legacy synonym for `report`. */ export function stepShouldFail(buildEffect, outcome) { - // Neither of these is a bad outcome. A skipped run is NO outcome, and - // reddening the step for one would fail every push in a repository that - // binds only pull requests — which is most of them. if (outcome === OUTCOMES.SUCCEEDED || outcome === OUTCOMES.SKIPPED) return false const effect = String(buildEffect ?? "") .trim() .toLowerCase() if (effect === BUILD_EFFECTS.REPORT || effect === "off") return false if (effect === BUILD_EFFECTS.FAIL) return true - // Anything else is the default, which reddens — so say which value we did - // not recognise, or the reddening has no explanation anywhere. `none` and - // `warn` are the likely spellings somebody reaches for. if (effect !== "" && effect !== BUILD_EFFECTS.FAIL_ON_BREAK) { warnUnrecognisedBuildEffect(effect) } @@ -216,16 +119,8 @@ export function stepShouldFail(buildEffect, outcome) { } /** - * Did this run use up the approval it was given? - * - * Replying `go` is spent by the run that carries the plan out, so a second - * `go` cannot quietly run the same work twice. - * - * The question is whether a RUN happened, not whether it went well. Red tests - * and a ceiling both did the work and may have written some of it; - * re-running those would duplicate it and charge again. A run that broke or - * never started did neither, so the person can simply ask again — otherwise a - * `fix` that dies in a second leaves the commit unapprovable for good. + * Whether the `go` approval is used up. Failed tests or a hit limit still did + * work; a run that broke or never started did not, so it can be approved again. */ export function approvalWasSpent(outcome) { return outcome !== OUTCOMES.RUN_BROKE && outcome !== OUTCOMES.COULD_NOT_RUN && outcome !== OUTCOMES.SKIPPED diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index e313938..cfa55d7 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -1,22 +1,10 @@ import { isApprovalOnly } from "./intent.mjs" /** - * Turn a GitHub event into a `tc workflow run` invocation. - * - * The CLI deliberately refuses to read GITHUB_* variables itself, so that the - * same CLI behaves identically on Jenkins. Every piece of context it gets - * comes from here. - * - * Pure — payload in, plan out, no I/O — because the only other way to test it - * is to push a commit and wait for a runner. - * - * ── The event vocabulary is a public contract ─────────────────────────────── - * `--event` is free text to the CLI, matched against whatever the customer - * wrote in their pipeline file's `on:` field. The names below ARE the API: - * renaming one silently stops matching every pipeline that used it, with no - * error — the binding just never fires. Add names; do not rename them. + * Turn a GitHub event into a `tc workflow run` invocation. The CLI reads no + * GITHUB_* variables itself, so all its context comes from here. */ -/** Every event name a pipeline file may bind to. */ +/** Event names pipeline files bind to. Public: add names, never rename them. */ export const TC_EVENTS = Object.freeze({ PR_COMMENT: "pr_comment", PR_OPENED: "pr_opened", @@ -28,39 +16,19 @@ export const TC_EVENTS = Object.freeze({ }) /** - * How a run is addressed in a comment: @browserstack, the name the GitHub App - * posts under, so the name people type matches the name that replies. - * - * Only the word on its own counts. "browserstack" also turns up in text that - * is not addressed to us, and each of these must NOT start a run: - * - * - an email address, someone@browserstack.com (a word character before @); - * - a team mention or npm scope, @browserstack/qa (a slash after); - * - a longer name, @browserstack-team or @browserstack_ci (a word character - * or hyphen after); - * - the App's own bot account, @browserstack[bot]; - * - a domain, @browserstack.com (a dot followed by more of the name). A dot - * that ends a sentence, "thanks @browserstack.", still counts. - * - * Case-insensitive, because people type BrowserStack. + * `@browserstack` as a whole word. Not an email, `@browserstack/team`, + * `@browserstack-x`, `@browserstack[bot]` or `@browserstack.com`; a trailing + * full stop still counts. */ const MENTION = /(^|[^\w.+@-])@browserstack(?![\w\-/[]|\.\w)/i const MENTIONS = new RegExp(MENTION.source, "gi") -/** - * The CLI rejects a context value that begins with `-` (CWE-88), and a pull - * request title or comment body is attacker-controlled on a public repo. Catch - * it here so it is refused before it reaches a command line, not after. - */ +// Refuse a value that could be read as a flag (CWE-88). const looksLikeAFlag = (v) => typeof v === "string" && v.trimStart().startsWith("-") const skip = (reason, detail) => ({ skip: { reason, detail } }) -/** - * Every context key the CLI actually reads. An allowlist rather than a - * denylist: a key it does not know is silently ignored, so passing one is at - * best noise and at worst a way to smuggle something past review. - */ +/** Context keys the CLI reads. Anything else is dropped. */ const ALLOWED_CONTEXT_KEYS = new Set([ "sha", "branch", @@ -77,42 +45,19 @@ const ALLOWED_CONTEXT_KEYS = new Set([ "task", "instructions", "actor", - // the ticket this change is for, found in the branch, title or body. "ticket", - // the approved plan this run must carry out, written by the Action. "plan_file", - // how large the change is, when it is too big to cover in one go. "focus", - // this run is the gate's planning pass — plan, do not write. "propose_only", ]) -/** - * `--context` is a comma-separated `k=v` list, so a value containing `,` or - * `=` can append keys of its own — "@browserstack go,sha=deadbeef" would - * redirect the run at another commit. Comments and titles are - * attacker-controlled on a public repo, so the separators are removed rather - * than escaped: readable prose matters more than its punctuation. - * - * `"` is here because pflag parses a multi-pair --context value as CSV, which - * rejects a bare quote — a PR titled `Fix "Pay now" button` failed flag - * parsing outright. contextArgs below is the real fix; this is the belt. - */ +// A `,` or `=` in user text could inject extra context keys (e.g. `sha=`), +// and pflag's CSV parsing rejects a bare `"`. const stripSeparators = (v) => String(v).replace(/[,="]/g, " ").replace(/\s+/g, " ").trim() /** - * Fill a comment-triggered plan in with the pull request's own head commit. - * - * `issue_comment` carries the issue number and nothing about the code: no - * sha, no branch, and `GITHUB_SHA` is the DEFAULT BRANCH tip for that event. - * Without this the run reads main and its check lands on main's latest - * commit. - * - * Fails CLOSED. A head repository that is missing, or not ours, is treated as - * a fork and skipped — a comment is an untrusted trigger. - * - * `pr` is the API's pull request object; `repo` is "owner/name" for this - * repository. A plan that already names a commit is returned untouched. + * Add the pull request's head commit to a comment-triggered plan; + * `issue_comment` carries none. A missing or foreign head repo is a fork. */ export function withPullRequestHead(plan, pr, repo) { if (plan?.context?.sha) { @@ -130,18 +75,11 @@ export function withPullRequestHead(plan, pr, repo) { return { ...plan, context: { ...plan.context, sha, branch: ref } } } -/** - * The --context flags for one plan, ONE FLAG PER PAIR. - * - * Joining pairs into a single `--context a=1,b=2` sends pflag down its CSV - * path, where one double quote anywhere fails the whole flag. A value with - * exactly one `=` takes the single-pair path instead, which never parses CSV. - */ +/** One `--context` flag per pair, which keeps pflag off its CSV parser. */ export function contextArgs(context = {}) { return Object.entries(context).flatMap(([k, v]) => ["--context", `${k}=${v}`]) } -/** Drop empty values rather than passing `k=` through to the CLI. */ function cleanContext(pairs) { const out = {} for (const [k, v] of Object.entries(pairs)) { @@ -154,19 +92,14 @@ function cleanContext(pairs) { } /** - * Decide what, if anything, this event should run. - * - * Returns either `{skip: {reason}}` — a clean no-op the step reports and passes - * or `{event?, workflow?, preview?, context}`. A manual run carries NO event, - * which is how the CLI is told to ignore the pipeline bindings. + * Decide what this event should run: `{skip}` or + * `{event?, workflow?, preview?, context}`. No event means ignore the bindings. */ export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } = {}) { const built = route({ eventName, payload, inputs, repo, sha, ref }) if (built.skip) return built - // An explicitly named job wins on any trigger, not only a manual one. - // Explicit beats implicit, and naming a job is how the bindings are - // bypassed — same as a manual run, so the event is dropped with it. + // A named job wins on any trigger and bypasses the bindings. const named = (inputs.workflow ?? "").trim() if (named && !built.workflow) { built.workflow = named @@ -174,14 +107,9 @@ export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } delete built.event } - // Clean FIRST, then gate — the order matters and was the wrong way round. - // cleanContext calls stripSeparators, which turns `,`, `=` and `"` into - // spaces and trims. Checking the raw value let `",--config /x"` look safe - // and come out as `--config /x`, while the same text unwrapped was - // correctly refused. The gate has to see what the CLI will see. + // Clean before checking for flags: the check must see what the CLI sees. const context = cleanContext(built.context ?? {}) - // One gate for every path, so a new trigger cannot forget it. for (const [k, v] of Object.entries(context)) { if (looksLikeAFlag(v)) return skip("unsafe-context", `${k} starts with "-"`) } @@ -207,42 +135,23 @@ function route({ eventName, payload, inputs, repo, sha, ref }) { case "workflow_dispatch": return fromManual(inputs, sha, ref) default: - // Better a clean skip than a guess: an unrecognised event with invented - // context would run the wrong job against the wrong commit. return skip("unsupported-event", eventName) } } function fromComment(payload, inputs) { - // an edit must not start a run, or correcting a typo in an old comment - // re-runs the job. if (payload.action !== "created") return skip("comment-edited", payload.action) - // GitHub delivers pull request comments as issue_comment; only the presence - // of `pull_request` distinguishes them from real issues. if (!payload.issue?.pull_request) return skip("not-a-pull-request") const body = payload.comment?.body ?? "" const addressed = MENTION.test(body) - // A bare approval needs no mention: the proposal says "Reply go to run - // this for real", and people do exactly that. - // - // EXACTLY the approval and nothing else. Merely opening with the word is - // too wide — "Approved, but rename the login case" is an ordinary review - // comment, and it would start a live write run with the rest carried in - // as approver edits. Edits alongside an approval are still supported, but - // then the person has to address us. - // - // A plan reference may follow the word and nothing else: `go #` or - // `go `, which is how you say WHICH plan when two are - // waiting. Six digits at least, and it must be the whole rest of the - // comment, so "go and look at #25 too" still needs the mention. + // A bare approval ("go", optionally with a plan reference) needs no mention. + // Anything more must address us, so a review comment never starts a run. const approvalOnly = !addressed && isApprovalOnly(body) if (!addressed && !approvalOnly) return skip("not-addressed-to-us") - // Everything after the mention is what the person actually asked for. The - // mention itself is addressing, not instruction, so it is stripped. - // The character before the mention is part of the match, so put it back. + // Strip the mention; `$1` restores the character the match consumed. const instructions = body.replace(MENTIONS, "$1").replace(/\s+/g, " ").trim() return { @@ -250,17 +159,11 @@ function fromComment(payload, inputs) { context: { pr: String(payload.issue.number), instructions, - // The commenter is who the run is attributed to and whose permission - // the caller checks before this plan is acted on. actor: payload.comment?.user?.login, }, requiresWriteAccess: true, commenter: payload.comment?.user?.login, - // Where the request was made. One comment edited in place cannot - // otherwise say WHICH request it is answering, and on a busy pull - // request that is most of the question. askedAt: payload.comment?.html_url, - // Carried so main.mjs can refuse it when no proposal is waiting. approvalOnly, } } @@ -268,20 +171,8 @@ function fromComment(payload, inputs) { function fromPullRequest(payload, repo) { const p = payload.pull_request ?? {} - // A fork's workflow gets a read-only token and no secrets, so a run would - // fail partway with a confusing permissions error. Refuse up front. - // - // Fails CLOSED, which matters most for `pull_request_target`: that trigger - // runs in the BASE repository's context WITH secrets even for a fork's - // pull request, so this check is the only thing in front of it. - // - // GitHub sets `head.repo` to null when the fork has been DELETED, so a - // missing repo must refuse too — otherwise opening a pull request from a - // fork and then deleting it runs attacker-authored code with our - // credentials and --yolo. - // - // Compared case-insensitively, because GitHub treats repository names - // that way. + // Refuse forks. Fails closed: `pull_request_target` has secrets even for a + // fork, and `head.repo` is null once a fork is deleted. const head = p.head?.repo?.full_name if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { return skip("fork", head ?? "unknown-origin") @@ -292,8 +183,7 @@ function fromPullRequest(payload, repo) { return { event: TC_EVENTS.PR_MERGED, context: { - // The merge commit is what landed; the head sha is a branch that may - // no longer exist. Scoping to head would examine code never merged. + // The merge commit is what landed. sha: p.merge_commit_sha ?? p.head?.sha, branch: p.base?.ref, pr: String(p.number), @@ -301,24 +191,18 @@ function fromPullRequest(payload, repo) { } } - // a draft never starts a run by itself. A comment still can, which is - // why this sits here rather than in the comment path. + // A comment can still start a run on a draft. if (p.draft) return skip("draft") const event = payload.action === "opened" || payload.action === "reopened" ? TC_EVENTS.PR_OPENED : TC_EVENTS.PR_UPDATED if (payload.action !== "opened" && payload.action !== "reopened" && payload.action !== "synchronize") { - // labelled, assigned, review_requested and friends are noise here. return skip("unsupported-event", `pull_request.${payload.action}`) } return { event, - // No `subject` on purpose. generateSubject in the CLI builds the - // commit-scoped prompt — "the change in PR #N, commit SHA … covering only - // what that change touches" — ONLY when no subject is given. Sending the - // title made `generate` write cases for the title rather than the diff, - // when a run should only ever work on the change that triggered it. + // No `subject`: without one the CLI scopes the run to this commit's diff. context: { sha: p.head?.sha, branch: p.head?.ref, pr: String(p.number) }, } } @@ -327,20 +211,13 @@ function fromWorkflowRun(payload, inputs, repo) { const run = payload.workflow_run ?? {} if (payload.action !== "completed") return skip("unsupported-event", `workflow_run.${payload.action}`) - // workflow_run is a PRIVILEGED trigger: it runs in the base repository's - // context with full secrets even when the run it reacts to came from a fork. - // The fork check in the pull_request path does not cover it, and missing - // this is the documented way these workflows get compromised. - // Fails CLOSED: an absent or unmatched head repository is treated as a fork. - // For a trigger that carries full secrets, "cannot confirm it is ours" has - // to mean no. + // workflow_run has full secrets even for a fork's run. Fails closed. const head = run.head_repository?.full_name if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { return skip("fork", head ?? "unknown-origin") } - // never explain our own red run, or a failing Test Companion check - // triggers a run to explain the failing Test Companion check, forever. + // Never react to our own failed run, or it loops. const self = inputs["self-workflow-name"] if (self && run.name === self) return skip("own-run", run.name) @@ -362,8 +239,7 @@ function fromManual(inputs, sha, ref) { const workflow = (inputs.workflow ?? "").trim() if (!workflow) return skip("no-workflow-named", "workflow_dispatch needs a `workflow` input") return { - // Deliberately no event: that is how the CLI is told to ignore the - // pipeline bindings and run exactly what was asked for. + // No event, so the CLI ignores the pipeline bindings. workflow, preview: String(inputs.preview ?? "") === "true", context: { sha, branch: branchFromRef(ref) }, diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 4b6fcad..7539566 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -1,22 +1,11 @@ /** - * Keeping the work. - * - * An approved run writes test code on a runner that is thrown away minutes - * later. The CLI opens a pull request only for the `automate` and `fix` jobs, - * so without this a `test-pr` run writes eight passing tests and loses all - * eight — the very write the approval gate exists to authorise. - * - * It lands as a commit on the pull request's own branch, not a second pull - * request: a reviewer who asked for tests on their change wants them there. - * - * Pure decisions about strings. The git commands live at the call site in - * main.mjs, so every rule below is testable without a repository. + * Saving an approved run's work as a commit on the pull request's own branch. + * Pure decisions; the git commands live in main.mjs. */ -/** Who we commit as. Must match the identity action.yml configures. */ +/** Must match the identity action.yml configures. */ export const BOT_EMAIL = "test-companion[bot]@users.noreply.github.com" -/** Did we write the commit this run is reacting to? */ export function authoredByUs(email) { return ( String(email ?? "") @@ -25,53 +14,28 @@ export function authoredByUs(email) { ) } -/** - * May this run push at all? - * - * Every arm fails closed, and each one is a different way the push could be - * wrong rather than merely unnecessary. - */ +/** May this run push? Every check fails closed. */ export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch } = {}) { - // The proposal pass. The brain refuses every write tool during preview, so - // there should be nothing here to push — but a shell command the agent ran - // is outside that refusal, and pushing its output would break the single - // promise the proposal makes: nothing is written until you say go. + // A shell command the agent ran during preview is outside the brain's write refusal. if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } - // A fork's branch belongs to someone else, and `pull_request_target` hands - // us write credentials even for fork code. Pushing there is the exact shape - // of the attack that trigger is known for. - // Strict `!== true`, not falsy: an unanswerable question is a no. Passing - // a hard-coded true here was a review finding, and the same fail-open shape - // had already appeared once in this Action on `pull_request_target`. + // Never push to a fork's branch with pull_request_target credentials. Unknown counts as no. if (sameRepo !== true) return { ok: false, why: "the change does not come from this repository" } if (!String(branch ?? "").trim()) return { ok: false, why: "there is no branch to push to" } - // The checkout is advisory — a run continues against whatever is there, so - // a proposal still gets built. Pushing from that tree is not advisory: - // it would send the default branch's content to the pull request's branch - // and clobber the change under review. + // Pushing from the wrong tree would overwrite the change under review. if (onWantedCommit !== true) return { ok: false, why: "the checkout is not on the commit being reviewed" } - // The branch name comes from the pull request's own head ref, which its - // author controls. The default branch is the one target where a mistake is - // not undone by closing a pull request, so it is refused by name. + // The head ref is author-controlled; never let it name the default branch. const base = String(defaultBranch ?? "") .trim() .toLowerCase() if (base && String(branch).trim().toLowerCase() === base) { return { ok: false, why: "that is the repository's default branch" } } - // The loop guard. Our push makes a commit; if that commit triggers a run - // that pushes again, the pull request fills with commits until someone - // notices. One generation is enough. + // Loop guard: never push in reaction to our own commit. if (authoredByUs(headAuthorEmail)) return { ok: false, why: "this run is reacting to our own commit" } return { ok: true } } -/** - * The paths in `git status --porcelain` output. - * - * Deletions count: removing a test is a change worth keeping. A rename reports - * the destination, which is where the file actually is now. - */ +/** Paths in `git status --porcelain` output; a rename reports its destination. */ export function changedPaths(porcelain) { return String(porcelain ?? "") .split("\n") @@ -85,48 +49,19 @@ export function changedPaths(porcelain) { .map(unquote) } -/** - * Drop the status code, without assuming it is exactly two characters wide. - * - * Porcelain writes `XYpath`, and X or Y may itself be a space — " M" - * for an unstaged modification. A caller that trims git's whole stdout - * removes that leading space from the FIRST LINE ONLY, leaving - * `Mpath`, and a fixed slice(3) then eats the first letter of the - * path: `tests/pricing.spec.js` becomes `ests/pricing.spec.js` and the file - * is never staged. - */ +// A caller that trims git's stdout drops the first line's leading space +// (" M path" becomes "M path"), so the status is not always two characters. function stripStatus(line) { if (line[2] === " ") return line.slice(3) if (line[1] === " ") return line.slice(2) return line.slice(3) } -/** - * Decode a porcelain path. - * - * Git quotes a path containing a space, a quote, a backslash or — with - * `core.quotePath` on, which is the DEFAULT — any non-ASCII byte, using C - * escapes for the bytes plus the usual two-character ones. - * - * Stripping the quotes without decoding leaves a path that does not exist, - * and `git add` with several pathspecs aborts on the FIRST miss and stages - * nothing — so one accented filename loses every file the run produced. - * - * The call site also passes `-c core.quotePath=false`. This decodes anyway, - * so a repository that forces the setting back on cannot bring the bug back. - * - * Octal escapes are decoded as BYTES and then read as UTF-8 together: one - * character is often several escapes, and decoding them singly gives - * mojibake. - */ +// Git C-quotes some paths. `git add` stages nothing if any pathspec is missing, +// so an undecoded path loses every file. Octal escapes are bytes, decoded as UTF-8 together. function unquote(path) { if (!(path.startsWith('"') && path.endsWith('"') && path.length >= 2)) return path - // Walk BYTES, not characters. Indexing the string by UTF-16 code unit - // split an emoji into its two surrogate halves and Buffer.from turned each - // half into U+FFFD, so "my 🎉 file.spec.ts" came back as a path that does - // not exist. Reachable precisely because `core.quotePath=false` stops the - // octal escaping but still QUOTES a path with a space — so the emoji - // arrives raw inside quotes and lands here. + // Walk bytes, not UTF-16 units, so raw emoji inside quotes survive. const src = Buffer.from(path.slice(1, -1), "utf8") const bytes = [] for (let i = 0; i < src.length; i++) { @@ -135,30 +70,18 @@ function unquote(path) { continue } const next = src[++i] - // Exactly three octal digits, which is what git emits. if (next >= 0x30 && next <= 0x37) { bytes.push(parseInt(String.fromCharCode(next, src[i + 1], src[i + 2]), 8) & 0xff) i += 2 continue } const simple = { n: 10, t: 9, r: 13, b: 8, f: 12, v: 11, a: 7 }[String.fromCharCode(next)] - // `\"` and `\\` land in the else, and so does anything git did not escape. bytes.push(simple === undefined ? next : simple) } return Buffer.from(bytes).toString("utf8") } -/** - * What we refuse to commit, whatever the repository's .gitignore says. - * - * `git add` honours .gitignore already, so this is the second line of defence, - * for a repository that happens not to ignore one of these. A pushed `.env` is - * the worst single outcome in this feature: it publishes a credential into the - * customer's own history, where deleting the file does not take it back. - * - * Matched on whole path SEGMENTS, so `src/env.js` is kept and only a real - * `.env` is dropped. - */ +// Never committed, whatever .gitignore says. Matched per path segment. const NEVER_COMMIT = [ /^\.env($|\.)/i, /^node_modules$/i, @@ -170,16 +93,8 @@ const NEVER_COMMIT = [ ] /** - * Also refused: - * - * - Anything under `.github/`. That folder tells GitHub what to RUN: workflows, - * the repository's own reusable actions, and who must review what. GitHub - * already rejects workflow edits from our tokens, but not the rest, and an - * action file changed on a branch runs with that branch's secrets. - * - Symlinks. A link can point anywhere on the runner, and committing one - * puts a path chosen by the agent into the customer's tree. - * - * `isSymlink` is asked per path, so this stays testable without a disk. + * Also refuses anything under `.github/` (it controls what GitHub runs) and + * symlinks (they can point anywhere on the runner). */ export function splitPushable(paths, { isSymlink = () => false } = {}) { const push = [] @@ -195,15 +110,7 @@ export function splitPushable(paths, { isSymlink = () => false } = {}) { return { push, skip } } -/** - * What the pull request comment says about the push. - * - * A write that did not happen is said out loud, because a silent skip reads - * as "there was nothing to write". - * - * The file list is not decoration: it is the only record of what an - * autonomous agent changed in someone's branch. - */ +/** The pull request note about the push, including when it failed. */ export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { const list = files.map((f) => `- \`${f}\``).join("\n") if (error) { @@ -223,19 +130,7 @@ export function renderPushNote({ branch, sha, files = [], skipped = [], error } const countOf = (files) => (files.length === 1 ? "1 file" : `${files.length} files`) -/** - * Is the working tree sitting on the commit we think it is? - * - * On an `issue_comment` event GitHub checks out the DEFAULT branch. main.mjs - * corrects that, but advisorily — it warns and carries on, so a proposal can - * still be built from the diff when the fetch fails. - * - * This is the check that the advisory step actually worked. Pushing from a - * tree that is not the reviewed commit would send the default branch's - * content to the pull request's branch and clobber the change under review. - * - * Not knowing where we are is never a reason to push. - */ +/** Is the working tree on a different commit than the one under review? */ export function needsCheckout({ headSha, wantSha } = {}) { const want = String(wantSha ?? "") .trim() @@ -247,18 +142,7 @@ export function needsCheckout({ headSha, wantSha } = {}) { return head !== want } -/** - * Is this branch in the repository we are running in? - * - * Tri-state, like every other access answer here: undefined means we could - * not find out, and shouldDoPushWork treats that as a no. - * - * Takes the head repository's full name rather than fetching it, because - * where it comes from depends on the trigger: a comment makes main.mjs fetch - * the pull request, while a pull_request event already carries it on the - * payload. Reading only the fetched object refuses every - * pull_request-triggered push. - */ +/** Tri-state: undefined means unknown, which callers treat as no. */ export function isSameRepo(headRepoFullName, repo) { const head = String(headRepoFullName ?? "").trim() const ours = String(repo ?? "").trim() @@ -267,31 +151,15 @@ export function isSameRepo(headRepoFullName, repo) { } /** - * The git arguments that authenticate a push as the GitHub App. - * - * The App's installation token lives in GH_TOKEN, which only `gh` reads, - * while the push is plain git authenticating with whatever - * `actions/checkout` wrote into .git/config. Without this, a team that - * deleted the `permissions:` block as we advise gets a 403 on every approved - * run. - * - * Passed as configuration rather than baked into the remote URL: a token in - * a URL ends up in the reflog, in error messages and in `git remote -v`. - * - * No token returns nothing, so the checkout's own credential is used — which - * is correct whenever the App is not installed. + * Environment that makes git push with the App's token. Empty without a + * token, so the checkout's own credential is used. */ export function pushCredentialEnv(token, url) { const t = String(token ?? "").trim() const u = String(url ?? "").trim() if (!t || !u) return {} const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") - // Delivered through the environment rather than `-c` on the command line. - // argv is world-readable on the runner — any process can read the token - // out of `ps` — while /proc//environ is readable only by its owner. - // - // SCOPED to the one URL. A global `http.extraheader` attaches to whatever - // the push resolves to, and the repository's own config can change that. + // Via the environment, not argv (visible in `ps`), and scoped to the one URL. return { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: `http.${u}.extraheader`, @@ -299,14 +167,7 @@ export function pushCredentialEnv(token, url) { } } -/** - * Where to push, built from the trusted repository name rather than resolved - * through the repository's own config. - * - * `git push origin` asks .git/config where origin is, and the agent can - * rewrite that. Naming the URL outright means a rewritten remote is simply - * not used. - */ +/** The push URL, built from the repository name so a rewritten `origin` is ignored. */ export function pushUrlFor(repo, server = "https://github.com") { const r = String(repo ?? "").trim() if (!/^[^/\s]+\/[^/\s]+$/.test(r)) return undefined @@ -314,35 +175,18 @@ export function pushUrlFor(repo, server = "https://github.com") { } /** - * The environment our own git commands run in — deliberately almost empty. - * - * These run AFTER the agent, which has had --yolo write access to the working - * tree. An agent that writes `.git/hooks/pre-commit` gets it executed by our - * `git commit`, with whatever environment we hand it — and spawnSync with no - * `env` inherits everything, including the App key, GH_TOKEN and the - * customer's BrowserStack key. On `pull_request_target` the diff under test - * is attacker-authored, so that is one prompt injection from a key leaving - * the runner. - * - * Proxy settings are kept because a push has to reach the network; nothing - * else is. + * A near-empty environment for our git commands. They run after the agent, + * so anything git might execute must not inherit our secrets. */ export function gitEnvFrom(env = {}) { const out = {} for (const k of ["HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { if (env[k] !== undefined) out[k] = env[k] } - // PINNED, not inherited. The agent writes to the runner as the same user we - // do, and the default PATH includes directories that user can write — a - // planted `git` there would be handed the installation token. git shells out - // to its own subcommands, so it still needs a PATH; this is the smallest one - // that works and contains nothing the agent could have written to. + // Pinned: the default PATH includes directories the agent can write. out.PATH = "/usr/bin:/bin" - // Ignore the system and user config too, so the only configuration in play - // is what this process passes with -c. out.GIT_CONFIG_NOSYSTEM = "1" out.GIT_CONFIG_GLOBAL = "/dev/null" - // Never ask a human, and never run a program to ask one. out.GIT_TERMINAL_PROMPT = "0" out.GIT_ASKPASS = "" out.SSH_ASKPASS = "" @@ -351,44 +195,15 @@ export function gitEnvFrom(env = {}) { } /** - * Where the real git is, as an absolute path — or nothing, when we cannot - * find one we are sure of. - * - * `spawnSync("git", ...)` resolves through PATH, and on a GitHub runner - * several PATH directories are writable by the job's own user, which is also - * the user the agent just ran as. A planted `git` would receive the - * credential we pass in the environment. - * - * "Sure of" means it exists and WE cannot write it: a binary this process - * could overwrite is one the agent could have overwritten. - * - * KNOWN LIMIT: the job's user has passwordless sudo on a GitHub-hosted - * runner, so an agent could replace /usr/bin/git itself and pass this check. - * Nothing inside this process can close that — the same access reaches our - * own source, node, and the token in memory. It closes only with - * containment. - * - * Deliberately no fallback to the bare name: falling back would reinstate the - * hole on exactly the machines where it matters, so the caller skips the - * commit instead. + * The first candidate git that exists and that we cannot write, or nothing. + * No fallback to a PATH lookup, where the agent could plant a binary. + * Known limit: with passwordless sudo this cannot be guaranteed. */ export function trustedGitPath({ candidates = ["/usr/bin/git", "/bin/git"], exists, canWrite } = {}) { return candidates.find((p) => exists(p) && !canWrite(p)) } -/** - * Where the real gh is. The same rule as git, for the same reason, and the - * stakes are higher: every gh call we make carries a GitHub token, and the - * App's calls carry a token signed with the App's private key. - * - * The obvious hijack is `$HOME/.local/bin`. The install step puts it on PATH - * so the CLI can be found, it comes before the system folders, and the agent - * can write to it. A `gh` planted there would be the one a bare name finds. - * - * The candidates are where gh is normally installed: apt on Linux, the - * official package and Homebrew on macOS. Same known limit as git (sudo), and - * the same refusal to fall back to the bare name. - */ +/** The same rule for gh, which carries our GitHub tokens. */ export function trustedGhPath({ candidates = ["/usr/bin/gh", "/usr/local/bin/gh", "/opt/homebrew/bin/gh", "/bin/gh"], exists, @@ -398,18 +213,8 @@ export function trustedGhPath({ } /** - * The environment for one of OUR gh calls: the caller's, with the token we - * chose and a settings folder nobody else has touched. - * - * The settings folder matters as much as the binary. The job-wide one is - * shared with the agent on purpose (the CLI's own gh needs it), so the agent - * can write a config.yml there. gh obeys that file, and one of its settings, - * `http_unix_socket`, sends every API request, token included, to whatever - * is listening on a local socket. A fresh empty folder per call means gh - * runs on its defaults. - * - * The update check is off because it is network noise that writes a state - * file, and prompts are off because nobody is there to answer them. + * Environment for one of our gh calls. A fresh config dir, because the shared + * one can hold settings such as `http_unix_socket` that redirect requests. */ export function ghEnvFrom(source, { token, configDir }) { return { @@ -422,24 +227,14 @@ export function ghEnvFrom(source, { token, configDir }) { } /** - * The git config an ordinary `actions/checkout` leaves behind. Anything else - * means the repository's config was changed after the checkout. - * - * An ALLOW list, not a deny list. Config is where git keeps every "run this - * program for me" hook it has — credential helpers, clean filters, textconv, - * ssh commands, pagers — plus `insteadOf`, which rewrites any URL. Listing - * the dangerous keys loses to the next key git adds; listing the expected - * ones refuses anything we have never heard of. - * - * `.gitattributes` is deliberately absent: it only names a filter, and the - * command a filter runs comes from `filter..*` in config. + * Config keys an ordinary `actions/checkout` writes. An allow list, because + * many git config keys can make git run a program. */ const EXPECTED_CONFIG = [ /^core\.(repositoryformatversion|filemode|bare|logallrefupdates|symlinks|ignorecase|precomposeunicode|quotepath|autocrlf|safecrlf|sparsecheckout|sparsecheckoutcone)$/, /^remote\.[^.]+\.(url|fetch|mirror|tagopt|partialclonefilter|promisor)$/, /^branch\.[^.]+\.(remote|merge|rebase)$/, - // Scoped only. A bare `http.extraheader` goes to every host, so it is not - // expected and not allowed. + // Scoped only; a bare `http.extraheader` goes to every host. /^http\..+\.extraheader$/, /^gc\.auto$/, /^user\.(name|email)$/, @@ -448,27 +243,8 @@ const EXPECTED_CONFIG = [ ] /** - * How to read the repository's own config. - * - * `--local` is load-bearing: a bare `git config --list` also includes config - * given on the command line with `-c`, which is where all of GIT_SAFE_ARGS - * lives. Without it the check reads our own hardening back and calls it - * tampering. - * - * Worktree config needs no separate read: turning it on requires - * `extensions.worktreeConfig` in the local file, which is not an expected - * key, so a repository using it is refused by this same check. - */ -/** - * Did the agent commit something itself? - * - * We push `HEAD`, so anything the agent committed rides along — edits to - * source, lockfiles or `.github/`, none of it filtered by splitPushable, and - * a workflow-file change fails the whole push with an unreadable permissions - * error. Our own commit is made after this check, so a HEAD that moved during - * the run was not us. - * - * Fails CLOSED. Not knowing where HEAD started is not permission to push. + * Did HEAD move during the run? We push HEAD, so an agent commit would ride + * along unfiltered. Unknown counts as yes. */ export function agentCommitted(headBefore, headNow) { const before = String(headBefore ?? "").trim() @@ -477,14 +253,10 @@ export function agentCommitted(headBefore, headNow) { return Boolean(now) && before !== now } +// `--local`, so our own `-c` arguments are not read back as tampering. export const CONFIG_READ_ARGS = Object.freeze(["config", "--local", "--list", "--name-only"]) -/** - * The keys in that output that an ordinary - * checkout would not have written — empty when the config is as we left it. - * - * Fails closed by construction: an unknown key is unexpected. - */ +/** Keys in that listing a plain checkout would not have written. */ export function unexpectedConfigKeys(listing) { return String(listing ?? "") .split("\n") @@ -492,25 +264,16 @@ export function unexpectedConfigKeys(listing) { .filter((l) => l && !EXPECTED_CONFIG.some((re) => re.test(l.toLowerCase()))) } -/** - * Arguments that stop git running anything the agent may have planted. - * - * `core.hooksPath` to nowhere, because a hook is code in the repository that - * `git commit` would run for us. fsmonitor off for the same reason — it is a - * configurable command git will start on its own. - */ +/** Arguments that stop git running anything the agent may have planted. */ export const GIT_SAFE_ARGS = Object.freeze([ "-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", - // Reset the credential helper LIST. A helper is a shell command git runs - // when a request comes back 401. "-c", "credential.helper=", "-c", "core.askPass=", - // https only, so an ext: or a local path in a rewritten URL cannot run. "-c", "protocol.allow=never", "-c", @@ -521,8 +284,6 @@ export const GIT_SAFE_ARGS = Object.freeze([ "core.gitProxy=", "-c", "http.sslVerify=true", - // Every other key that is a way for the repository to make git run a - // program on our behalf. "-c", "core.sshCommand=", "-c", @@ -533,7 +294,6 @@ export const GIT_SAFE_ARGS = Object.freeze([ "commit.gpgsign=false", "-c", "gpg.program=false", - // .gitattributes can name a filter command; ignore the file entirely. "-c", "core.attributesFile=/dev/null", ]) diff --git a/test-companion/src/relay.mjs b/test-companion/src/relay.mjs index 9e4f12b..f40d951 100644 --- a/test-companion/src/relay.mjs +++ b/test-companion/src/relay.mjs @@ -1,26 +1,15 @@ /** - * The read-only GitHub relay the agent's `gh` talks to. - * - * The agent's gh is configured with `http_unix_socket` pointing here and a - * placeholder token. gh then sends every request, as plain HTTP, to this - * socket instead of to GitHub. The relay checks each one against - * relayDecision, swaps the placeholder for the real pass on the ones that - * may go through, and forwards them over HTTPS. - * - * So the agent can read what its jobs need (a failed run's logs, a pull - * request's details) and never holds a pass that could write. - * - * Runs inside this Action's process, while the agent's run is going. + * Read-only GitHub relay for the agent's `gh` (via `http_unix_socket`). The + * real token is added here to allowed requests, so the agent never holds one. */ import http from "node:http" import https from "node:https" import { chmodSync, rmSync } from "node:fs" import { relayDecision } from "./sandbox-policy.mjs" -/** The most we read of a request body. GraphQL queries are small. */ const MAX_BODY = 1024 * 1024 -/** Headers we never pass on: they belong to this hop, or would reveal ours. */ +// Hop-by-hop headers, plus the agent's placeholder authorization. const HOP_HEADERS = new Set([ "connection", "keep-alive", @@ -35,13 +24,6 @@ const HOP_HEADERS = new Set([ "content-length", ]) -/** - * Start the relay on `socketPath`. `token` is the pass added to allowed API - * requests. `log` receives one line per refused request, so a run that - * fails because the agent tried a write says so. - * - * Resolves to { close }. - */ export function startRelay({ socketPath, token, log = () => {}, forward = forwardRequest }) { rmSync(socketPath, { force: true }) const server = http.createServer((req, res) => { @@ -81,8 +63,7 @@ export function startRelay({ socketPath, token, log = () => {}, forward = forwar return new Promise((resolve, reject) => { server.once("error", reject) server.listen(socketPath, () => { - // Reachable by the agent's user. The folder around it is what limits - // who can get to it (see sandbox.mjs). + // Access is limited by the enclosing folder's permissions (sandbox.mjs). chmodSync(socketPath, 0o666) resolve({ close: () => @@ -95,7 +76,6 @@ export function startRelay({ socketPath, token, log = () => {}, forward = forwar }) } -/** Send one checked request to GitHub over HTTPS and stream the answer back. */ function forwardRequest({ host, method, path, headers, body }, res) { const [hostname, port] = host.split(":") const upstream = https.request( diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 8e86a03..7764d19 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -1,58 +1,24 @@ -/** - * What the run leaves behind. - * - * Only the rendering and the decisions live here, with no network calls, so the - * awkward parts are testable without a runner: whether a preview run is marked - * as one, and whether a second run adds a second comment or edits the first. - * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is - * preinstalled on every runner and already how this product talks to GitHub. - */ +/** Rendering and decisions for the run's comment, check and job summary. No network calls. */ import { CAPABILITIES, isApprovalOnly } from "./intent.mjs" import { emojiFor } from "./outcome.mjs" -/** - * The outcome with its mark in front — never instead of the words, so a reader - * with images off, or a screen reader, still gets the whole outcome. An - * outcome with no mark reads exactly as it did before. - */ const withMark = (outcome) => { const mark = emojiFor(outcome) return mark ? `${mark} ${outcome}` : String(outcome ?? "") } /** - * How a later run finds the comment it wrote earlier: ONE comment, edited in - * place, not a new one per run. An HTML comment so a reader never sees it. - */ -/** - * Is this comment one WE wrote? - * - * "Any bot" is not good enough. Another App on the repository, or - * `github-actions[bot]` driven by any other workflow, could post a comment - * carrying our proposal marker — and a maintainer's `go` would then feed that - * body to a live `--yolo` run as the approved plan. No write access needed, - * just a workflow, and plenty of repositories run one on a fork's pull - * request. - * - * Fails CLOSED: with no login to compare against, nothing is ours. The worst - * case is proposing again; the alternative is running someone else's text. + * Only comments posted by our own bot login count. Another App or workflow + * could post our markers, and a `go` would then run their text as the plan. */ export const isOurs = (comment, us) => Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us +// Hidden marker so later runs edit one comment instead of posting new ones. export const COMMENT_MARKER = "" const previewRun = (summary) => summary?.preview === "true" -/** - * What the run produced, in the words a reviewer actually wants. - * - * Returns nothing for a run that generates no cases (rca, fix, verdict): a - * row reading "0 generated" on one of those looks like something went wrong. - * - * The case that matters most is cases generated and NONE saved. As a bare - * pair of numbers, "12 generated · 0 saved" reads as success to a skimming - * reader — a run can produce 27 cases, report "Succeeded", and lose them all. - */ +// Flags generated-but-unsaved cases loudly; bare counts read as success. function producedLines(summary, resultArtifact) { const generated = Number(summary.cases_generated ?? 0) const saved = Number(summary.cases_saved ?? 0) @@ -75,43 +41,22 @@ function producedLines(summary, resultArtifact) { return [lines.join(" "), ""] } -/** - * How many failing tests the report names before it stops. - * - * A suite that loses a hundred tests turns the check into a wall of text - * nobody reads. The first few plus a count is more useful than all of them, - * and every one is still annotated on the diff. - */ const FAILURES_LISTED = 20 /** - * Make text that came from outside safe to put in our own comment. - * - * The answer and the failing-test names are the only parts of the comment - * this Action did not write. On a fork's pull request both are - * attacker-influenceable, and even without malice a log line can contain one - * of our markers — which are HTML comments, so one arriving inside borrowed - * text is read back as our own control data. - * - * Only the OPENING delimiter needs breaking; without it `-->` is three - * characters of prose. `<!--` still renders as `` } -/** - * The ONE pattern for a proposal marker. Several readers need it, and when - * there were several patterns they drifted. - * - * Groups: 1 = requester, 2 = workflow. Both optional, so markers written - * before either field existed stay readable. - * - * LITERAL, with the sha captured rather than interpolated. Built per-search - * from the sha, it would have to be regex-escaped or it would quietly match - * OTHER commits — `a.c*d+` also finds `abcccd`, so a plan for one commit - * could be approved against another. Comparing captured text with `===` - * removes the rule instead of restating it. - */ +// Groups: 1 = sha, 2 = requester, 3 = workflow. The sha is captured and compared, +// never built into the regex. const MARKER = // -/** The marker's fields, or undefined when the comment has none. */ -// -// The LAST marker, because ours is always the last line of the comment. The -// plan above it is escaped (renderProposalBody), but if anything ever slips -// through unescaped, the first match would be the planted one, and a planted -// marker can name a different job for `go` to run. +// The last marker, since ours ends the comment and anything earlier could be planted. const readMarker = (body) => { const all = [...String(body ?? "").matchAll(new RegExp(MARKER.source, "g"))] const found = all.at(-1) return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined } -/** - * The commit a check we already posted belongs to. - * - * The "in progress" check goes up BEFORE the CLI runs, on the PULL REQUEST's - * head, so anything finishing the run afterwards has to complete THAT check - * on THAT commit. Reaching for the event's own sha is wrong on a comment run, - * where GITHUB_SHA is the default branch's: the second check lands on main - * and the pull request's stays pending for ever. - */ +// The commit our in-progress check went on (the PR head), which on a comment run is not GITHUB_SHA. export function checkSubjectFor(made, fallbackSha) { for (const sha of made?.keys?.() ?? []) return sha return fallbackSha } -/** - * Whether the job a proposal names may be run by a `go`. - * - * Only a job this repository has enabled. The name came out of a comment, and - * it goes onto the CLI's command line, so anything else is refused: a job the - * settings never bound, or something shaped like an option (`--config=…`). - * Exact match against the settings' own list does both. - */ +// Exact match against enabled jobs; the name reaches the CLI's command line. export function isApprovableJob(workflow, enabledWorkflows) { const job = String(workflow ?? "") return job !== "" && !job.startsWith("-") && (enabledWorkflows ?? []).includes(job) } -/** The proposal comment: what the run would do, and how to let it. */ export function renderProposal(body, sha, requester, workflow) { return [ body, @@ -486,57 +276,27 @@ export function renderProposal(body, sha, requester, workflow) { ].join("\n") } -/** - * Has the plan for this commit already been used, and how? - * - * Distinguishes "used up" from "proposed against an earlier commit": with - * nothing waiting, the caller would otherwise reach for a superseded proposal - * still in the thread and report the wrong reason. - * - * Returns "approved", "dropped", or undefined when the superseded reading is - * the right one after all. - */ +/** "approved", "dropped", or undefined when no plan on this commit was used. */ export function proposalAlreadyUsed(comments, sha, us, workflow) { const want = String(sha ?? "").trim() const job = String(workflow ?? "").trim() - // No job asked about means "any plan on this commit", which is what the - // caller wants when the reply named no job either. const here = spentPlans(comments, us).filter((m) => m.sha === want && (!m.workflow || !job || m.workflow === job)) if (here.some((m) => m.kind === "consumed")) return "approved" if (here.some((m) => m.kind === "dropped")) return "dropped" return undefined } -/** - * The proposal waiting on this commit, if there is one. - * - * Only a comment WE wrote counts, for the same reason chooseCommentAction - * checks: otherwise a person quoting our proposal in their reply could approve - * their own request. - * - * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst - * case is a fresh proposal rather than approving one of unknown age. - */ +// Only our own comments count; an unreadable timestamp counts as expired. export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { - // `by=` and `workflow=` are both optional, so proposals written before - // either was recorded stay approvable rather than becoming stuck. const want = String(sha ?? "").trim() - // Spent, either way round: approved and run, or dropped by `stop` or - // by the sweep. Checked before anything else, so neither a second `go` nor - // the sweep can act on a plan that is already finished with. - // - // PER JOB, not per commit. A plan a - // marker does not name is still waiting, however many OTHER plans on this - // commit have been used up. + // Spent plans are tracked per job, so using one leaves the others on this commit waiting. const spent = spentPlans(comments, us).filter((m) => m.sha === want) if (spent.some((m) => !m.workflow)) return undefined const spentJobs = new Set(spent.map((m) => m.workflow)) let ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) ours = ours.filter((c) => !spentJobs.has(readMarker(c.body)?.workflow ?? "")) - // `go ` names ONE plan. With two waiting there is otherwise nothing in - // what was typed that picks between them, and the newest wins by default - // fine when there is one, a coin toss when there are two. + // `go ` picks one plan when several are waiting. const named = String(opts?.id ?? "").trim() if (named) ours = ours.filter((c) => String(c.id) === named) if (ours.length === 0) return undefined @@ -545,47 +305,21 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { const mark = readMarker(newest.body) return { id: newest.id, - // So the run's own comment can link the plan it carried out. url: newest.html_url, expired: !(age <= PROPOSAL_WINDOW_MS), requester: mark?.requester, - // The job the person actually read and approved. Undefined on a - // proposal written before the marker carried one. workflow: mark?.workflow, - // The run has to READ the approved plan to carry it out, so the - // body travels with the decision rather than being fetched again. body: newest.body ?? "", } } -/** - * The plan an approved run must carry out. - * - * A reply of "go" may carry edits to the case list, and the edits win: - * someone looked at the proposal and changed it before approving. - * Handing the run only the original proposal would quietly discard that. - * - * The proposal's own marker is stripped. It is bookkeeping, and leaving it in - * invites the run to read it as an instruction. - */ -/** - * Did the approver change anything, or just say go? - * - * Worth knowing separately from the plan itself: edits made at approval time - * were reviewed by nobody but the approver, and on a self-approved run by - * nobody at all. That combination is reported rather than refused — see - * decideGate — but it cannot be reported if it is not detected. - */ export function hasApproverEdits(replyText) { const reply = String(replyText ?? "").trim() if (!reply) return false - // A bare approval is not an edit, and neither is one that names WHICH plan - // `go `. Presenting either as an edit would have the run looking for - // an instruction nobody gave, and renderApprovedPlan would hand the agent - // the comment id itself under "these win". return !isApprovalOnly(reply) } +// Edits in the `go` reply win over the proposal. export function renderApprovedPlan(proposalBody, replyText) { const plan = String(proposalBody ?? "") .replace(//g, "") @@ -599,23 +333,7 @@ export function renderApprovedPlan(proposalBody, replyText) { return parts.join("\n") } -/** - * How a dropped proposal is remembered. - * - * The sweep that drops proposals runs nightly. Without a record, it would post - * a fresh "Action required" check and a fresh comment on the same dead proposal - * every night for as long as the pull request stayed open. - */ -/** - * A proposal has been APPROVED and run. - * - * Without this record the proposal stays "waiting" after it has been carried - * out: a second `go` runs the job again, and the sweep later reports the - * approved plan as unanswered. - * - * Separate from the dropped marker because they are different events and the - * reply text differs, but both mean the same thing here — this plan is spent. - */ +// Spent-plan markers, so neither a second `go` nor the nightly sweep acts on a finished plan. export function proposalConsumedMarkerFor(sha, workflow) { return `` } @@ -624,28 +342,12 @@ export function proposalDroppedMarkerFor(sha, workflow) { return `` } -/** - * The job travels with a spent plan too. - * - * Keyed on the commit alone, one approval would finish off every plan on that - * commit: approving `fix` would leave a waiting `generate` plan unapprovable - * until somebody pushed. - */ function jobField(workflow) { const job = String(workflow ?? "").trim() return job ? ` workflow=${job}` : "" } -/** - * Every record in this thread of a plan being finished with, and what for. - * - * ONE pattern matching both kinds, for the same reason MARKER is one. - * - * `workflow=` is absent on markers written before the job was recorded. Those - * mean the whole commit, which is what they meant when written — reading them - * as "no particular job" would let an upgrade re-run work somebody had - * already approved. - */ +// A marker without `workflow=` covers every plan on the commit. const SPENT_MARKER = //g function spentPlans(comments, us) { @@ -659,55 +361,17 @@ function spentPlans(comments, us) { return out } -/** - * Is there a proposal on this commit that has run out of time and not yet - * been dropped? - * - * An in-progress check is only safe if something is guaranteed to end it. A - * proposal nobody answers would otherwise leave the check on "in progress" - * forever, blocking anyone who made it required. - */ +// An expired plan holds an in-progress check that would otherwise never complete. export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { - // findProposal already leaves out anything dropped or approved, per job, so - // an expired plan coming back from it is by definition one nobody has - // finished with. The separate dropped-marker check this used to do was the - // same question asked twice, and the second asking was commit-wide — which - // is exactly what per-job plans had to stop. return Boolean(findProposal(comments, sha, now, us)?.expired) } -/** - * The items out of a `gh api --paginate --slurp` response. - * - * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps - * even when there is only one page. Verified against gh 2.91.0: twelve pages at - * per_page=2, and still `[[...]]` at per_page=100. - * - * Without this, chooseCommentAction is handed an array of arrays, matches - * nothing, returns "create" every time, and every run posts a NEW comment - * the single-comment guarantee in, broken by the fix meant to protect it. - * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it - * does; the shape it produces instead was the part that got missed. - * - * Objects are not arrays, so flat leaves an already-flat list alone. That - * keeps this correct if gh ever changes its mind. - */ +// `gh api --paginate --slurp` returns an array of pages, even for a single page. export function flattenPages(slurped) { return Array.isArray(slurped) ? slurped.flat() : [] } -/** - * The accessibility and visual lines on the check. - * - * Both are silent unless the settings asked for them, and both read their - * numbers from the run's summary file. Those keys — accessibility_new, - * accessibility_total, visual_changes, visual_url — are the contract between - * the run and this report. - * - * A check that was asked for and produced no number says so, rather than - * rendering nothing: an absent line reads as "clean", and claiming a pass for - * a check that never ran is the worst of the three outcomes. - */ +// A requested check with no number says so; a missing line would read as clean. export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { const lines = [] const level = String(wcag ?? "").trim() @@ -737,61 +401,29 @@ export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { return lines } -/** - * May accessibility turn the check red? - * - * Only when the settings say so, and only for violations THIS CHANGE - * introduced. Failing a pull request for the state of a page it never touched - * is how a gate gets switched off for good, and a gate nobody runs protects - * nothing. - * - * A run that reported no number has not failed: absence is not a violation. - */ +// Only violations this change introduced, and only when configured. export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { if (!String(wcag ?? "").trim() || !failOnNew) return false const introduced = Number(summary.accessibility_new) return Number.isFinite(introduced) && introduced > 0 } -/** - * The body of a proposal. - * - * The first live run posted a proposal containing only a report header. The - * run had designed nine test cases and verified each one by probing the - * implementation; none of it reached the comment, because the proposal was - * only ever handed the job summary. - * - * A proposal with no plan in it cannot be reviewed, which makes approving it - * meaningless — so an absent plan says so out loud rather than rendering a - * confident-looking empty proposal. - */ +// A proposal with no plan says so rather than looking approvable. export function renderProposalBody({ summary, plan, sizeNote } = {}) { const parts = [] - // The size warning goes above the list, where it can still change what the - // reviewer approves. if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") - // Escaped, both of them. The plan is written by the agent, so it can carry - // a hidden `` that the person approving - // never sees, naming a different job for their `go` to run. + // The plan is agent-written; escape markers so it cannot redirect a `go`. const safeSummary = inertMarkers(String(summary ?? "")) if (safeSummary.trim()) parts.push(safeSummary, "") const body = inertMarkers(String(plan ?? "").trim()) - // The answer is now caught off the run's own output and rendered in the - // summary above, for every job. For the verbs whose result file holds that - // SAME closing text — `test-pr` proposing is one — appending it here - // printed the whole explanation twice. Compared on - // squashed whitespace, because the two copies travel by different routes - // and need not agree on line breaks. + // Skip the plan when the summary above already contains the same text. const squash = (t) => String(t ?? "") .replace(/\s+/g, " ") .trim() const alreadyAbove = body !== "" && squash(safeSummary).includes(squash(body)) - // Only `test-pr` writes a separate plan file. For every other job the - // proposal run is a PREVIEW run — it does the work and writes nothing - // so its report above IS what is being approved. Saying "there is nothing - // here to approve" told the reader to re-run something that had worked. + // Without a plan file, the preview report above is what is being approved. const noPlan = String(summary ?? "").trim() ? "_The report above is what this run would do. Nothing has been written._" : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" @@ -799,73 +431,29 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { return parts.join("\n") } -/** - * A proposal on this pull request that a newer commit has left behind. - * - * The marker keys a proposal to a commit, so a new commit silently replaces it. - * That is the right safety behaviour: a plan written against code that has - * since changed must never be carried out. But silence is the wrong way to - * say it. - * - * Observed live on 2026-10-29: someone replied "go" after the branch moved and - * was told "No @test-companion mention, so nothing was asked of Test - * Companion." They had given a valid approval word, and the actual reason was - * sitting in the comment history. - * - * Only our own comments count, so a person quoting the marker cannot - * manufacture one. - */ +// A proposal left behind by a newer commit, so a late `go` can be told why it no longer applies. export function findSupersededProposal(comments, sha, us) { const current = String(sha ?? "").trim() - // Same reader as findProposal, so the two cannot drift again. const stale = (comments ?? []).filter((c) => { if (!isOurs(c, us)) return false const found = readMarker(c.body) return Boolean(found) && found.sha !== current }) if (stale.length === 0) return undefined - // Newest wins: with several old plans, the one they most plausibly meant is - // the last one we put up. const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) const found = readMarker(newest.body) return { sha: found?.sha, requester: found?.requester, id: newest.id } } -/** - * Create the "Test Companion" check, or update the one this run already made: - * one check, never neutral. - * - * Every call used to POST. A single run posts up to three times — in progress - * before the work, the result after it, and the catch-all on a crash — so a - * pull request collected two or three checks of the same name, and a proposal - * left ITS in-progress one pending forever because nothing ever completed it. - * That is the "blocked by a check that never arrives" failure the design is - * meant to avoid. - * - * Keyed by commit: a run can legitimately check two (the head early on, and - * whatever the report names later), and those are different checks, not an - * update of one another. - */ +// One check per commit per run: PATCH the one we made, POST otherwise. export function checkRequestFor(sha, made) { const existing = made?.get?.(sha) return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } } -/** - * The check a PREVIOUS run left in progress on this commit, if there is one. - * - * checkRequestFor only knows what THIS process created. The nightly sweep runs - * in a different process with an empty memo, so it posted a fresh "Action - * required" check and left the proposal run's in-progress one pending for the - * life of the branch — the exact "blocked by a check that never arrives" - * failure the sweep exists to prevent. - * - * Only ours, and only one still running: a completed check is already - * answered, and another tool's check is none of our business. - */ +// An in-progress check left by an earlier run, so the sweep completes it instead of adding another. export function inFlightCheckId(checks, checkName = "Test Companion") { const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") if (ours.length === 0) return undefined - // Newest, so a stale one from an older run cannot win. return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id } diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index afa3f1c..e51ed40 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -1,30 +1,11 @@ /** - * Which runs a closed pull request should end. - * - * Pure, and separated from the `gh` calls in main.mjs so it can be tested - * without staging a live run. - * - * Two cases that are silent when they go wrong: - * - * - The run doing the cancelling is ITSELF in flight on that commit. Cancel - * it and the job disappears mid-reply, telling the person nothing. - * - Cancelling means cancelling Test Companion, never the repository's own - * build. A comment must not become a way to stop someone else's CI. + * Which runs to cancel. Never the cancelling run itself, and never the + * repository's own builds. */ -/** A run GitHub has not finished with yet. */ const IN_FLIGHT = new Set(["queued", "in_progress", "waiting", "requested", "pending"]) -/** - * The ids to cancel. - * - * `selfRunId` is GITHUB_RUN_ID, which arrives as a string while the API returns - * a number — compared as strings so the two always meet. - * - * `workflowName` is optional: without it we cannot tell our runs from the - * repository's, so the commit match is all that narrows the list. Pass it - * whenever it is known. - */ +/** Ids are compared as strings: GITHUB_RUN_ID is a string, the API's a number. */ export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = {}) { return (runs ?? []) .filter((r) => r && IN_FLIGHT.has(r.status)) @@ -35,19 +16,8 @@ export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = } /** - * Our in-flight run ids, from the check runs on a commit. - * - * Neither obvious route works. `actions/runs?head_sha=` misses every time, - * because a run started by an issue_comment carries the DEFAULT BRANCH's sha. - * A run's own `pull_requests` field reports the pull request whose head is the - * checked-out branch, which is a different pull request. - * - * The check run is the handle that holds: posted on the pull request's own - * head commit, carrying the run id in `external_id`, and written by nothing - * else under this name. - * - * `selfRunId` is GITHUB_RUN_ID, a string here and a number from the API, so - * both sides are compared as strings. + * Our in-flight run ids, from our check runs' `external_id`. A comment-started + * run carries the default branch's sha, so `actions/runs?head_sha=` misses it. */ export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companion" } = {}) { const ids = new Set() @@ -55,8 +25,6 @@ export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companio if (!c || c.name !== checkName) continue if (!IN_FLIGHT.has(c.status)) continue const id = String(c.external_id ?? "").trim() - // No id means we cannot say WHICH run this check belongs to, and - // cancelling a guess is worse than cancelling nothing. if (!id) continue if (id === String(selfRunId)) continue ids.add(id) @@ -64,23 +32,9 @@ export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companio return [...ids] } -/** Conclusions a person reads as "this build is red". */ const RED = new Set(["failure", "cancelled", "timed_out", "startup_failure"]) -/** - * The build a comment means by "the failure" — the newest red run on this - * commit, or nothing. - * - * `rca`, `fix` and `verdict` all need to know WHICH build failed. A - * build_failed trigger hands us the run; a comment does not, so it is found - * here. - * - * Returns the run's own URL, the shape `--build` and `--run` accept - * (`https://github.com///actions/runs/`). - * - * Two runs are never the answer: ours, because explaining our own failure is - * a mirror rather than an answer, and the one asking, which has not finished. - */ +/** URL of the newest failed run on this commit, excluding our own runs. */ export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { const red = (runs ?? []).filter( (r) => diff --git a/test-companion/src/sandbox-policy.mjs b/test-companion/src/sandbox-policy.mjs index 5a05460..793783e 100644 --- a/test-companion/src/sandbox-policy.mjs +++ b/test-companion/src/sandbox-policy.mjs @@ -1,40 +1,14 @@ /** - * The rules for running the agent as a separate, locked-down user. - * - * Why this exists. The agent runs shell commands it chooses, and anything in - * the pull request, the build logs or the app can try to steer it. On a - * GitHub-hosted runner it used to run as the same user as this Action, and a - * test on a real runner showed what that means: - * - * - any program can read the startup environment of another program run by - * the same user, so the agent could read the token and the App key out of - * this Action's own process, even though we never passed them to it; - * - the job's user has admin rights with no password, and with them the - * agent found secrets in the memory of the runner's own worker process. - * - * Running it as a different user with no admin rights closes both: Linux - * does not let one user read another's processes or private files. - * - * Everything here is a pure function. The commands that act on the machine - * live in sandbox.mjs. + * Pure rules for running the agent as a separate user without sudo. As the + * runner's own user it could read our secrets from /proc//environ, or via + * sudo from the runner worker's memory. */ -/** The locked-down user the agent runs as. */ export const AGENT_USER = "tc-agent" /** - * Where the agent works. Outside every home folder on purpose: the runner's - * home holds the job's temp files, gh's signed-in config and the runner's own - * credentials, so the agent is never given a way into it. - * - * - `work` is the checked-out repository, mounted here so the agent never - * needs to walk through the runner's home to reach it. - * - `in` holds files we hand the agent (the settings, an approved plan). - * Ours; the agent can only read it. - * - `out` holds the files the agent's run writes for us (the report, the - * result, a pull request to open). The agent's; we read it carefully. - * - `relay` holds the socket the agent's `gh` talks to. - * - `tmp` is the agent's own temp folder. + * Outside the runner's home, which holds gh's config and runner credentials. + * `in` is ours (agent reads), `out` is the agent's (we read carefully). */ export function sandboxLayout(root = "/var/tmp/tc-agent", home = `/home/${AGENT_USER}`) { return { @@ -51,15 +25,7 @@ export function sandboxLayout(root = "/var/tmp/tc-agent", home = `/home/${AGENT_ } } -/** - * The environment the agent's run gets: the usual allowlisted one, pointed at - * the agent's own folders instead of ours. - * - * GH_CONFIG_DIR is the important override. Ours holds a real token; the - * agent's holds a placeholder and the relay socket, so its `gh` reads work - * and it never sees a pass. RUNNER_TEMP and TMPDIR are moved because ours are - * inside the runner's home, which the agent cannot reach. - */ +/** GH_CONFIG_DIR points at the agent's relay-only gh config, never ours. */ export function agentEnvFrom(childEnv, layout, extra = {}) { const env = { ...childEnv } env.HOME = layout.home @@ -68,9 +34,7 @@ export function agentEnvFrom(childEnv, layout, extra = {}) { env.GH_CONFIG_DIR = layout.ghConfig env.RUNNER_TEMP = layout.tmp env.TMPDIR = layout.tmp - // The agent's own bin first: that is where its copy of the CLI lives. The - // rest of PATH is kept for the customer's toolchain (setup-node, Java, - // Python and so on install under /opt, which every user can read). + // The rest of PATH is kept for the customer's toolchain. env.PATH = [layout.bin, ...String(childEnv.PATH ?? "/usr/bin:/bin").split(":").filter((p) => p && p !== layout.bin)].join(":") env.GH_NO_UPDATE_NOTIFIER = "1" env.GH_PROMPT_DISABLED = "1" @@ -78,17 +42,8 @@ export function agentEnvFrom(childEnv, layout, extra = {}) { } /** - * The command that runs `command args` as the agent user. - * - * `setpriv` swaps to the agent's user and group and drops every other group, - * so nothing of the runner's identity comes along. `--reset-env` and `env -i` - * start from an empty environment. `env -C` moves into the mounted checkout. - * - * The environment itself is NOT on this command line. sudo writes the command - * line it runs to the machine's log, and on a self-hosted runner that log - * outlives the job, so the BrowserStack key would sit in it. Instead the - * launcher reads the environment from `envFile` (readable only by the agent's - * user), deletes it, and starts the command with it. + * The environment is passed via `envFile`, not this command line, because + * sudo logs its command lines. */ export function asAgentCommand({ node, launcher, envFile, cwd, command, args }) { return { @@ -115,11 +70,6 @@ export function asAgentCommand({ node, launcher, envFile, cwd, command, args }) } } -/** - * The launcher asAgentCommand runs: read the environment, delete its file, - * start the command with it, and pass its exit code back. Output is - * inherited, so the run's live log and answer capture work unchanged. - */ export const LAUNCHER_SOURCE = `import { readFileSync, rmSync } from "node:fs" import { spawn } from "node:child_process" const [envFile, command, ...args] = process.argv.slice(2) @@ -132,24 +82,9 @@ child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0))) ` /** - * Whether the relay passes a request from the agent's `gh` on to GitHub. - * - * Reads only. Everything the agent legitimately needs from GitHub is a read: - * a failed run's logs and artifacts, earlier runs, a pull request's details - * and diff. Every write the run needs is done by this Action itself, after - * the run, with checks the agent cannot skip. - * - * - GET and HEAD to the API are allowed. The pass the relay adds is scoped to - * this one repository, so the agent cannot read anyone else's code through - * it either. - * - POST is allowed only to /graphql, and only for a query. gh reads pull - * requests through GraphQL, and GraphQL sends reads and writes the same way, - * so the body decides. - * - GET to any other host is allowed WITHOUT the pass. Log and artifact - * downloads redirect to storage hosts with their own signed URLs, and - * gh sends the redirect through the relay too. The agent can reach the - * internet anyway, so passing these on gives it nothing new. - * - Everything else is refused. + * Reads only; the Action does every write itself. GraphQL POSTs pass only for + * queries. Other hosts get GET without the token (log and artifact downloads + * redirect to signed storage URLs). */ export function relayDecision({ method, host, path, body }) { const m = String(method ?? "").toUpperCase() @@ -159,8 +94,7 @@ export function relayDecision({ method, host, path, body }) { const p = String(path ?? "") const api = h === "api.github.com" if (!p.startsWith("/")) return { allow: false, why: "a path must start with /" } - // GraphQL only ever goes through the POST check below. A GET to /graphql - // would carry its query in the URL, past that check. + // A GET to /graphql would carry its query in the URL, past the body check. if (api && /^\/graphql(\/|\?|$)/.test(p) && m !== "POST") return { allow: false, why: "GraphQL is only passed on as a checked POST" } if (api && (m === "GET" || m === "HEAD")) return { allow: true, auth: true } if (api && m === "POST" && (p === "/graphql" || p.startsWith("/graphql?"))) { @@ -169,8 +103,6 @@ export function relayDecision({ method, host, path, body }) { : { allow: false, why: "only GraphQL queries are passed on, never mutations" } } if (!api && h && (m === "GET" || m === "HEAD")) { - // Never github.com itself: with a pass that would be the web UI, and - // without one there is nothing there the run needs. if (h === "github.com" || h.endsWith(".github.com")) { return { allow: false, why: `${h} is not passed on` } } @@ -180,31 +112,13 @@ export function relayDecision({ method, host, path, body }) { } /** - * Whether a GraphQL request body only reads. - * - * A GraphQL document is a list of operations. Each starts with `query`, - * `mutation` or `subscription`, or with `{` (shorthand for a query), and may - * also hold `fragment` definitions. - * - * The document is read ONCE, left to right, the way GraphQL itself reads it: - * a `#` starts a comment only outside a string, and a `"` starts a string only - * outside a comment. Removing strings and comments in two separate passes gets - * this wrong: a `"` inside a comment then looks like the start of a string, - * swallows the next line, and hides a mutation written there. That bypass was - * found in review. - * - * Two independent checks then have to agree it is a read: - * - no `mutation` or `subscription` keyword appears anywhere outside strings - * and comments, at any depth; - * - every top-level definition starts with `query`, `fragment` or `{`. - * - * Anything we cannot read cleanly counts as a write. + * Strings and comments are lexed in a single pass: separate passes let a `"` + * inside a `#` comment hide a mutation. Anything unparseable counts as a write. */ export function isReadOnlyGraphql(body) { let doc try { const parsed = JSON.parse(String(body ?? "")) - // A batch of several requests in one body: every one must be a query. if (Array.isArray(parsed)) return parsed.length > 0 && parsed.every((one) => isReadOnlyGraphql(JSON.stringify(one))) doc = parsed?.query } catch { @@ -217,7 +131,6 @@ export function isReadOnlyGraphql(body) { if (tokens.some((t) => t === "mutation" || t === "subscription")) return false let depth = 0 - // Whether the next top-level token starts a new definition. let expectDefinition = true for (const t of tokens) { if (t === "{") { @@ -239,12 +152,7 @@ export function isReadOnlyGraphql(body) { return depth === 0 } -/** - * The names and braces of a GraphQL document, with strings, comments, commas - * and whitespace skipped, read in one pass. Other punctuation is dropped: only - * names and braces decide what kind of document this is. Returns undefined for - * anything malformed, such as a string that never closes. - */ +/** Names and braces only; undefined if malformed (e.g. an unclosed string). */ function graphqlTokens(doc) { const out = [] let i = 0 @@ -257,7 +165,6 @@ function graphqlTokens(doc) { } if (c === '"') { if (doc.startsWith('"""', i)) { - // Block string: ends at the next """ that is not escaped as \""". i += 3 for (;;) { if (i >= n) return undefined @@ -273,7 +180,6 @@ function graphqlTokens(doc) { } continue } - // Plain string: one line only, with backslash escapes. i++ for (;;) { if (i >= n) return undefined @@ -305,14 +211,7 @@ function graphqlTokens(doc) { return out } -/** - * The CLI's request to open a pull request, checked before we act on it. - * - * The agent's run writes this file and the agent can change it, so nothing - * in it is trusted: the branch must be the CLI's own naming for this job and - * commit, the base must be a plain branch name, and the title and body are - * capped. Returns the cleaned request, or { error } saying why it was refused. - */ +/** Validates the agent-written PR request; none of it is trusted. */ export function checkPrHandoff(text, { sha } = {}) { let req try { @@ -335,7 +234,6 @@ export function checkPrHandoff(text, { sha } = {}) { return { workflow, sha: commit, branch, base, title: title || `tc ${workflow}: ${commit}`, body } } -/** A branch name with nothing git or a URL could read as something else. */ export function isPlainBranchName(name) { const n = String(name ?? "") return ( @@ -352,11 +250,7 @@ export function isPlainBranchName(name) { ) } -/** - * What `tc workflow claim` printed, read defensively. The CLI is ours, but - * an older one does not have the command at all, and anything unexpected - * must read as "we do not hold a claim", never as "already done". - */ +/** Anything unexpected (e.g. an older CLI) reads as "no claim held". */ export function readClaimResult(stdout, status) { let parsed try { diff --git a/test-companion/src/sandbox.mjs b/test-companion/src/sandbox.mjs index b8cf7e3..eae47b6 100644 --- a/test-companion/src/sandbox.mjs +++ b/test-companion/src/sandbox.mjs @@ -1,13 +1,7 @@ /** - * Setting up, and taking down, the locked-down user the agent runs as. - * - * The rules and the reasons are in sandbox-policy.mjs. This file is the part - * that acts on the machine, through `sudo`, which a GitHub-hosted runner gives - * the job without a password. A runner without it cannot isolate the agent, - * and canIsolate says so. - * - * Every command here runs BEFORE the agent starts or AFTER it has been - * stopped, and finds its programs by absolute path. + * Sets up and tears down the agent's locked-down user, via passwordless sudo. + * Runs only before the agent starts or after it is stopped. Policy lives in + * sandbox-policy.mjs. */ import { spawnSync } from "node:child_process" import { closeSync, constants, copyFileSync, existsSync, fstatSync, openSync, readSync, writeFileSync } from "node:fs" @@ -21,11 +15,6 @@ function sudo(args, input) { return { ok: r.status === 0, out: (r.stdout ?? "").trim(), err: (r.stderr ?? r.error?.message ?? "").trim() } } -/** - * Whether this machine can run the agent as a separate user, and if not, why. - * Needs Linux, password-free sudo, and the two tools we use (setpriv to - * switch user, setfacl to share the checkout). - */ export function canIsolate() { if (process.platform !== "linux") return { ok: false, why: "this is not a Linux runner" } if (!existsSync(SUDO)) return { ok: false, why: "sudo is not installed" } @@ -37,7 +26,6 @@ export function canIsolate() { return { ok: true } } -/** Write a file as root with the given owner and mode. */ function writeAs(path, content, owner, mode) { const w = sudo(["tee", path], content) if (!w.ok) throw new Error(`could not write ${path}: ${w.err}`) @@ -50,31 +38,19 @@ function must(r, what) { return r } -/** - * Create the agent's user and folders, share the checkout with it, give it - * its own copy of the CLI and a gh that talks only to the relay. - * - * `repoDir` is the checked-out repository. `tcPath` is the CLI we installed. - * Throws with a plain reason if any step fails; the caller decides whether - * to stop or carry on unisolated. - */ export function prepareSandbox({ repoDir, tcPath }) { const layout = sandboxLayout() const me = userInfo().username - // The user. On a fresh hosted runner it never exists; on a reused - // self-hosted one it may, from an earlier job. + // May already exist on a reused self-hosted runner. if (!sudo(["id", "-u", AGENT_USER]).ok) { must(sudo(["useradd", "--create-home", "--shell", "/bin/bash", AGENT_USER]), "creating the agent user") } - // It must have no admin rights. The default sudoers grants a new user - // nothing; this checks that no rule on this machine says otherwise. const rules = sudo(["-l", "-U", AGENT_USER]) if (/may run the following commands/i.test(rules.out)) { throw new Error(`${AGENT_USER} has sudo rights on this machine, so it would not be locked down`) } - // Anything a previous job's agent left running stops now. sudo(["pkill", "-KILL", "-u", AGENT_USER]) sudo(["umount", layout.work]) @@ -86,29 +62,24 @@ export function prepareSandbox({ repoDir, tcPath }) { must(sudo(["chown", `${AGENT_USER}:${AGENT_USER}`, layout.out, layout.tmp]), "chown out") must(sudo(["chmod", "755", layout.out]), "chmod out") must(sudo(["chmod", "700", layout.tmp]), "chmod tmp") - // The relay's folder: ours, and readable only by the agent's group, so - // nothing else on the machine can reach the socket inside it. + // Only the agent's group can reach the relay socket inside. must(sudo(["chown", `${me}:${AGENT_USER}`, layout.relay]), "chown relay") must(sudo(["chmod", "750", layout.relay]), "chmod relay") - // Share the checkout. ACLs give the agent read and write without changing - // who owns the files, and the "default" entries make files either of us - // creates usable by the other. Then the checkout is mounted inside the - // agent's folder, so it never has to walk through our home to reach it. + // ACLs share the checkout without changing ownership; the bind mount means + // the agent never needs access to the runner's home. must( sudo(["setfacl", "-R", "-m", `u:${AGENT_USER}:rwX,d:u:${AGENT_USER}:rwX,u:${me}:rwX,d:u:${me}:rwX,m::rwx,d:m::rwx`, repoDir]), "sharing the checkout", ) must(sudo(["mount", "--bind", repoDir, layout.work]), "mounting the checkout") - // The agent's own copy of the CLI, in its own home. It may change it; it - // only ever runs as the agent, and we never run this copy. + // The agent's own copy of the CLI; we never run this copy ourselves. must(sudo(["mkdir", "-p", layout.bin, layout.ghConfig]), "creating the agent's home folders") must(sudo(["cp", tcPath, `${layout.bin}/tc`]), "copying the CLI") must(sudo(["chmod", "755", `${layout.bin}/tc`]), "chmod tc") - // gh, pointed at the relay. The token is a placeholder: the relay - // replaces it on the requests it allows, and nothing else accepts it. + // The token is a placeholder; the relay swaps in the real one. writeAs(`${layout.ghConfig}/config.yml`, `http_unix_socket: ${layout.socket}\nprompt: disabled\n`, `${AGENT_USER}:${AGENT_USER}`, "600") writeAs( `${layout.ghConfig}/hosts.yml`, @@ -116,9 +87,7 @@ export function prepareSandbox({ repoDir, tcPath }) { `${AGENT_USER}:${AGENT_USER}`, "600", ) - // git refuses a repository owned by another user unless told it is safe. - // And a name to commit under: the CLI commits automate's and fix's work - // on their own branch before handing it back. + // safe.directory: git refuses a repository owned by another user. writeAs( `${layout.home}/.gitconfig`, `[safe]\n\tdirectory = ${layout.work}\n[user]\n\tname = Test Companion\n\temail = test-companion[bot]@users.noreply.github.com\n`, @@ -126,41 +95,30 @@ export function prepareSandbox({ repoDir, tcPath }) { "644", ) must(sudo(["chown", "-R", `${AGENT_USER}:${AGENT_USER}`, `${layout.home}/.local`, `${layout.home}/.config`]), "chown home") - // The launcher asAgentCommand runs. Ours, readable by the agent. writeFileSync(`${layout.in}/launch.mjs`, LAUNCHER_SOURCE, { mode: 0o644 }) return { ...layout, launcher: `${layout.in}/launch.mjs` } } -/** - * Hand the agent its environment in a file only its user can read. The - * launcher deletes it as soon as it has read it. Returns the path. - */ +/** The env goes in a 600 file, not on sudo's (logged) command line. */ export function writeAgentEnv(layout, env, name = "env.json") { const path = `${layout.tmp}/${name}` writeAs(path, JSON.stringify(env), `${AGENT_USER}:${AGENT_USER}`, "600") return path } -/** - * Stop everything the agent started and unmount the checkout. Run after the - * agent's run, BEFORE anything reads its files or commits its work: a - * process it left in the background could otherwise keep changing the tree - * while we save it. - */ +/** Call before reading the agent's output, so no leftover process can still change it. */ export function stopSandbox(layout) { sudo(["pkill", "-KILL", "-u", AGENT_USER]) const u = sudo(["umount", layout.work]) return { unmounted: u.ok } } -/** Copy one of our files into the agent's `in` folder. Returns the new path. */ export function handToAgent(layout, from, name) { const to = `${layout.in}/${name}` copyFileSync(from, to) return to } -/** Write content into the agent's `in` folder. Returns the path. */ export function writeForAgent(layout, name, content) { const to = `${layout.in}/${name}` writeFileSync(to, content) @@ -168,12 +126,8 @@ export function writeForAgent(layout, name, content) { } /** - * Read a file the agent's run wrote, without trusting it. - * - * The agent owns the folder, so the name could be a link to one of our own - * files: read through a link, its contents would land in the pull request - * comment. O_NOFOLLOW refuses a link, and only a regular file of a sane size - * is read. Returns undefined for anything else. + * Reads a file from the agent's folder. O_NOFOLLOW: the agent could plant a + * symlink to one of our files and get its contents into the PR comment. */ export function readAgentFile(path, maxBytes = 2 * 1024 * 1024) { let fd diff --git a/test-companion/src/starter.mjs b/test-companion/src/starter.mjs index da5bc32..62fdc84 100644 --- a/test-companion/src/starter.mjs +++ b/test-companion/src/starter.mjs @@ -1,16 +1,8 @@ /** - * The first run, when nothing is configured yet. - * - * A repository with no testcompanion-pipeline.yml would otherwise get a red - * build naming a file nobody has heard of. Instead it gets a clean pass and - * the file it needs, ready to copy. - * - * Handled here rather than in the CLI: making `tc workflow run` succeed on a - * missing config would swallow a real misconfiguration for every pipeline - * already relying on that error. + * A repository with no pipeline file gets a clean pass and a starter file to + * copy, instead of a red build. */ -/** The pipeline file we hand someone who has none. */ export const STARTER_PIPELINE = `# Test Companion — what runs, and when. # # Every entry below is a JOB BINDING: a workflow, the event that starts it, and @@ -57,13 +49,6 @@ workflows: credits: 500 ` -/** - * The pull request that adds the starter settings file. - * - * The detected framework is recorded when we can tell what it is, and left out - * when we cannot — a starter naming Cypress at a Playwright shop would claim an - * understanding we do not have. - */ export function starterPipelineFor(framework) { const name = String(framework ?? "").trim() if (!name) return STARTER_PIPELINE @@ -86,14 +71,7 @@ export function renderSetupPr(configPath, framework) { } } -/** - * Where the pipeline file lives: the flag, then TC_PIPELINE_CONFIG, then the - * repository root. - * - * Mirrors the CLI's own resolution exactly. Checking a different path than - * the CLI reads would be worse than not checking: the Action would announce - * "nothing is configured" to a repository that is. - */ +/** The pipeline file path. Must match the CLI's own resolution. */ export function pipelineConfigPath(configInput, env = {}) { const flag = (configInput ?? "").trim() if (flag) return flag @@ -102,11 +80,6 @@ export function pipelineConfigPath(configInput, env = {}) { return "testcompanion-pipeline.yml" } -/** - * What the job page says when there is no pipeline file. Deliberately free of - * the words "error" and "failed": nothing went wrong, there is just nothing to - * run yet. - */ export function renderNoPipelineReport(path) { return [ "## Test Companion", @@ -123,17 +96,9 @@ export function renderNoPipelineReport(path) { } /** - * Which branch the settings file is read from: always the default branch. - * - * Read from the branch under test, a pull request could rewrite the rules for - * its own run and be obeyed — switch itself from preview to live, delete its - * spending limit, or point its Test Management writes elsewhere. - * - * Deliberate consequence: a pull request that ADDS the settings file for the - * first time is not picked up until it merges. + * Settings are read from the default branch, so a pull request cannot change + * the rules for its own run. */ export function settingsRef(payload = {}, env = {}) { - // Guessing "main" last is safe: the worst case is "no settings file", which - // is a clean skip, not a run obeying rules the pull request wrote. return payload?.repository?.default_branch || env.GITHUB_DEFAULT_BRANCH || "main" } diff --git a/test-companion/src/ticket.mjs b/test-companion/src/ticket.mjs index 1b5a6d3..928c98c 100644 --- a/test-companion/src/ticket.mjs +++ b/test-companion/src/ticket.mjs @@ -1,33 +1,12 @@ -/** - * The ticket this change is for. - * - * A run tests against what the ticket asks for, not only what the diff does, - * so finding the key comes first. People put it in the branch name far more - * often than anywhere else, but the title is where they correct it. - * - * Pure: three strings in, a key or nothing out. - */ +/** Find the ticket key a change is for, from its title, branch or body. */ /** - * A Jira-style key: a project of two or more letters, a hyphen, a number. - * - * Each lookaround guards a different mistake: - * (?- test-companion-${{ github.event.pull_request.number || github.event.issue.number @@ -71,85 +45,45 @@ concurrency: (github.event.action == 'closed' || startsWith(github.event.comment.body, '@browserstack stop')) && '-control' || '' }} - # Queue rather than cancel: a run already in flight has spent credits, and - # the newer one waits its turn instead of superseding it. Control events - # are in their own group, so this no longer delays them. + # Queue rather than cancel: a run in flight has already spent credits. cancel-in-progress: false jobs: test-companion: runs-on: ubuntu-latest timeout-minutes: 35 - # Don't boot a runner for a conversation we are not part of. - # - # Without this, EVERY comment on every pull request started a job that - # cloned the repository, installed the CLI and signed in to - # BrowserStack, only to decide "not addressed to us" and stop. On a - # busy repository that is most of the runs. - # - # This is a quick, loose first filter, not the real decision. These - # expressions can only test whether the text CONTAINS something, so a - # comment holding an email address such as someone@browserstack.com - # still gets through. The Action then reads the comment properly, sees - # it was not addressed to it, and stops without posting anything on - # the pull request. The cost is one short job in the Actions tab. Do - # not try to tighten this line: anything stricter than "contains" here - # would also drop real requests. - # - # The last two lines are the bare approval: a proposal says "reply - # go", and people reply without a mention. String comparison in these - # expressions is case-insensitive, so `Go` is covered. + # A loose first filter so unrelated comments don't boot a runner; the Action makes + # the real decision. The last two lines catch a bare "go" approval (case-insensitive). if: >- github.event_name != 'issue_comment' || contains(github.event.comment.body, '@browserstack') || startsWith(github.event.comment.body, 'go') || startsWith(github.event.comment.body, 'approve') - # Keep this block, App or not. Without it the workflow token falls back - # to your repository's default, which can be write access to - # everything, and Test Companion still uses that token when the App - # cannot be reached. + # Keep this block even with the App: the workflow token is the fallback. permissions: contents: write # branches and commits for fix / reconcile jobs pull-requests: write # the one comment, and opening a pull request checks: write # the "Test Companion" check actions: write # reading the red run, and cancelling one when a PR closes steps: - # v4.4.0. Pinned to a commit because a tag can be repointed by - # whoever owns it, and this step runs in YOUR job with YOUR token. - # Safe to bump whenever you like — it is your workflow now. + # v4.4.0, pinned to a commit because a tag can be repointed. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 # jobs that look at what changed need history - # The workflow token would otherwise be left in .git/config, - # where any shell command the agent chooses to run can read - # it — and it carries `contents: write`. Test Companion does - # not need it there: it authenticates its own push. + # Keeps the workflow token out of .git/config, where the agent could read it. persist-credentials: false - # A fixed release. Version tags in this repository cover every - # action in it, so there is no floating tag for Test Companion - # alone. Move to a newer release when you choose, or let - # Dependabot propose it. For the strongest pin, replace the tag - # with the release's full commit SHA. + # A fixed release. For the strongest pin, use the release's full commit SHA. # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag - uses: browserstack/github-actions/test-companion@v1.1.0 with: username: ${{ secrets.BROWSERSTACK_USERNAME }} access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} - # PIN THE CLI once you know which version you want. Without - # a pin the CLI upgrades itself the first time it runs on a - # release day: your pipeline spends time on a download, and - # two runs a day apart are not the same software. - # - # Both lines or neither. A version with no checksum is - # REFUSED — "refusing an unverified pinned install" — and the - # job stops before it reports anything. Take both from the - # release you tested with. + # Pin the CLI once you've picked a version. Set both lines or neither: + # a version without a checksum is refused. # cli-version: "1.2.0" # cli-sha256: "" - # With the App installed, uncomment these. Keep the - # permissions block above either way: the workflow token is - # still the fallback when the App cannot be reached. + # With the GitHub App installed, uncomment these. # app-id: ${{ secrets.TEST_COMPANION_APP_ID }} # app-private-key: ${{ secrets.TEST_COMPANION_APP_PRIVATE_KEY }} # Manual runs only; event-driven runs read the pipeline file. diff --git a/test-companion/test/access.test.mjs b/test-companion/test/access.test.mjs index 3a66e0d..007f111 100644 --- a/test-companion/test/access.test.mjs +++ b/test-companion/test/access.test.mjs @@ -11,12 +11,6 @@ describe("mayAsk", () => { assert.equal(mayAsk({ hasWriteAccess: false }), false) }) - /** - * the pipeline file "MAY narrow this to a named team and SHALL NOT - * widen it". Being on the team is never a way IN for someone who cannot - * write to the repository — a run executes an agent with --yolo against the - * customer's own machine and credentials. - */ test("being on the team does not widen past write access", () => { assert.equal(mayAsk({ hasWriteAccess: false, askTeam: "qa-leads", inTeam: true }), false) }) @@ -29,8 +23,6 @@ describe("mayAsk", () => { assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: false }), false) }) - // Fails CLOSED, like hasWriteAccess itself: an unanswerable "are they on the - // team?" — no token for the org, API down, rate limited — is a no. test("an unanswerable membership question is a no", () => { assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: undefined }), false) assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: null }), false) @@ -54,12 +46,6 @@ describe("describeTestRepoAccess", () => { assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: true }).ok, true) }) - /** - * "without either, the run SHALL write nothing there and the summary - * SHALL say which is missing." Naming what is missing is the requirement - * a bare "permission denied" halfway through a run leaves someone guessing - * between the App and a token. - */ test("without access, it says so and names both ways to fix it", () => { const d = describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: false }) assert.equal(d.ok, false) @@ -68,8 +54,6 @@ describe("describeTestRepoAccess", () => { assert.match(d.message, /token/i) }) - // Fails closed: an unanswerable "can we push?" must not start a run that - // discovers the answer halfway through, after spending credits. test("an unanswerable question is a no", () => { assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop" }).ok, false) }) diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs index 53af621..230890f 100644 --- a/test-companion/test/action-yml.test.mjs +++ b/test-companion/test/action-yml.test.mjs @@ -6,32 +6,11 @@ import { dirname, join } from "node:path" const actionYml = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "action.yml"), "utf8") -/** The composite steps, split on the step marker so each carries its own env. */ +// The composite steps, split on the step marker so each carries its own env. const steps = actionYml.split(/\n {8}- name: /).slice(1) const nameOf = (s) => s.split("\n")[0].trim() describe("action.yml — every step that runs tc must pin it", () => { - /** - * Found live on 2026-10-29, the day tc 1.1.9 shipped. - * - * The pin was set on the Run step only. But `Sign in` runs `tc auth login` - * FIRST, unpinned, so the CLI self-upgraded there — and the Run step then - * correctly reported a mismatch against the very binary the pin existed to - * preserve: - * - * tc is pinned to version 1.1.8 but the installed binary is 1.1.9 - * - * So asking for a pinned version MADE THE RUN FAIL, which is the exact - * opposite of what promises. The spec lists as "Done, but NEVER - * EXECUTED"; this is what was hiding in it. - * - * `tc --version` is exempt: upgrade.shouldSkip returns true for it, because - * swapping the binary before reporting its version would defeat the point - * of asking. - */ - // Directly (`tc auth login`) or indirectly — the Run step shells out to - // main.mjs, which spawns tc for the workflow run and for `workflow - // bindings`. Both need the pin; only one of them says "tc" in the YAML. const runsTc = (body) => { const withoutVersionChecks = body.replace(/^.*--version.*$/gm, "") return /(^|[\s"'`/])tc\s/m.test(withoutVersionChecks) || /main\.mjs/.test(withoutVersionChecks) @@ -55,18 +34,7 @@ describe("action.yml — every step that runs tc must pin it", () => { }) }) -/** - * A live run's dedupe claim is a real `git push` to origin, and the workflow - * sets `persist-credentials: false`, so git has no credential for it. Pointing - * git at gh's stored credential fixes that without putting a token in any - * environment variable. - * - * The teardown is the part worth pinning. The helper goes in the GLOBAL git - * config, which outlives the job on a self-hosted runner, and it points at a - * config dir the cleanup deletes. Left behind, the next job on that runner - * gets a helper that answers nothing and a git command that stops to ask for - * a password instead of failing. - */ +// The helper goes in the global git config, which outlives the job on a self-hosted runner. describe("action.yml — git's credential helper is cleaned up", () => { test("git is pointed at gh's credential", () => { assert.match(actionYml, /gh auth setup-git/, "a live run's dedupe claim has no credential to push with") @@ -78,10 +46,7 @@ describe("action.yml — git's credential helper is cleaned up", () => { test("whatever sets the helper also unsets it", () => { if (!/gh auth setup-git/.test(actionYml)) return - // `--unset` WITH A VALUE PATTERN, not `--unset-all`. The blanket form - // removed every helper for that URL, including ones the runner's owner - // set up before us, and ran even when our own setup-git had failed - // + // Not `--unset-all`: that also removes helpers the runner's owner set up. assert.match( actionYml, /--unset "credential:\/\/\$host\.helper"|--unset "credential\.https:\/\/\$host\.helper"/, @@ -94,8 +59,6 @@ describe("action.yml — git's credential helper is cleaned up", () => { assert.match(teardown.slice(0, 200), /if: always\(\)/) }) - // Set up after the login succeeds, not before: there is nothing stored to - // hand git until then. test("the helper is set up only on the successful-login path", () => { const login = actionYml.indexOf("gh auth login --with-token") const setup = actionYml.indexOf("gh auth setup-git") @@ -103,10 +66,6 @@ describe("action.yml — git's credential helper is cleaned up", () => { }) }) -/** - * The cleanup must not damage a - * self-hosted runner's own git configuration. - */ describe("action.yml — the cleanup only removes what this job added", () => { test("it never uses the blanket --unset-all on a credential helper", () => { assert.doesNotMatch( @@ -121,12 +80,7 @@ describe("action.yml — the cleanup only removes what this job added", () => { }) }) -/** - * `gh auth setup-git` writes TWO values per host: an - * empty helper that clears any general helper inherited from a broader scope, - * and then its own. Removing only the second leaves the runner's own helper - * disabled for that host. - */ +// `gh auth setup-git` writes two values per host: an empty helper, then its own. describe("action.yml — the cleanup removes both of setup-git's entries", () => { test("the empty helper is unset too", () => { assert.match(actionYml, /--unset "credential[^"]*\.helper" '\^\$'/, "an empty helper keeps a general one disabled") diff --git a/test-companion/test/annotations.test.mjs b/test-companion/test/annotations.test.mjs index be4477f..5cfdb9f 100644 --- a/test-companion/test/annotations.test.mjs +++ b/test-companion/test/annotations.test.mjs @@ -11,8 +11,6 @@ const REPORT = [ ].join("\n") describe("parseFailedTests — reading the report back", () => { - // parseSummary builds an object, so repeated keys collapse and only the - // last failing test would survive. These need their own reader. test("keeps every failing test, not just the last one", () => { assert.equal(parseFailedTests(REPORT).length, 3) }) @@ -35,8 +33,6 @@ describe("parseFailedTests — reading the report back", () => { assert.deepEqual(parseFailedTests(undefined), []) }) - // A name is whatever the reporter printed. It must never be read as a - // location, which is why the writer separates them with a tab. test("a colon in the test name is not a location", () => { const [t] = parseFailedTests("failed_test: handles a:b\treal.spec.ts:4") assert.deepEqual(t, { name: "handles a:b", file: "real.spec.ts", line: 4 }) @@ -53,23 +49,18 @@ describe("buildAnnotations", () => { assert.match(a.message, /Checkout > pays/) }) - // GitHub rejects an annotation with no line. A file with no line is still - // worth pointing at, so it goes on line 1 — and the message says so, rather - // than silently implying the failure is on the first line. + // GitHub rejects an annotation with no line. test("falls back to the top of the file and says so", () => { const [a] = buildAnnotations([{ name: "Checkout > refunds", file: "tests/checkout.spec.ts" }]) assert.equal(a.start_line, 1) assert.match(a.message, /line .*(not|unknown)|which line/i) }) - // Nowhere to point means no annotation. Inventing a path would put a - // failure on a file that has nothing to do with it. test("skips a test with no file, rather than guessing one", () => { assert.deepEqual(buildAnnotations([{ name: "should log in" }]), []) }) - // GitHub accepts at most 50 annotations per request. A suite with 300 - // failures must not silently lose the whole set to a rejected call. + // GitHub accepts at most 50 annotations per request. test("never sends more than GitHub accepts", () => { const many = Array.from({ length: 300 }, (_, i) => ({ name: `t${i}`, file: "a.spec.ts", line: i + 1 })) assert.equal(buildAnnotations(many).length, MAX_ANNOTATIONS) @@ -81,8 +72,6 @@ describe("buildAnnotations", () => { assert.deepEqual(buildAnnotations(undefined), []) }) - // An absolute path, or one climbing out of the repository, annotates - // nothing on GitHub and may point at a file outside the checkout. test("refuses a path that is not inside the repository", () => { assert.deepEqual(buildAnnotations([{ name: "t", file: "/etc/passwd", line: 1 }]), []) assert.deepEqual(buildAnnotations([{ name: "t", file: "../outside.spec.ts", line: 1 }]), []) diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs index 50c34b5..44a46e8 100644 --- a/test-companion/test/answer.test.mjs +++ b/test-companion/test/answer.test.mjs @@ -2,7 +2,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE } from "../src/answer.mjs" -/** One `answer` event as the CLI prints it under --json. */ +// One `answer` event as the CLI prints it under --json. const said = (text) => JSON.stringify({ event: "answer", text }) + "\n" describe("createAnswerCapture", () => { @@ -37,9 +37,6 @@ describe("createAnswerCapture", () => { assert.equal(cap.answer(), "split down the middle") }) - // Agent prose is full of em dashes and arrows. Decoding each chunk on its - // own splits one of those in half and the JSON no longer parses, so the - // answer silently disappears on exactly the runs that have most to say. test("a multi-byte character split across two chunks is not corrupted", () => { const buf = Buffer.from(said("preview — nothing was written"), "utf8") const cut = buf.indexOf(Buffer.from("—", "utf8")) + 1 @@ -102,7 +99,6 @@ describe("createAnswerCapture", () => { }) // A command the agent chose can print megabytes with no newline in it. - // Buffering that forever would take the runner down with us. test("a runaway line with no newline does not grow without bound", () => { const cap = createAnswerCapture() for (let i = 0; i < 40; i++) cap.push("x".repeat(100_000)) @@ -114,15 +110,9 @@ describe("createAnswerCapture", () => { }) }) -/** - * The capture is only worth anything wired to a real process: chunk - * boundaries, encoding and the close/drain race are exactly what unit-testing - * a parser on strings cannot show. main.mjs is the untested I/O shell by - * design (see imports.test.mjs), so the spawn lives here where it can be run. - */ describe("runCapturingAnswer", () => { const node = process.execPath - /** A stand-in for `tc`: prints the lines it is given, then exits. */ + // A stand-in for `tc`: prints the lines it is given, then exits. const fake = (script) => [node, ["-e", script]] test("it returns the child's exit status", async () => { @@ -151,14 +141,8 @@ describe("runCapturingAnswer", () => { assert.equal(r.answer, "tests are still red") }) - // A long answer arrives in many pipe-sized pieces, split anywhere - // including through a multi-byte character. test("an answer larger than the pipe buffer survives being split up", async () => { - // The child BUILDS the payload; it is not passed in as an argument. - // Linux caps a single argv entry at 128KB (MAX_ARG_STRLEN) however - // large ARG_MAX is, so embedding 180KB of text here spawned fine on - // macOS and failed with E2BIG on every Linux runner — which is where - // CI runs. Caught by this test's first trip through CI, on. + // The child builds the payload; it is not passed in as an argument. const text = "— ".repeat(60_000) + "END" const [cmd, args] = fake( `const t = "— ".repeat(60000) + "END"; process.stdout.write(JSON.stringify({event:"answer",text:t})+"\\n")`, @@ -183,8 +167,7 @@ describe("runCapturingAnswer", () => { assert.ok(seen.join("").includes('"event":"answer"'), "the child's output never reached our stdout") }) - // `tc` not installed. The old spawnSync gave status null here and the - // caller turned it into 1; throwing instead would lose the whole report. + // `tc` not installed. test("a command that does not exist resolves instead of throwing", async () => { const r = await runCapturingAnswer("tc-does-not-exist-anywhere", [], process.env) assert.equal(r.status, null) diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index 1d99050..872d454 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -6,7 +6,7 @@ import { appCredentials, jwtClaims, signJwt, installationTokenRequest } from ".. const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) const pem = privateKey.export({ type: "pkcs1", format: "pem" }) -describe("appCredentials — Release 4", () => { +describe("appCredentials", () => { test("both halves are needed", () => { assert.equal(appCredentials({ TC_ACTION_APP_ID: "1", TC_ACTION_APP_PRIVATE_KEY: "k" })?.appId, "1") assert.equal(appCredentials({ TC_ACTION_APP_ID: "1" }), undefined) @@ -20,18 +20,14 @@ describe("appCredentials — Release 4", () => { }) }) -describe("jwtClaims — Release 4", () => { +describe("jwtClaims", () => { const now = 1_800_000_000 test("issued by the app", () => { assert.equal(jwtClaims("12345", now).iss, "12345") }) - /** - * GitHub rejects a JWT whose iat is in the future, and runner clocks drift. - * Backdating by a minute is the documented workaround; without it a run - * fails with "issued at is in the future" on an otherwise correct setup. - */ + // GitHub rejects a JWT whose iat is in the future, and runner clocks drift. test("issued a minute ago, to survive clock drift", () => { assert.equal(jwtClaims("1", now).iat, now - 60) }) @@ -44,7 +40,7 @@ describe("jwtClaims — Release 4", () => { }) }) -describe("signJwt — Release 4", () => { +describe("signJwt", () => { test("three base64url parts", () => { const parts = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }).split(".") assert.equal(parts.length, 3) @@ -72,19 +68,11 @@ describe("signJwt — Release 4", () => { }) describe("installationTokenRequest", () => { - /** - * the Action uses the App's identity "FOR THAT REPO ONLY". An - * installation token minted with no body carries every permission the - * installation holds, across every repository it covers — so a run in one - * repository would hold write access to all of an organisation's. - */ test("scoped to this repository alone", () => { assert.deepEqual(installationTokenRequest("acme/shop").repositories, ["shop"]) }) test("the owner is not part of the repository name", () => { - // The API wants bare names; "acme/shop" is rejected as not found, which - // would silently fall back to the workflow token. assert.deepEqual(installationTokenRequest("acme/deep-name").repositories, ["deep-name"]) }) @@ -95,8 +83,6 @@ describe("installationTokenRequest", () => { }) test("a malformed repository asks for nothing rather than everything", () => { - // Falling back to an unscoped request would be the worst reading of a - // value we could not parse. assert.equal(installationTokenRequest("not-a-repo"), undefined) assert.equal(installationTokenRequest(""), undefined) assert.equal(installationTokenRequest(undefined), undefined) diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index a0d21a0..e2b7920 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -15,22 +15,12 @@ import { } from "../src/report.mjs" import { OUTCOMES } from "../src/outcome.mjs" -/** - * "go" should say which request it is answering. - * - * Two problems, one cause: the approval machinery was keyed on the COMMIT and - * nothing else. With two plans waiting there was nothing in what you typed to - * pick one, and the record of which plans had been used could not tell two - * jobs apart — so approving one blocked approving any other on that commit - * until somebody pushed. - */ - const US = "test-companion[bot]" const SHA = "abc123def4567890" const now = Date.parse("2026-11-03T12:00:00Z") const fresh = "2026-11-03T11:00:00Z" -/** A proposal comment from us, for one job. */ +// A proposal comment from us, for one job. const proposal = (id, workflow, { requester = "dev", sha = SHA, at = fresh } = {}) => ({ id, user: { type: "Bot", login: US }, @@ -39,7 +29,7 @@ const proposal = (id, workflow, { requester = "dev", sha = SHA, at = fresh } = { body: `Here is the plan.\n\n${proposalMarkerFor(sha, requester, workflow)}`, }) -/** Our own comment recording that a plan was spent. */ +// Our own comment recording that a plan was spent. const consumed = (id, workflow, sha = SHA) => ({ id, user: { type: "Bot", login: US }, @@ -61,9 +51,6 @@ describe("go can name the plan it answers", () => { }) test("the intent carries the id a reply named", () => { - // What readIntent actually receives: plan-run.mjs strips the mention - // before this, so the pattern is anchored to the approval word and not - // to the start of the raw comment. const got = readIntent("go #6580278184", { proposalWaiting: true }) assert.equal(got.kind, INTENT.GO) assert.equal(got.proposalId, "6580278184") @@ -82,7 +69,6 @@ describe("go can name the plan it answers", () => { }) test("a pull request number is not a comment id", () => { - // "go #25" is somebody pointing at a pull request, not at a plan. const got = readIntent("go #25", { proposalWaiting: true }) assert.equal(got.kind, INTENT.GO) assert.equal(got.proposalId, undefined) @@ -101,7 +87,6 @@ describe("go can name the plan it answers", () => { describe("the record of a spent plan knows which job it was", () => { test("approving one job leaves another job's plan approvable", () => { - // The bug, in one assertion. Both plans are on the same commit. const comments = [proposal(1001, "generate"), proposal(1002, "fix"), consumed(1003, "fix")] const got = findProposal(comments, SHA, now, US) assert.equal(got?.id, 1001, "the generate plan was never approved and must still be waiting") @@ -124,8 +109,6 @@ describe("the record of a spent plan knows which job it was", () => { }) test("a marker written before jobs were recorded still spends the whole commit", () => { - // Back-compat: an old thread mid-upgrade. Treating it as spending - // nothing would re-run work somebody already approved. const old = { id: 1003, user: { type: "Bot", login: US }, created_at: fresh, body: proposalConsumedMarkerFor(SHA) } const comments = [proposal(1001, "generate"), proposal(1002, "fix"), old] assert.equal(findProposal(comments, SHA, now, US), undefined) @@ -174,18 +157,6 @@ describe("the result comment says which plan it carried out", () => { }) describe("naming a plan does not need the mention", () => { - /** - * FOUND ON A LIVE RUN, 3 November. `readIntent` and `findProposal` both - * learned about `go `, and the gate in plan-run.mjs that decides - * whether an UNADDRESSED comment reaches us at all did not — it still - * only accepted a bare `go`. So `go ` was dropped as "no - * @browserstack mention" before the intent was ever read, and naming a - * plan only worked if you also addressed the comment. - * - * Still narrow. `go` followed by a plan reference and nothing else is - * unmistakably aimed at us; "Approved, but rename the login case" is still - * an ordinary review comment and still needs the mention. - */ const comment = (body) => ({ action: "created", issue: { number: 7, pull_request: {} }, @@ -220,22 +191,12 @@ describe("naming a plan does not need the mention", () => { }) test("and a number that is not a plan reference does not open the gate", () => { - // "go #25" is somebody pointing at a pull request. It is a bare-ish go, - // so it is carried through — but as a BARE one, with no plan named. const out = planRun({ eventName: "issue_comment", payload: comment("go and look at #25 too"), repo: "acme/shop" }) assert.equal(out.skip?.reason, "not-addressed-to-us") }) }) describe("naming a plan is not editing it", () => { - /** - * `hasApproverEdits("go ")` - * was true, because the bare-approval pattern allowed no plan reference. - * That is not just a wrong line in the report: renderApprovedPlan uses the - * same answer to append "# Changes the approver made … these win" to the - * plan the agent is handed. So the per-job approval feature injected its own - * comment id into the plan as an authoritative instruction. - */ test("go #id is a bare approval", () => { assert.equal(hasApproverEdits("go #6600030205"), false) assert.equal(hasApproverEdits("approved #6600030205"), false) @@ -262,14 +223,6 @@ describe("naming a plan is not editing it", () => { }) describe("a plan reference has to sit next to the approval", () => { - /** - * GO_TARGET matched a `#id` anywhere in the comment, so - * "@browserstack go — and pull the steps from #issuecomment-…" - * became an approval of a plan that does not exist, and was refused. - * - * readIntent receives the comment with the mention already removed by - * plan-run.mjs, so that is what is passed here. - */ test("a link quoted later in the sentence is not a plan reference", () => { const got = readIntent("go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", { proposalWaiting: true, @@ -287,11 +240,6 @@ describe("a plan reference has to sit next to the approval", () => { }) }) -/** - * The approval word is only recognised at the START of the instruction, so - * this pins the hand-off it depends on: plan-run.mjs removes the mention, and - * what reaches readIntent opens with the word the person actually led with. - */ describe("a real approval comment, end to end through plan-run", () => { const comment = (body) => ({ action: "created", diff --git a/test-companion/test/build-effect.test.mjs b/test-companion/test/build-effect.test.mjs index 0dc27a2..caa74ca 100644 --- a/test-companion/test/build-effect.test.mjs +++ b/test-companion/test/build-effect.test.mjs @@ -3,20 +3,9 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" import { OUTCOMES, stepShouldFail, BUILD_EFFECTS } from "../src/outcome.mjs" -/** - * A run that BROKE should redden the build. - * - * It never did unless a repository opted in, which meant every bad outcome so - * far passed silently. That is not what anyone looking at a green tick would - * assume. - * - * The line is drawn at whether the RUN worked, not at whether the tests - * passed. A test failure is a result; making it block the build is a much - * bigger decision than this one, and `fail` is still there for anyone who - * wants it. - */ - -/** The run itself did not work. */ +// A run that broke should redden the build. + +// The run itself did not work. const BROKEN = [OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT] describe("the default reddens a broken run and nothing else", () => { @@ -36,23 +25,11 @@ describe("the default reddens a broken run and nothing else", () => { }) test("saying nothing means the default", () => { - // The step is what a repository gets without reading the setting, so it - // has to BE the default rather than merely document one. assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) assert.equal(stepShouldFail("", OUTCOMES.RUN_BROKE), true) assert.equal(stepShouldFail(undefined, OUTCOMES.TESTS_FAILED), false) }) - /** - * This reverses the old rule, deliberately. - * - * It used to be "anything unrecognised behaves as report", on the reasoning - * that reddening builds is the one direction a misconfiguration must not - * fail in. That reasoning made sense while report WAS the default. Now that - * it is not, a typo buying you a quieter setting than everybody else gets - * is exactly the silent pass to avoid — and a misspelling leaves you exactly - * where a fresh install would be, which is defensible. - */ test("a misspelling lands on the default, not on the quietest setting", () => { assert.equal(stepShouldFail("reprot", OUTCOMES.RUN_BROKE), true) assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), true) @@ -67,8 +44,6 @@ describe("the two explicit settings keep their meaning", () => { }) test("an existing off keeps working as report", () => { - // Undocumented from now on, but a workflow already saying it must not - // start failing builds because we stopped advertising the word. for (const o of Object.values(OUTCOMES)) { assert.equal(stepShouldFail("off", o), false, `${o} reddened the step under off`) } @@ -107,11 +82,6 @@ describe("the input documents what it does", () => { }) describe("an unrecognised value explains the reddening it causes", () => { - /** - * Reversing the fail-closed rule - * means `build-effect: none` / `warn` / `false` starts reddening builds. - * That is the right default, but it should say why. - */ test("it warns, naming the value and the default", () => { const said = [] const real = process.stdout.write.bind(process.stdout) diff --git a/test-companion/test/child-env.test.mjs b/test-companion/test/child-env.test.mjs index f88dc28..ff15e31 100644 --- a/test-companion/test/child-env.test.mjs +++ b/test-companion/test/child-env.test.mjs @@ -16,15 +16,6 @@ describe("childEnvFor — what the agent's shell can see", () => { assert.equal(out.GITHUB_TOKEN, undefined) }) - /** - * The one the allowlist would have let through. TC_ is a kept PREFIX, so - * the App's private key — TC_ACTION_APP_PRIVATE_KEY — matched it and would - * have been handed to a shell running commands the model chose. - * - * That key can mint installation tokens for every repository the App is on, - * which is a far worse thing to leak than the workflow token the allowlist - * was written to keep out. - */ test("the App's private key never does either, despite its TC_ prefix", () => { const out = childEnvFor({ TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN RSA PRIVATE KEY-----", diff --git a/test-companion/test/framework.test.mjs b/test-companion/test/framework.test.mjs index 754ad37..560b02a 100644 --- a/test-companion/test/framework.test.mjs +++ b/test-companion/test/framework.test.mjs @@ -24,10 +24,6 @@ describe("detectFramework", () => { assert.equal(found, "Cypress") }) - /** - * Guessing wrong is worse than not guessing: the starter file would tell the - * team Test Companion had understood their setup when it had not. - */ test("nothing recognisable is undefined, not a guess", () => { assert.equal(detectFramework({ filenames: ["README.md"], deps: { react: "^18" } }), undefined) assert.equal(detectFramework({}), undefined) diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs index b9d2b5e..1380738 100644 --- a/test-companion/test/gate.test.mjs +++ b/test-companion/test/gate.test.mjs @@ -13,12 +13,6 @@ describe("decideGate", () => { assert.equal(decideGate({ intent: asking, approval: "straight-through" }).action, GATE.RUN) }) - /** - * "the run SHALL post its proposal and end. Nothing SHALL be written - * before 'go'." The proposal run is a PREVIEW run — preview is already - * enforced in tool dispatch as of Release 1, so "nothing is written" is a - * property of the run rather than a promise in a prompt. - */ test("review-before-run proposes instead of running", () => { assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).action, GATE.PROPOSE) }) @@ -41,25 +35,16 @@ describe("decideGate", () => { assert.equal(decideGate({ intent: go, approval: "review-before-run" }).action, GATE.NOTHING_TO_APPROVE) }) - // Nothing was ever held, so there is nothing to approve. Running on the - // strength of "go" alone would start a job nobody named. test('"go" on a straight-through repository has nothing to approve either', () => { assert.equal(decideGate({ intent: go, approval: "straight-through" }).action, GATE.NOTHING_TO_APPROVE) }) - /** - * Fails CLOSED. An approval value that never went through the CLI's - * validation — or a bindings read that failed — holds for review rather - * than running unreviewed, matching ResolvedApproval on the Go side. - */ test("an unknown approval setting holds for review", () => { assert.equal(decideGate({ intent: asking, approval: undefined }).action, GATE.PROPOSE) assert.equal(decideGate({ intent: asking, approval: "nonsense" }).action, GATE.PROPOSE) }) test("asking for a preview explicitly is not held for approval", () => { - // They already asked for the safe thing; holding it for review would be - // asking permission to write nothing. const previewing = { kind: INTENT.WORKFLOW, workflow: "test", preview: true } const d = decideGate({ intent: previewing, approval: "review-before-run" }) assert.equal(d.action, GATE.RUN) @@ -68,14 +53,6 @@ describe("decideGate", () => { }) describe("decideGate — who approved it", () => { - /** - * Both the asker and the approver need write access, so this is not a - * privilege boundary — it is a workflow-integrity signal. user story - * has an SDET approving a developer's request; the normative text only says - * "someone who can ask". So self-approval is allowed and REPORTED, rather - * than silently permitted or blocked outright (blocking would strand a solo - * maintainer, who could then never approve anything). - */ test("approving your own proposal is flagged", () => { const d = decideGate({ intent: { kind: INTENT.GO }, @@ -109,8 +86,6 @@ describe("decideGate — who approved it", () => { }) describe("a very large change", () => { - // thresholds, marked ASSUMPTION in the PRD: more than 20 files or - // more than 1,000 changed lines. test("an ordinary change is not large", () => { assert.equal(describeChangeSize({ changed_files: 3, additions: 40, deletions: 10 }).large, false) }) @@ -134,24 +109,12 @@ describe("a very large change", () => { assert.equal(d.lines, 1100) }) - // A pull request object we could not read must not be called large; the - // consequence would be telling every reviewer their change is too big. test("an unreadable pull request is not assumed to be large", () => { assert.equal(describeChangeSize(undefined).large, false) assert.equal(describeChangeSize({}).large, false) }) }) -/** - * Asking "why did this fail?" cost two comments and a round trip: the first - * run posted an empty proposal — empty because only `test-pr` has a plan to - * show — and the answer only came after someone replied "go". Seen on live - * runs 41048492170 and 41049253928. - * - * The gate exists to stop WRITES. `rca` and `verdict` produce an explanation - * and nothing else, which is the same reasoning already applied one line - * above for a preview that was asked for directly. - */ describe("decideGate — a job that only answers does not ask permission first", () => { const asked = { kind: INTENT.WORKFLOW } @@ -159,8 +122,6 @@ describe("decideGate — a job that only answers does not ask permission first", for (const workflow of ["rca", "verdict"]) { const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) assert.equal(got.action, GATE.RUN, `${workflow} writes nothing; holding it asks permission to write nothing`) - // Skipping the hold is only safe because the run cannot write, so - // the run is MADE unable to write rather than assumed to be. assert.equal(got.preview, true, `an unapproved ${workflow} must not get write tools`) } }) diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index 69f1588..4133404 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -3,24 +3,10 @@ import assert from "node:assert/strict" import { readFileSync, readdirSync } from "node:fs" import { join } from "node:path" -/** - * main.mjs is the one file here with no unit tests, by design: everything - * decidable lives in the pure modules beside it and this is the I/O shell. - * - * The cost of that showed up twice in live runs. `renderProposalBody` and - * `hasApproverEdits` were used in main.mjs and never imported, and nothing - * caught it — a ReferenceError inside a function body does not throw until - * that function runs, so neither `node --check` nor importing the module would - * have found it. The whole 278-test suite passed while the Action crashed on a - * runner. - * - * This closes that gap statically: every helper main.mjs CALLS must be in its - * import list, and everything it imports must actually be exported. - */ const SRC = new URL("../src/", import.meta.url).pathname const main = readFileSync(join(SRC, "main.mjs"), "utf8") -/** `import { a, b } from "./x.mjs"` → { "./x.mjs": ["a","b"] } */ +// `import { a, b } from "./x.mjs"` → { "./x.mjs": ["a","b"] } function localImports(source) { const out = {} for (const m of source.matchAll(/import\s*\{([^}]+)\}\s*from\s*"(\.\/[^"]+)"/g)) { @@ -32,7 +18,6 @@ function localImports(source) { return out } -/** The names a module exports. */ function exportsOf(file) { const text = readFileSync(join(SRC, file), "utf8") return [...text.matchAll(/export\s+(?:async\s+)?function\s+(\w+)|export\s+const\s+(\w+)/g)].map((m) => m[1] ?? m[2]) @@ -58,8 +43,7 @@ describe("main.mjs imports everything it uses", () => { test(`every ${file} helper main.mjs CALLS is imported`, () => { for (const name of exportsOf(file)) { - // A call, not a mention: `foo(` or `foo (`. Constants are not - // called and are covered by the check above. + // A call, not a mention: `foo(` or `foo (`. const called = new RegExp(`\\b${name}\\s*\\(`).test(withoutImportBlock) if (called) { assert.ok(named.includes(name), `main.mjs calls ${name}() but does not import it from ${file}`) @@ -70,8 +54,7 @@ describe("main.mjs imports everything it uses", () => { }) describe("every src module is reachable", () => { - // A module nobody imports is either dead or a missing wire-up. Both are - // worth noticing; neither shows up in a passing test suite. + // A module nobody imports is either dead or a missing wire-up. test("no source file is orphaned", () => { const files = readdirSync(SRC).filter((f) => f.endsWith(".mjs") && f !== "main.mjs") const allSource = files.map((f) => readFileSync(join(SRC, f), "utf8")).join("\n") + main @@ -81,23 +64,6 @@ describe("every src module is reachable", () => { }) }) -/** - * Found by a live run. Asking "@browserstack explain why - * the contract check is failing" worked: the Action found the red build and - * passed it to the CLI. Replying "go" to the proposal it put up did NOT - * the CLI was spawned with no build at all and died with - * `rca requires --context build=`. - * - * The cause is pure ordering. `rca`, `fix` and `verdict` need to know which - * build failed, and the lookup is guarded by `NEEDS_A_BUILD.has(plan.workflow)`. - * On an approval, the workflow is not known from the comment — "go" names no - * job — it comes from the proposal we are approving, and that assignment sat - * BELOW the lookup. So the lookup ran against the wrong workflow, decided no - * build was needed, and the run broke at the last step of the feature. - * - * Ordering inside one long procedure is not something the pure modules can - * pin, so it is pinned here. - */ describe("main.mjs — the failing build is resolved after the workflow is final", () => { const at = (needle) => { const i = main.indexOf(needle) @@ -118,15 +84,7 @@ describe("main.mjs — the failing build is resolved after the workflow is final }) }) -/** - * The App's private key arrives in the environment and stays there. Anything - * spawned afterwards inherits it — and the agent runs with --yolo, so "what - * gets spawned afterwards" includes programs it chose. - * - * The git half of keeping credentials from the agent is closed in push-work.mjs (minimal env, hooks and - * fsmonitor off, config allow-listed, binary pinned). This is the other half: - * once the key has been used, it should not still be readable. - */ +// The App's private key arrives in the environment and stays there. describe("main.mjs — the App private key does not outlive its use", () => { const at = (needle) => { const i = main.indexOf(needle) @@ -139,13 +97,7 @@ describe("main.mjs — the App private key does not outlive its use", () => { }) test("deleted while minting the token, before anything else is spawned", () => { - // The agent is the process that matters: it runs with --yolo and picks - // its own commands. - // - // Anchored on the CALL. The spawn itself moved into answer.mjs when the - // Action started reading the agent's output, and anchoring on - // `spawn(` would now find that module's own line, which says nothing - // about the order things happen here. + // The agent is the process that matters: it runs with --yolo and picks its own commands. assert.ok( at("delete env.TC_ACTION_APP_PRIVATE_KEY") < at('await runCapturingAnswer("tc"'), "the agent could still read it", @@ -153,15 +105,6 @@ describe("main.mjs — the App private key does not outlive its use", () => { }) }) -/** - * An approved `fix` that breaks in under a second does no work and writes - * nothing, so the result comment must not carry the marker saying the plan - * was spent — otherwise every later `go` on that commit is refused, a fresh - * proposal included, with no way out but a new commit. - * - * The decision lives in approvalWasSpent and is unit-tested there. What cannot - * be unit-tested is that main.mjs actually asks it, so it is pinned here. - */ describe("main.mjs — a run that never ran does not spend the approval", () => { test("the consumed marker is gated on the outcome", () => { assert.match( diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 855d4d8..4014229 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -2,22 +2,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { readIntent, INTENT } from "../src/intent.mjs" -/** - * — what did the comment actually ask for? - * - * The mapping has to happen HERE, before anything runs, because says a - * request we cannot map gets a reply and "nothing SHALL run". Handing the free - * text to the agent and letting it decide would spend the customer's credits - * working out that "make coffee" was never a job. - */ describe("readIntent — a comment naming a job", () => { - /** - * Caught by the FIRST LIVE RUN (rag_assignment ). This table was - * written before `test-pr` existed and still said "test", so the flagship - * phrase resolved to the job whose wrongness is the entire reason test-pr - * exists — and the reply read "`test` is not bound to pull request - * comments" on a repository whose pipeline bound test-pr. - */ test('"test this PR" asks for test-pr, the job that reads the diff', () => { assert.deepEqual(readIntent("test this PR against SHOP-221", { workflows: ["test-pr"] }), { kind: INTENT.WORKFLOW, @@ -26,8 +11,6 @@ describe("readIntent — a comment naming a job", () => { }) }) - // A repository still bound to the older `test` keeps working: the phrase - // prefers test-pr and falls back to whatever is actually bound. test("falls back to `test` where that is what the repository bound", () => { assert.equal(readIntent("test this PR", { workflows: ["test"] }).workflow, "test") }) @@ -54,8 +37,6 @@ describe("readIntent — a comment naming a job", () => { }) describe("readIntent — a request that is not a job", () => { - // "A request it cannot map SHALL get a reply listing what it can do, - // and nothing SHALL run." Unknown is a real answer, not a failure. test("a request it cannot place is unknown, not a guess", () => { assert.equal(readIntent("make coffee").kind, INTENT.UNKNOWN) }) @@ -70,8 +51,6 @@ describe("readIntent — asking for something other than a job", () => { assert.equal(readIntent("stop").kind, INTENT.STOP) }) - // gives stop its own 30-second deadline, so it must not lose to a job - // word that happens to share the sentence. test("stop wins over a job named in the same breath", () => { assert.equal(readIntent("stop, don't fix anything").kind, INTENT.STOP) }) @@ -91,9 +70,6 @@ describe("readIntent — preview is a modifier, not a job", () => { }) }) - // "Any comment MAY ask for a single preview run." With no job named, - // the pipeline file still decides WHICH job — exactly as an event-driven run - // does — so the workflow is left null and only the preview flag is set. test("preview on its own previews whatever the pipeline binds", () => { assert.deepEqual(readIntent("preview"), { kind: INTENT.WORKFLOW, @@ -112,12 +88,7 @@ describe("readIntent — preview is a modifier, not a job", () => { }) describe("readIntent — quoted text is not a request", () => { - /** - * GitHub's "Quote reply" button copies the whole earlier comment in as `>` - * lines. Without this, replying to Test Companion's own "say stop to cancel" - * message would cancel the run — the quote matches before the person's - * actual words do. - */ + // GitHub's "Quote reply" button copies the whole earlier comment in as `>` lines. test("a quoted line cannot start or stop anything", () => { assert.equal(readIntent("> stop\n\nthanks, carry on").kind, INTENT.UNKNOWN) }) @@ -128,8 +99,6 @@ describe("readIntent — quoted text is not a request", () => { }) describe("readIntent — approving a proposal", () => { - // "a reply of 'go', with or without edits to the case list, SHALL - // start a new run on exactly the approved list." test('a bare "go" approves', () => { assert.equal(readIntent("go").kind, INTENT.GO) }) @@ -142,11 +111,6 @@ describe("readIntent — approving a proposal", () => { assert.equal(readIntent("go, but drop the third case").kind, INTENT.GO) }) - /** - * Checked AFTER the job words on purpose. "go" is two letters and turns up - * inside ordinary instructions, so a sentence that names a job is read as - * asking for that job, and only a reply that names none is an approval. - */ test("naming a job still asks for the job", () => { assert.equal(readIntent("go on then, fix the failing tests").workflow, "fix") }) @@ -157,10 +121,6 @@ describe("readIntent — approving a proposal", () => { }) describe("readIntent — any BOUND workflow, not just the pretty phrases", () => { - // says "any bound workflow in plain words". Four hand-written phrases - // covered four of the CLI's fourteen jobs; asking for the rest by name got - // the can't-place reply. Naming a job IS plain words, so the job names the - // repository has bound are accepted literally. const bound = ["generate", "test", "rca", "fix", "validate", "results", "verdict", "automate", "mobile run"] test("a job asked for by name is found", () => { @@ -177,8 +137,6 @@ describe("readIntent — any BOUND workflow, not just the pretty phrases", () => assert.equal(d.workflow, "mobile run") }) - // The written-out phrases still win, so "explain this failure" stays rca - // rather than matching some job literally called "explain". test("the plain phrases still take precedence", () => { assert.equal(readIntent("explain this failure", { workflows: bound }).workflow, "rca") }) @@ -203,18 +161,6 @@ describe("readIntent — any BOUND workflow, not just the pretty phrases", () => }) describe("stop and set up are what the comment ASKS FOR, not words it contains", () => { - /** - * STOP and SETUP are tested before the job phrases — deliberately, so - * "stop, don't fix anything" cannot start a fix run. Matching the word - * ANYWHERE was the bug: - * - * "don't stop until the tests pass" cancelled the run - * "the test setup is broken, fix it" opened a settings pull request - * - * Anchoring to the start keeps the case the ordering exists for and drops - * the ones where the word is incidental. A leading politeness is allowed, - * because "please stop" is plainly a stop. - */ const jobs = { workflows: ["test-pr", "fix", "rca"] } test('"stop" still stops, with or without politeness', () => { @@ -240,10 +186,6 @@ describe("stop and set up are what the comment ASKS FOR, not words it contains", }) }) -/** - * These pin the behaviour that matters, so the matcher can be rewritten - * without building a pattern out of input. - */ describe("naming a bound job — matched on whole words, never as a substring", () => { const bound = { workflows: ["generate", "test-pr", "mobile run", "rca"] } @@ -268,16 +210,6 @@ describe("naming a bound job — matched on whole words, never as a substring", }) }) -/** - * Job words are matched anywhere in the comment and were tested BEFORE the - * approval word. So a reply that approves a waiting plan and adds an edit - * exactly what invites, "you can edit the list first" — was read as a - * fresh request for a different job: - * - * "go, and fix case 3's title" -> proposes `fix`, instead of approving - * - * When a plan is waiting, the approval reading comes first. - */ describe("a reply to a waiting proposal is read as an approval first", () => { const bound = { workflows: ["generate", "fix", "rca"], proposalWaiting: true } @@ -293,8 +225,6 @@ describe("a reply to a waiting proposal is read as an approval first", () => { }) test("stop still wins over an approval, even with one waiting", () => { - // gives stop its own deadline; approving and cancelling in one - // breath must cancel. assert.equal(readIntent("stop", bound).kind, INTENT.STOP) }) @@ -303,17 +233,6 @@ describe("a reply to a waiting proposal is read as an approval first", () => { }) }) -/** - * The approval word counts only when the comment OPENS with it, like stop and - * set up. Matched anywhere, a comment that merely mentions it approved a - * waiting plan, and that starts a LIVE run that writes to the branch: - * - * "don't go ahead yet, explain this failure" -> approved - * "no, do not approve that, test this PR instead" -> approved - * - * and with nothing waiting, "let's go over the login flow" was told there was - * no plan to approve instead of being offered what it can ask for. - */ describe("go approves only when the comment opens with it", () => { const waiting = { workflows: ["test-pr", "fix", "rca"], proposalWaiting: true } const idle = { workflows: ["test-pr", "fix", "rca"], proposalWaiting: false } diff --git a/test-companion/test/outcome-emoji.test.mjs b/test-companion/test/outcome-emoji.test.mjs index c9ba40e..b9f3f61 100644 --- a/test-companion/test/outcome-emoji.test.mjs +++ b/test-companion/test/outcome-emoji.test.mjs @@ -3,14 +3,6 @@ import assert from "node:assert/strict" import { OUTCOMES, emojiFor } from "../src/outcome.mjs" import { renderJobSummary } from "../src/report.mjs" -/** - * The outcome should be readable at a glance. - * - * "## Test Companion — Run broke" and "## Test Companion — Succeeded" look the - * same in a crowded pull request thread, and the one that needs attention is - * the one that blends in. - */ - describe("every outcome has a mark of its own", () => { test("no two outcomes share one", () => { const marks = Object.values(OUTCOMES).map(emojiFor) @@ -25,7 +17,6 @@ describe("every outcome has a mark of its own", () => { test("an outcome we do not know does not break the heading", () => { // A newer CLI could report something this Action has never heard of. - // A missing mark is a cosmetic loss; a crash is a lost report. assert.equal(emojiFor("Something new"), "") assert.equal(emojiFor(undefined), "") }) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs index cd45cbe..7ee60ba 100644 --- a/test-companion/test/outcome.test.mjs +++ b/test-companion/test/outcome.test.mjs @@ -38,8 +38,6 @@ describe("resolveOutcome", () => { assert.equal(resolveOutcome(5, {}), OUTCOMES.TESTS_FAILED) }) - // Exit 4 covers three different causes. Only the report distinguishes a - // ceiling from a crash, which is why the Action reads both. test("a ceiling is told apart from a crash, using the report", () => { assert.equal(resolveOutcome(4, { error_type: "ceiling_exceeded" }), OUTCOMES.STOPPED_AT_LIMIT) assert.equal(resolveOutcome(4, {}), OUTCOMES.RUN_BROKE) @@ -57,9 +55,6 @@ describe("resolveOutcome", () => { assert.equal(resolveOutcome(7, {}), OUTCOMES.COULD_NOT_RUN) }) - // The observed false green: an expired session ended with status completed - // and exit 0 having done nothing. If the CLI said it could not run, believe - // the report over the exit code. test("believes the report when it contradicts a success exit code", () => { assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) }) @@ -87,8 +82,6 @@ describe("checkConclusion", () => { assert.equal(checkConclusion(OUTCOMES.TESTS_FAILED), "failure") }) - // A run that could not finish must not be able to satisfy a required check, - // but it is not the same as a test failure — someone has to do something. test("asks for a person when the run could not finish", () => { assert.equal(checkConclusion(OUTCOMES.STOPPED_AT_LIMIT), "action_required") assert.equal(checkConclusion(OUTCOMES.COULD_NOT_RUN), "action_required") @@ -117,22 +110,12 @@ describe("stepShouldFail — build effect", () => { assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) }) - // REVERSED by. While `report` was the - // default, "a misconfiguration must never redden a build" was the safe - // direction. The default is now fail-on-break, so a typo landing you on a - // quieter setting than a fresh install is exactly the silent pass to avoid. - // The whole rule lives in build-effect.test.mjs now; this is the hinge. test("anything unrecognised behaves as the default", () => { assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), true) }) }) -// P0 from review: an event nothing is bound to must not look like a failure. -// -// The CLI now exits 0 and writes `outcome: skipped` for an unbound event. The -// old code took exit 0 with any non-"succeeded" outcome as RUN_BROKE, so this -// would have gone from one wrong answer to another. describe("resolveOutcome — an unbound event is skipped, not broken", () => { test("exit 0 with outcome skipped is SKIPPED", () => { assert.equal(resolveOutcome(0, { outcome: "skipped" }), OUTCOMES.SKIPPED) @@ -152,33 +135,17 @@ describe("resolveOutcome — an unbound event is skipped, not broken", () => { }) test("a skipped outcome never reddens the step or the check", () => { - // The arguments were the wrong way round here, which made this pass - // without ever asking the question. stepShouldFail(buildEffect, outcome). assert.equal(stepShouldFail("fail", OUTCOMES.SKIPPED), false) assert.notEqual(checkConclusion(OUTCOMES.SKIPPED), "failure") }) }) -/** - * Approving a plan is spent by the run that carries it out — but only if a run - * actually happened. - * - * Found live: an approved `fix` died in under a second before doing any work - * (the dedupe claim had no credential), and the result comment - * still carried the consumed marker. Every later `go` on that commit was then - * refused — "already been approved and run on this commit" — including after a - * fresh proposal was put up. The only way out was a new commit. - * - * The dedupe claim already has this property: it releases itself on a - * non-success outcome. An approval should behave the same way. - */ +// Approving a plan is spent by the run that carries it out — but only if a run actually happened. describe("approvalWasSpent", () => { test("a run that succeeded spends it", () => { assert.equal(approvalWasSpent(OUTCOMES.SUCCEEDED), true) }) - // The run did the work it was approved for. Red tests are a result, not a - // failure to run — re-running would redo the work and charge for it again. test("failing tests still spend it", () => { assert.equal(approvalWasSpent(OUTCOMES.TESTS_FAILED), true) }) @@ -200,8 +167,6 @@ describe("approvalWasSpent", () => { assert.equal(approvalWasSpent(OUTCOMES.SKIPPED), false) }) - // Totality: a new outcome must be considered rather than silently spending - // an approval because it is not one of the two named failures. test("every outcome has an answer", () => { for (const o of Object.values(OUTCOMES)) { assert.equal(typeof approvalWasSpent(o), "boolean", `${o} has no answer`) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index bfec758..a9917d8 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -2,8 +2,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { planRun, TC_EVENTS, contextArgs, withPullRequestHead } from "../src/plan-run.mjs" -// Minimal payload builders. Each returns the shape GitHub actually puts in -// $GITHUB_EVENT_PATH for that event, trimmed to the fields we read. const pr = (over = {}) => ({ number: 412, draft: false, @@ -29,8 +27,6 @@ describe("comment trigger", () => { assert.equal(r.context.pr, "412") }) - // The comment's wording is the user's instruction — it is the whole point of - // the trigger, so it has to reach the run rather than being thrown away. test("carries what the person actually asked for", () => { const r = plan("issue_comment", { action: "created", @@ -59,8 +55,7 @@ describe("comment trigger", () => { assert.equal(r.skip.reason, "not-addressed-to-us") }) - // editing a comment must not start a run. Otherwise fixing a typo in - // a months-old comment re-runs the job. + // editing a comment must not start a run. test("ignores an edited comment", () => { const r = plan("issue_comment", { action: "edited", @@ -90,9 +85,6 @@ describe("pull request triggers", () => { pull_request: pr({ merged: true, state: "closed", merge_commit_sha: "deadbee" }), }) assert.equal(r.event, TC_EVENTS.PR_MERGED) - // The merge commit is the change that landed on the base branch; the head - // sha is the branch that no longer exists. Scoping to the wrong one makes - // the run look at code that was never merged. assert.equal(r.context.sha, "deadbee") assert.equal(r.context.branch, "main") }) @@ -102,15 +94,12 @@ describe("pull request triggers", () => { assert.equal(r.skip.reason, "closed-unmerged") }) - // a draft never starts a run on its own. A comment still can, which is - // why this check belongs here and not in the comment path. + // a draft never starts a run on its own. test("a draft pull request does not start a run on its own", () => { const r = plan("pull_request", { action: "opened", pull_request: pr({ draft: true }) }) assert.equal(r.skip.reason, "draft") }) - // . A fork PR runs with a read-only token and no secrets, so starting - // would fail confusingly partway. Refusing up front is the clean skip. test("a pull request from a fork does not run", () => { const r = plan("pull_request", { action: "opened", @@ -153,9 +142,6 @@ describe("red build trigger", () => { assert.equal(r.skip.reason, "build-not-red") }) - // never explain a red run that is our own. Without this, a failing - // Test Companion check triggers a run to explain the failing Test Companion - // check, which triggers another one. test("never explains its own failed run", () => { const r = plan( "workflow_run", @@ -189,8 +175,6 @@ describe("other triggers", () => { assert.equal(r.event, TC_EVENTS.SCHEDULE) }) - // a manual run names its own job and ignores the bindings entirely, - // which the CLI expresses as "no --event at all". test("a manual run carries no event, so bindings are bypassed", () => { const r = plan( "workflow_dispatch", @@ -209,9 +193,7 @@ describe("other triggers", () => { }) describe("safety", () => { - // The CLI rejects a flag-like context value (CWE-88). A pull request title or - // comment is attacker-controlled on a public repo, so catch it before it is - // ever put on a command line. + // The CLI rejects a flag-like context value (CWE-88). test("refuses a context value that looks like a flag", () => { const r = plan("issue_comment", { action: "created", @@ -231,9 +213,6 @@ describe("safety", () => { }) describe("workflow_run is a privileged trigger", () => { - // workflow_run runs in the BASE repo's context with full secrets, even when - // the run it reacts to came from a fork. That is the documented GitHub - // footgun: the fork check in the pull_request path does not cover this one. test("a red run from a fork does not start a privileged run", () => { const r = plan("workflow_run", { action: "completed", @@ -266,9 +245,6 @@ describe("workflow_run is a privileged trigger", () => { }) describe("context values cannot inject extra keys", () => { - // context is a comma-separated k=v list, so a comment body containing a - // comma and an equals sign could otherwise append keys of its own, e.g. - // overriding the commit the run looks at. test("strips separators out of free text", () => { const r = plan("issue_comment", { action: "created", @@ -316,10 +292,6 @@ describe("workflow_run fails closed", () => { }) describe("an explicitly named job", () => { - // Naming a job should win on ANY trigger, not just a manual one. Explicit - // beats implicit, and it gives a workflow file a way to pin one job without - // touching the pipeline file. Found while testing: the input was silently - // ignored on a pull_request event, which is a confusing no-op. test("overrides the binding on an event-driven run", () => { const r = plan("pull_request", { action: "opened", pull_request: pr() }, { inputs: { workflow: "generate" } }) assert.equal(r.workflow, "generate") @@ -335,11 +307,6 @@ describe("an explicitly named job", () => { }) describe("contextArgs — one flag per pair", () => { - // A single `--context a=1,b=2` is parsed by pflag with encoding/csv, which - // returns ErrBareQuote when an unquoted field contains a double quote. So a - // PR titled `Fix "Pay now" button` made flag parsing fail and the whole run - // reported "Could not run". One flag per pair takes pflag's single-pair - // path, which never reaches the CSV reader. test("emits a separate --context flag for every pair", () => { assert.deepEqual(contextArgs({ sha: "abc", pr: "412" }), ["--context", "sha=abc", "--context", "pr=412"]) }) @@ -369,10 +336,6 @@ describe("cleanContext strips quotes as well as separators", () => { }) describe("a pull request does not send its title as the subject", () => { - // generateSubject in the CLI builds the commit-scoped prompt ("the change in - // PR #N, commit SHA … covering only what that change touches") ONLY when no - // subject is given. Sending the title meant `generate` on a pull request - // wrote cases for the title instead of the diff, which is. test("pr_opened carries sha, branch and pr but no subject", () => { const plan = planRun({ eventName: "pull_request", @@ -385,12 +348,6 @@ describe("a pull request does not send its title as the subject", () => { }) }) -// P1 from review: a comment-triggered run worked on the WRONG TREE. -// -// The issue_comment plan carries `pr` but no `sha`, so checkSha fell back to -// GITHUB_SHA — the default branch tip for that event — and the starter's -// checkout took the default branch too. `@browserstack generate` on -// therefore read main, and the check landed on main's latest commit. describe("withPullRequestHead — a comment run must name the PR's own commit", () => { const plan = () => ({ event: "pr_comment", context: { pr: "42", instructions: "generate" } }) const head = (over = {}) => ({ @@ -430,19 +387,6 @@ describe("withPullRequestHead — a comment run must name the PR's own commit", }) describe("a bare approval reaches us", () => { - /** - * Found by the first approved run. The proposal comment says "Reply **go** - * to run this for real", a person replies exactly that, and the run skipped - * with "No @browserstack mention, so nothing was asked of Test Companion". - * - * The proposal was instructing an action that could not work. says "a - * reply of 'go'", and the PRD's own user story has the SDET replying "go" - * with no mention at all. - * - * Narrow on purpose: the comment must OPEN with the approval word. An - * unmentioned comment is still nothing to do with us in every other case, - * and main.mjs drops this one too unless a proposal is actually waiting. - */ const comment = (body) => ({ action: "created", issue: { number: 7, pull_request: {} }, @@ -455,16 +399,6 @@ describe("a bare approval reaches us", () => { assert.equal(out.approvalOnly, true) }) - /** - * CHANGED. This used to be carried through, so - * "edit the list while approving" worked with no mention. But - * "Approved, but rename the login case" is an ordinary review comment on - * any pull request, and unaddressed it started a live write run with the - * trailing text as approver edits that win. - * - * Editing while approving still works — it just has to be addressed to us - * now, and the proposal comment says so. - */ test("go WITH EDITS needs the mention, or an ordinary review comment starts a run", () => { const out = planRun({ eventName: "issue_comment", payload: comment("go, but drop case two"), repo: "acme/shop" }) assert.equal(out.skip?.reason ?? out.skip, "not-addressed-to-us") @@ -482,8 +416,6 @@ describe("a bare approval reaches us", () => { assert.equal(planRun({ eventName: "issue_comment", payload: comment("approved"), repo: "acme/shop" }).skip, undefined) }) - // The discriminator. "go" has to OPEN the comment, or ordinary chatter on a - // pull request starts runs. test("ordinary chatter that merely contains the word is still ignored", () => { for (const body of ["let's go to lunch", "I'll go over this tomorrow", "this is good to go once CI passes"]) { const out = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) @@ -504,19 +436,6 @@ describe("a bare approval reaches us", () => { }) describe("pull_request_target is a privileged trigger — it must fail CLOSED", () => { - /** - * `pull_request_target` runs in the BASE repository's context with full - * secrets, even when the pull request comes from a fork. That is the - * documented way these workflows get compromised, and this fork check is the - * only thing in front of it: unlike `pull_request`, the credentials ARE - * present, so main.mjs's no-credentials skip never fires. - * - * GitHub sets `head.repo` to null when the fork has been DELETED. The old - * check required head.repo.full_name to be truthy before it would refuse, so - * a deleted fork sailed through — open a pull request from a fork, delete - * the fork, and the run executes attacker-authored code with the base - * repository's credentials and --yolo. - */ const targetEvent = (head) => ({ action: "opened", pull_request: { number: 7, draft: false, head }, @@ -569,8 +488,6 @@ describe("pull_request_target is a privileged trigger — it must fail CLOSED", assert.equal(out.context.sha, "abc") }) - // GitHub treats repository names case-insensitively; workflow_run already - // compares this way and pull_request did not. test("a case difference is the same repository, not a fork", () => { const out = planRun({ eventName: "pull_request_target", @@ -582,19 +499,6 @@ describe("pull_request_target is a privileged trigger — it must fail CLOSED", }) describe("the flag guard sees the FINAL value, not the raw one", () => { - /** - * `looksLikeAFlag` used to run - * BEFORE `cleanContext`, which calls `stripSeparators` and turns `,`, `=` - * and `"` into spaces before trimming. So a value crafted to look safe - * beforehand became flag-like afterwards and the gate never saw it: - * - * `",--config /tmp/evil.yml"` → guard passes → `--config /tmp/evil.yml` - * - * while the same text unwrapped was correctly refused. Not injectable - * today only because every Go arg builder re-checks — the user-visible - * effect was a cryptic "Could not run" instead of the clean - * `unsafe-context` skip this has a message for. - */ const comment = (body) => ({ action: "created", issue: { number: 7, pull_request: {} }, @@ -618,16 +522,6 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { }) }) -/** - * An UNADDRESSED comment reaches us only as a possible approval. It used to - * be enough to open with the word, so "Approved, but rename X" — an ordinary - * review comment on any pull request — started a live write run, with - * "but rename X" carried in as approver edits that win. - * - * Unaddressed, the comment must now BE the approval and nothing else. - * Addressed, "@browserstack go and also cover X" still works: that is - * "you can edit the list first", and they did address us. - */ describe("an unaddressed comment counts as approval only when it is exactly that", () => { const comment = (body) => plan("issue_comment", { @@ -659,12 +553,7 @@ describe("an unaddressed comment counts as approval only when it is exactly that }) }) -/** - * The result comment is edited in place, so it can only ever show the latest - * run. On a pull request where several people have asked for different jobs, - * nothing said whose request the current contents were answering. The report - * links back to the asking comment, which means carrying its URL out of here. - */ +// The result comment is edited in place, so it can only ever show the latest run. describe("the asking comment is carried through", () => { const at = "https://github.com/acme/shop/pull/412#issuecomment-99" @@ -695,17 +584,6 @@ describe("the asking comment is carried through", () => { }) }) -/** - * People address the Action as @browserstack, the name the GitHub App posts - * under, so the name they type matches the name that replies. - * - * "browserstack" also appears in text that is NOT addressed to us: email - * addresses (someone@browserstack.com), GitHub team mentions - * (@browserstack/qa), npm package scopes (@browserstack/wdio-service) and the - * App's own bot account (@browserstack[bot]). A plain "does the comment - * contain @browserstack" check would start a run for every one of those, so - * only the word on its own counts. - */ describe("the @browserstack mention", () => { const comment = (body) => ({ action: "created", @@ -733,7 +611,7 @@ describe("the @browserstack mention", () => { test("text that merely contains the name is not addressed to us", () => { for (const body of [ - "ping naimish@browserstack.com about this", + "ping someone@example.com about this", "sent to QA@BrowserStack.com", "@browserstack/qa please review", "bump @browserstack/wdio-browserstack-service to 9.1", diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 9abafaa..d07c6eb 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -32,12 +32,6 @@ describe("shouldPushWork — the approved work must survive the runner", () => { assert.equal(shouldPushWork(live).ok, true) }) - /** - * A preview run is the proposal pass. The brain refuses every write tool, - * so there should be nothing to push — but if a shell command wrote - * something anyway, pushing it would break the one promise the proposal - * makes: "nothing is written until you say go". - */ test("a preview run never pushes", () => { assert.equal(shouldPushWork({ ...live, preview: true }).ok, false) assert.match(shouldPushWork({ ...live, preview: true }).why, /preview/i) @@ -51,11 +45,6 @@ describe("shouldPushWork — the approved work must survive the runner", () => { assert.equal(shouldPushWork({ ...live, branch: "" }).ok, false) }) - /** - * The loop guard. Our own push creates a commit; if that commit triggers a - * run that pushes again, the pull request fills with commits forever. One - * generation is enough: we never push on top of our own commit. - */ test("a run triggered by our own commit does not push again", () => { assert.equal(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).ok, false) assert.match(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).why, /own commit/i) @@ -99,12 +88,6 @@ describe("splitPushable — what we refuse to commit", () => { assert.deepEqual(skip, []) }) - /** - * `git add` already honours .gitignore, so this is the second line of - * defence, for a repository that does not ignore these. A pushed .env is - * the worst outcome in this whole feature — it publishes a credential to - * the customer's own history, where deleting it does not take it back. - */ test("an environment file is never committed", () => { const { push, skip } = splitPushable([".env", "tests/a.spec.js", "config/.env.local"]) assert.deepEqual(push, ["tests/a.spec.js"]) @@ -150,12 +133,6 @@ describe("renderPushNote — the reader must see exactly what landed", () => { }) describe("needsCheckout — the run must happen on the branch it writes to", () => { - /** - * main.mjs already detaches onto the pull request's commit when it resolves - * a comment's issue number, but that fetch is advisory — it warns and - * carries on so a proposal can still be built. This is what turns "we tried - * to be on the right commit" into "we are", before anything is pushed. - */ test("a checkout sitting on another commit needs switching", () => { assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "54ceef4" }), true) }) @@ -181,24 +158,11 @@ describe("needsCheckout — the run must happen on the branch it writes to", () }) describe("shouldPushWork — the two fail-open holes a review caught", () => { - /** - * `sameRepo` was hard-coded true at the call site, on the reasoning that - * plan-run.mjs already skips forks. True today; a hard-coded security - * value is wrong the moment the code around it moves, and this is the - * second time this exact shape has appeared in this Action — the first was - * `pull_request_target` failing open on a deleted fork. - */ test("not knowing whose repository this is refuses the push", () => { assert.equal(shouldPushWork({ ...live, sameRepo: undefined }).ok, false) assert.equal(shouldPushWork({ ...live, sameRepo: "yes" }).ok, false) }) - /** - * The checkout is advisory — a run continues against whatever is there so - * the proposal still gets built. But pushing from the wrong tree would - * send the DEFAULT branch's content to the pull request's branch and - * clobber the change under review. - */ test("a tree that is not on the wanted commit never pushes", () => { assert.equal(shouldPushWork({ ...live, onWantedCommit: false }).ok, false) assert.match(shouldPushWork({ ...live, onWantedCommit: false }).why, /commit/i) @@ -210,12 +174,7 @@ describe("shouldPushWork — the two fail-open holes a review caught", () => { }) describe("shouldPushWork — never the default branch", () => { - /** - * The branch name comes from the pull request's own head ref, which its - * author controls. Same-repo plus on-the-right-commit already bound that, - * but the default branch is the one target where a mistake is not - * recoverable by closing a pull request — so it is refused by name. - */ + // The branch name comes from the pull request's own head ref, which its author controls. test("a pull request branch is fine", () => { assert.equal(shouldPushWork({ ...live, branch: "feat/x", defaultBranch: "main" }).ok, true) }) @@ -235,21 +194,7 @@ describe("shouldPushWork — never the default branch", () => { }) }) -describe("changedPaths — the bug a live run found that 345 unit tests did not", () => { - /** - * Run 40845669513. The Action wrote ten passing tests and then could not - * save them: "pathspec 'ests/pricing.spec.js' did not match any files". - * - * `git status --porcelain` puts a SPACE in the first column for an - * unstaged modification, so the line is " M tests/pricing.spec.js" — three - * characters before the path. The caller trimmed git's whole stdout, which - * removed that leading space from the FIRST LINE ONLY, leaving a two - * character prefix and a fixed slice(3) that ate the "t". - * - * Both ends are fixed: the caller no longer trims, and this parse no - * longer assumes a fixed width. The tests passed before because they all - * fed it well-formed porcelain — the corruption happened upstream. - */ +describe("changedPaths — parsing git status output", () => { test("a first line whose leading space was trimmed away still parses", () => { assert.deepEqual(changedPaths("M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) }) @@ -275,17 +220,6 @@ describe("changedPaths — the bug a live run found that 345 unit tests did not" }) describe("isSameRepo — whose branch is this, really", () => { - /** - * Run 40857884973 refused to save with "the change does not come from this - * repository", on a branch in this very repository. - * - * The caller read the head repository off the pull request object that - * main.mjs fetches — but it only fetches that for a COMMENT event, where a - * PR number has to be resolved to a head sha. On a pull_request event the - * sha is already in the payload, so nothing was fetched, the answer was - * undefined, and undefined is a refusal. The head repository was in the - * event payload the whole time. - */ test("the same repository, compared case-insensitively", () => { assert.equal(isSameRepo("Acme/Shop", "acme/shop"), true) assert.equal(isSameRepo("acme/shop", "acme/shop"), true) @@ -303,22 +237,6 @@ describe("isSameRepo — whose branch is this, really", () => { }) describe("changedPaths — paths git had to escape", () => { - /** - * With `core.quotePath` on - * (the default) git octal-escapes non-ASCII bytes, so `café.spec.ts` comes - * out as `"caf\\303\\251.spec.ts"`. Stripping the quotes without decoding - * leaves a path that does not exist, and `git add` with several pathspecs - * ABORTS ON THE FIRST MISS and stages nothing — so one accented filename - * loses every file the run produced. - * - * The call site now passes `-c core.quotePath=false`, which stops the - * escaping at source. This decodes it anyway, so a repository that forces - * the setting back on cannot reintroduce it. - * - * The old test picked the one quoted input that round-trips — a path with - * a space, which git quotes but does not escape — so the suite was green - * right beside the bug. - */ test("an octal-escaped non-ASCII path is decoded back to its real name", () => { assert.deepEqual(changedPaths('?? "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) }) @@ -343,24 +261,6 @@ describe("changedPaths — paths git had to escape", () => { }) describe("changedPaths — characters outside the basic plane", () => { - /** - * The decoder walked the string by UTF-16 code unit and ran Buffer.from on - * each one, so an astral character — an emoji — was split into its two - * surrogate halves and each half became U+FFFD: - * - * "my 🎉 file.spec.ts" -> "my \uFFFD\uFFFD file.spec.ts" - * - * Reachable BECAUSE of the other half of the fix: with - * `core.quotePath=false` git stops octal-escaping non-ASCII but still - * QUOTES a path containing a space, so an emoji in a spaced filename - * arrives raw inside quotes and goes through this branch. The result is a - * path that does not exist, `git add` aborts, and the whole approved run's - * work is lost — the original bug, reached through the setting added to - * prevent it. - * - * BMP characters like é survived, which is why the first round of tests - * passed. - */ test("an emoji in a quoted path survives", () => { assert.deepEqual(changedPaths('?? "my 🎉 file.spec.ts"'), ["my 🎉 file.spec.ts"]) }) @@ -376,20 +276,6 @@ describe("changedPaths — characters outside the basic plane", () => { }) describe("the App token has to reach git, not just gh", () => { - /** - * action.yml and the starter both tell - * customers they can delete the `permissions:` block once the App is - * installed, "because the App's installation token carries these - * permissions itself". The token was put in GH_TOKEN, which only `gh` - * reads — the push is a plain `git push` authenticating with whatever - * actions/checkout left in .git/config. - * - * So following our own advice, on an org whose default workflow token is - * read-only, made every approved run fail its push with a 403. - * - * Passed as config rather than written into the remote URL: a token in a - * URL ends up in the reflog, in error messages and in `git remote -v`. - */ const URL = "https://github.com/acme/shop.git" test("builds a basic auth header SCOPED to the push url", () => { @@ -399,12 +285,6 @@ describe("the App token has to reach git, not just gh", () => { assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") }) - /** - * `url..insteadOf` in the repository's own config — which the agent - * can write — redirects a push to another host. Verified: one line turns - * github.com into evil.test. A GLOBAL header follows it there and hands - * over the App's installation token. - */ test("the header is never global", () => { const got = pushCredentialEnv("ghs_abc123", URL) assert.notEqual(got.GIT_CONFIG_KEY_0, "http.extraheader") @@ -439,15 +319,7 @@ describe("the App token has to reach git, not just gh", () => { }) describe("our own git commands must not carry secrets or run hooks", () => { - /** - * These commands run AFTER the agent, which has had --yolo write access to - * the working tree. spawnSync with no `env` inherits everything, so an - * agent that wrote .git/hooks/pre-commit would have it executed by our - * `git commit` with TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's - * BrowserStack key in its environment. On pull_request_target the diff is - * attacker-authored, so that is one prompt injection from a key leaving - * the runner. - */ + // These commands run after the agent, which could write to the working tree. const dirty = { PATH: "/usr/bin", HOME: "/home/runner", @@ -482,9 +354,6 @@ describe("our own git commands must not carry secrets or run hooks", () => { }) test("an empty environment still disables the system and user config", () => { - // Nothing is inherited, and git is told to ignore /etc/gitconfig and - // ~/.gitconfig too, so the only configuration in play is what this - // process passes with -c. for (const got of [gitEnvFrom({}), gitEnvFrom()]) { assert.equal(got.GIT_CONFIG_NOSYSTEM, "1") assert.equal(got.GIT_CONFIG_GLOBAL, "/dev/null") @@ -497,11 +366,6 @@ describe("our own git commands must not carry secrets or run hooks", () => { } }) - /** - * Every one of these is a way the REPOSITORY can ask git to run a program - * on our behalf, and the agent can write all of them. Asserted by name so - * removing one is a failing test rather than a quiet loss. - */ test("every exec-capable config key is pinned shut", () => { const settings = GIT_SAFE_ARGS.filter((a) => a !== "-c") for (const key of [ @@ -532,17 +396,8 @@ describe("our own git commands must not carry secrets or run hooks", () => { }) }) -/** - * The agent runs with --yolo write access to the tree, and `.git/config` is - * part of the tree it can write. Two separate things go wrong there, so there - * are two separate defences. - */ describe("the repository's own git config must not steer the push", () => { test("a redirect is refused — insteadOf rewrites even an explicit URL", () => { - // Verified against real git: `git push https://github.com/acme/shop.git` - // with url."https://evil.test/".insteadOf "https://github.com/" set - // reports `failed to push some refs to 'https://evil.test/acme/shop.git'`. - // Naming the URL on the command line is NOT enough. assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ "url.https://evil.test/.insteadOf", ]) @@ -587,11 +442,6 @@ describe("the repository's own git config must not steer the push", () => { assert.deepEqual(unexpectedConfigKeys("CORE.FILEMODE"), []) }) - /** - * The point of an allow list: a key nobody has thought about yet is - * refused without anyone having to add it. git gains config keys with - * every release, and several of them run programs. - */ test("a key we have never heard of is refused by default", () => { assert.deepEqual(unexpectedConfigKeys("some.future.hook"), ["some.future.hook"]) assert.deepEqual(unexpectedConfigKeys("http.sslVerify"), ["http.sslVerify"]) @@ -605,11 +455,6 @@ describe("the repository's own git config must not steer the push", () => { assert.deepEqual(unexpectedConfigKeys("http.https://github.com/.extraheader"), []) }) - /** - * This is the half that decides whether the Action works at all. Every one - * of these is written by actions/checkout on a perfectly ordinary run, so - * refusing any of them would refuse every push we have ever made. - */ test("an ordinary checkout is NOT refused", () => { const afterCheckout = [ "core.repositoryformatversion", @@ -623,8 +468,6 @@ describe("the repository's own git config must not steer the push", () => { "branch.main.merge", "user.name", "user.email", - // safe.directory is deliberately absent: actions/checkout writes it - // to a TEMPORARY GLOBAL config, and we read local + worktree only. "gc.auto", ].join("\n") assert.deepEqual(unexpectedConfigKeys(afterCheckout), []) @@ -656,11 +499,6 @@ describe("the credential travels in the environment, not in argv", () => { }) }) -/** - * The agent writes to the runner as the same user we do, and several - * directories on the default PATH are writable by that user. A planted `git` - * earlier on PATH would be handed the installation token. - */ describe("git itself must be the real git", () => { const exists = (p) => ["/usr/bin/git", "/bin/git"].includes(p) const notWritable = () => false @@ -675,8 +513,6 @@ describe("git itself must be the real git", () => { }) test("nothing trustworthy means nothing, not a fallback to PATH", () => { - // A fallback to bare "git" would reintroduce exactly the hole this - // closes, so the caller is made to handle the absence instead. assert.equal(trustedGitPath({ exists: () => false, canWrite: notWritable }), undefined) assert.equal(trustedGitPath({ exists, canWrite: () => true }), undefined) }) @@ -687,11 +523,7 @@ describe("git itself must be the real git", () => { }) }) -/** - * Every gh call we make carries a token, the App's included. The agent can - * write to `$HOME/.local/bin`, which is on PATH ahead of the system folders, - * and to the job's shared gh settings folder. - */ +// Every gh call we make carries a token, the App's included. describe("gh itself must be the real gh, with settings nobody planted", () => { const notWritable = () => false @@ -729,8 +561,6 @@ describe("gh itself must be the real gh, with settings nobody planted", () => { }) test("no source file starts gh by its bare name", () => { - // Every call must go through runGh in main.mjs, which uses the path - // found above and a fresh settings folder. const src = new URL("../src/", import.meta.url).pathname for (const f of readdirSync(src).filter((n) => n.endsWith(".mjs"))) { const code = readFileSync(join(src, f), "utf8") @@ -743,18 +573,6 @@ describe("gh itself must be the real gh, with settings nobody planted", () => { }) }) -/** - * Found by a live run, which refused to save with a list of - * fifteen "unexpected" keys — every one of them a flag WE pass in - * GIT_SAFE_ARGS. - * - * `git config --list` includes config given on the command line with `-c`. - * So the check read our own hardening back and reported it as tampering, - * which would have blocked every push the Action ever makes. - * - * This runs real git, because the bug was in what git returns, not in how - * the listing is parsed — a fixture would have agreed with the broken code. - */ describe("the config check reads the repository, not our own flags", () => { test("-c values do not come back as unexpected keys", () => { const dir = mkdtempSync(join(tmpdir(), "tc-config-")) @@ -776,15 +594,8 @@ describe("the config check reads the repository, not our own flags", () => { }) }) -/** - * Two ways the push could carry more than the run meant to write. - */ describe("the push carries the run's test files and nothing else", () => { test("a new directory must arrive as its files, not as one entry", () => { - // `git status --porcelain` without -uall collapses a new directory to - // `?? tests/`. splitPushable then sees ONE path with one segment, its - // deny list never sees `tests/.env` or `tests/run.log`, and - // `git add tests/` commits the lot. const collapsed = changedPaths("?? tests/\n") assert.deepEqual(collapsed, ["tests/"], "this is what the old command produced") const { push } = splitPushable(collapsed) @@ -798,8 +609,6 @@ describe("the push carries the run's test files and nothing else", () => { }) test("a commit the agent made itself is not ours to push", () => { - // We push HEAD. If the agent committed, HEAD carries whatever it put in - // source, lockfiles.github/ — none of it filtered by splitPushable. assert.equal(agentCommitted("aaa111", "bbb222"), true) assert.equal(agentCommitted("aaa111", "aaa111"), false) }) @@ -810,10 +619,7 @@ describe("the push carries the run's test files and nothing else", () => { }) }) -/** - * `.github/` tells GitHub what to run, and a symlink can point anywhere on the - * runner. Neither is test work, so neither is pushed. - */ +// `.github/` tells GitHub what to run, and a symlink can point anywhere on the runner. describe("splitPushable refuses .github and symlinks", () => { test("anything under .github is skipped", () => { const { push, skip } = splitPushable([ diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 4a4d3df..1d8b016 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -32,7 +32,7 @@ import { inFlightCheckId, } from "../src/report.mjs" -/** The login this Action posts as. Identity, not account type. */ +// The login this Action posts as. const US = "test-companion[bot]" const base = { @@ -49,16 +49,11 @@ describe("renderJobSummary", () => { assert.match(md, /generate/) }) - // a preview run must be marked as one wherever it reports, or a reader - // sees a clean result and assumes the work landed. test("marks a preview run, and scopes the claim to Test Companion's own writes", () => { const md = renderJobSummary(base) assert.match(md, /Preview/i) assert.match(md, /write tools/i) - // It must NOT claim nothing at all was written. Preview deliberately - // allows `execute_command`, since that is how tests run, so a shell - // command the agent chose can still edit a file or push. The old wording - // promised a guarantee the gate does not give. + // Preview must not claim nothing at all was written: shell commands can still write. assert.doesNotMatch(md, /\bnothing was written\b/i) assert.match(md, /[Ss]hell commands.*not covered/i) }) @@ -68,8 +63,6 @@ describe("renderJobSummary", () => { assert.doesNotMatch(md, /Preview —/i) }) - // Every run leaves a readable result, including the ones that went wrong - // a run with no report at all is exactly when the reader needs the page most. test("still renders when the CLI left no report", () => { const md = renderJobSummary({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, summary: {} }) assert.match(md, /Could not run/) @@ -82,8 +75,7 @@ describe("renderJobSummary", () => { }) describe("renderComment", () => { - // The marker is how the next run finds this comment instead of adding a - // second one. It must be in every comment we ever write. + // The marker is how the next run finds this comment instead of adding a second one. test("always carries the marker that makes it findable", () => { assert.match(renderComment(base), new RegExp(COMMENT_MARKER)) }) @@ -125,9 +117,6 @@ describe("chooseCommentAction", () => { describe("which commit the check lands on", () => { // On a pull_request event GITHUB_SHA is the MERGE commit, not the branch. - // A check posted there is invisible on the pull request and cannot satisfy - // branch protection — found on the first live run, where the check existed - // but on a commit nobody looks at. test("prefers the head commit the event named over GITHUB_SHA", async () => { const { checkSha } = await import("../src/report.mjs") assert.equal(checkSha({ sha: "head123" }, {}, "merge999"), "head123") @@ -141,9 +130,6 @@ describe("which commit the check lands on", () => { }) describe("what the run cost and produced", () => { - // A report that says a run "Succeeded" and nothing else leaves the two - // questions a reviewer actually has unanswered: what did it cost me, and - // where did its work go. test("reports the cost in credits, the unit the ceiling is written in", () => { const out = renderJobSummary({ outcome: "Succeeded", summary: { credits: "13" } }) assert.match(out, /13 credits/) @@ -163,8 +149,6 @@ describe("what the run cost and produced", () => { assert.match(out, /3/) }) - // "3 saved" with no destination sends the reader hunting through Test - // Management for a folder nobody named. test("names where the cases were saved", () => { const out = renderJobSummary({ outcome: "Succeeded", @@ -173,8 +157,6 @@ describe("what the run cost and produced", () => { assert.match(out, /Regression \/ Weather/) }) - // The sharpest case: cases exist, nothing was written. Without saying so - // plainly, "12 generated" reads as "12 saved". test("spells out that nothing was written when nothing landed", () => { const out = renderJobSummary({ outcome: "Succeeded", @@ -192,8 +174,7 @@ describe("what the run cost and produced", () => { assert.match(out, /test-companion-result/) }) - // An rca or fix run generates no cases at all. A row reading "0 generated" - // on it looks like something went wrong. + // An rca or fix run generates no cases at all. test("leaves the whole section out for a run that generates nothing", () => { const out = renderJobSummary({ outcome: "Succeeded", summary: { duration: "2m" } }) assert.ok(!/generated/i.test(out), out) @@ -201,8 +182,6 @@ describe("what the run cost and produced", () => { }) describe("naming what failed", () => { - // "Tests failed" with no names makes the reviewer open the run and read the - // logs, which is the thing a pull request report exists to avoid. test("lists the failing tests by name", () => { const out = renderJobSummary({ outcome: "Tests failed", @@ -222,8 +201,7 @@ describe("naming what failed", () => { assert.match(out, /tests\/checkout\.spec\.ts:18/) }) - // A hundred names turn the check into a wall. Show the first few and say - // how many more there are. + // A hundred names turn the check into a wall. test("caps the list and says how many it left out", () => { const many = Array.from({ length: 40 }, (_, i) => ({ name: `test number ${i}` })) const out = renderJobSummary({ outcome: "Tests failed", summary: {}, failedTests: many }) @@ -238,11 +216,6 @@ describe("naming what failed", () => { }) describe("replying when we will not run", () => { - /** - * Both of these used to go only to the job summary. The person who commented - * never opens the run's logs, so from where they sat Test Companion simply - * ignored them. - */ test("someone without write access is told so, and who can ask", () => { const body = renderNotAuthorized("octocat") assert.match(body, /@octocat/) @@ -251,19 +224,11 @@ describe("replying when we will not run", () => { test("a request we cannot place gets the list of what we can do", () => { const body = renderUnknownRequest() - // `stop` left the list with the capability itself. - // See stop-removed.test.mjs. for (const phrase of ["test this PR", "explain this failure", "fix the failing tests", "re-check"]) { assert.ok(body.includes(phrase), `the list should offer "${phrase}"`) } }) - /** - * A refusal is not a run report. If it carried the marker, chooseCommentAction - * would treat it as the one tracked comment: the oldest wins, so the first - * refusal on a pull request would become the place every later run wrote its - * results — or sit there forever while results went elsewhere. - */ test("neither reply carries the run marker", () => { assert.ok(!renderNotAuthorized("octocat").includes(COMMENT_MARKER)) assert.ok(!renderUnknownRequest().includes(COMMENT_MARKER)) @@ -291,18 +256,10 @@ describe("the waiting proposal", () => { assert.equal(findProposal([proposal()], SHA, undefined, US)?.id, 10) }) - /** - * "New commits while waiting SHALL replace the proposal." Keying the - * marker to the commit is what makes that true without any sweeping — a - * proposal simply stops being found once the branch moves, so "go" on a - * stale proposal cannot approve work nobody reviewed. - */ test("a proposal for an older commit is not found", () => { assert.equal(findProposal([proposal()], "deadbee", undefined, US), undefined) }) - // Same reasoning as chooseCommentAction: a person quoting our proposal in - // their reply would otherwise be able to approve their own request. test("only a comment WE wrote counts", () => { const quoted = proposal({ user: { type: "User" } }) assert.equal(findProposal([quoted], SHA, undefined, US), undefined) @@ -331,8 +288,6 @@ describe("the waiting proposal", () => { assert.match(body, /\bgo\b/i) }) - // It must not be mistaken for the tracked run comment, for the same reason - // a refusal must not be. test("a proposal is not the tracked run comment", () => { const p = { id: 1, user: { type: "Bot", login: US }, body: renderProposal("x", SHA) } assert.deepEqual(chooseCommentAction([p], US), { action: "create" }) @@ -350,13 +305,6 @@ describe("dropping a proposal nobody answered", () => { }) const dropNotice = { id: 11, user: { type: "Bot", login: US }, body: proposalDroppedMarkerFor(SHA) } - /** - * Without this sweep, a proposal nobody replies to leaves the check on - * "in progress" forever — and anyone who made it a required check is blocked - * by a check that never resolves. That is the exact failure Release 2 avoided - * by posting the tick at the end, so in-progress check may only exist - * if something is guaranteed to end it. - */ test("an expired proposal needs dropping", () => { assert.equal(proposalNeedsDropping([proposal(25)], SHA, undefined, US), true) }) @@ -365,8 +313,6 @@ describe("dropping a proposal nobody answered", () => { assert.equal(proposalNeedsDropping([proposal(2)], SHA, undefined, US), false) }) - // The sweep runs nightly. Without this it would post a fresh check and a - // fresh comment on the same dead proposal every single night. test("one already dropped is not dropped again", () => { assert.equal(proposalNeedsDropping([proposal(25), dropNotice], SHA, undefined, US), false) }) @@ -381,16 +327,6 @@ describe("dropping a proposal nobody answered", () => { }) describe("reading a paginated gh response", () => { - /** - * `gh api --paginate --slurp` returns an ARRAY OF PAGES, not a flat array of - * items — and it wraps even when there is only one page. Verified against - * gh 2.91.0: twelve pages at per_page=2, and still `[[...]]` at per_page=100. - * - * This matters because chooseCommentAction filters on `c.user.type`. Handed - * an array of arrays it matches nothing, returns "create" every time, and - * every run posts a NEW comment — which is exactly the single-comment - * guarantee asks for, broken in the fix that was meant to protect it. - */ test("pages are flattened into items", () => { const pages = [[{ id: 1 }, { id: 2 }], [{ id: 3 }]] assert.deepEqual(flattenPages(pages), [{ id: 1 }, { id: 2 }, { id: 3 }]) @@ -400,8 +336,7 @@ describe("reading a paginated gh response", () => { assert.deepEqual(flattenPages([[{ id: 1 }]]), [{ id: 1 }]) }) - // Defensive: objects are not arrays, so flat leaves an already-flat list - // alone. That keeps this correct if gh ever changes its mind. + // Defensive: objects are not arrays, so flat leaves an already-flat list alone. test("an already-flat list is unchanged", () => { assert.deepEqual(flattenPages([{ id: 1 }]), [{ id: 1 }]) }) @@ -414,7 +349,6 @@ describe("reading a paginated gh response", () => { test("our comment is found once the pages are flattened", () => { const pages = [[{ id: 5, user: { type: "Bot", login: US }, body: `x ${COMMENT_MARKER}` }]] assert.deepEqual(chooseCommentAction(flattenPages(pages), US), { action: "update", id: 5 }) - // and is NOT found without flattening, which is the bug assert.deepEqual(chooseCommentAction(pages, US), { action: "create" }) }) }) @@ -438,8 +372,6 @@ describe("who asked for a proposal", () => { assert.equal(p?.requester, "octocat") }) - // Proposals written before the requester was recorded must keep working - // rather than becoming unapprovable. test("a marker with no requester is still found", () => { const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA)}`)], SHA, undefined, US) assert.equal(p?.id, 10) @@ -506,8 +438,6 @@ describe("accessibility and visual lines", () => { }) describe("when accessibility may fail the check", () => { - // "SHALL fail the check only when the file says so, and by default only - // violations this change introduced count." test("never by default, however many violations there are", () => { assert.equal(accessibilityFailsCheck({ accessibility_new: "9" }, { wcag: "AA" }), false) }) @@ -575,18 +505,12 @@ describe("the approved plan handed to the run", () => { assert.match(plan, /drop case two/) }) - /** - * "go" may carry edits or none. A bare "go" must not leave the run - * looking for instructions that were never given. - */ test("a bare approval carries no edits", () => { const plan = renderApprovedPlan("- case one", "go") assert.match(plan, /case one/) assert.ok(!/go/i.test(plan.split("case one")[1] ?? ""), "a bare go should not be presented as an edit") }) - // The marker is bookkeeping, not part of the plan, and leaving it in invites - // the run to treat it as an instruction. test("the proposal's own marker is stripped out", () => { const plan = renderApprovedPlan(`- case one\n\n${proposalMarkerFor(SHA, "octocat")}`, "go") assert.ok(!plan.includes("test-companion-proposal")) @@ -612,25 +536,12 @@ describe("telling an edited approval from a bare one", () => { }) describe("the proposal carries the plan", () => { - /** - * The first live run posted a proposal containing only a report header. The - * agent had designed nine test cases and verified them; none of it reached - * the comment, because renderProposal was only ever handed the job summary. - * - * A proposal with no plan in it cannot be reviewed, which makes approving it - * meaningless. - */ test("the plan is included when the run produced one", () => { const body = renderProposalBody({ summary: "## Test Companion", plan: "- case one\n- case two" }) assert.match(body, /case one/) assert.match(body, /Test Companion/) }) - // This used to assert "did not produce a plan" whenever the plan was empty. - // That is right only when the run produced NOTHING. Only `test-pr` writes a - // separate plan file; for rca, fix and verdict the proposal run is a - // preview run whose report IS the thing being approved, and telling that - // reader to "re-run once that is fixed" sent them to fix a run that worked. test("with a report and no plan file, the report is what is being approved", () => { const body = renderProposalBody({ summary: "## Test Companion", plan: "" }) assert.doesNotMatch(body, /did not produce a plan/i) @@ -664,20 +575,6 @@ describe("renderJobSummary — the saved work", () => { }) describe("findSupersededProposal — telling a stale plan from silence", () => { - /** - * Defect found live on 2026-10-29. Someone replied "go" after the branch - * had moved on, and was told: - * - * "No @browserstack mention, so nothing was asked of Test Companion." - * - * Wrong twice. They DID give a valid approval word, and the real reason is - * that their plan was for an earlier commit. The guard that drops - * unaddressed chatter fired before anything noticed there was a superseded - * proposal sitting right there. - * - * The safety behaviour was already correct — a stale plan is never run. - * This is only about saying why. - */ const proposal = (sha) => ({ user: { type: "Bot", login: US }, body: `a plan\n\n`, @@ -716,18 +613,6 @@ describe("findSupersededProposal — telling a stale plan from silence", () => { }) describe("the proposal marker carries the JOB, not just the commit", () => { - /** - * The marker used to record the commit and the requester and nothing - * else. On the GO path `intent.workflow` is - * undefined, so `plan.workflow` was never set and the CLI resolved the - * binding itself from `--event pr_comment` — MatchEvent(...)[0], whichever - * is first in the file. - * - * So on a repository with two pr_comment bindings: ask for `fix`, read a - * proposal for `fix`, reply `go`, and something else runs live with - * yolo. is "what you saw is what runs"; nothing carried the job - * across. - */ test("the marker records the workflow", () => { assert.match(proposalMarkerFor("abc1234", "someone", "fix"), /workflow=fix/) }) @@ -742,7 +627,7 @@ describe("the proposal marker carries the JOB, not just the commit", () => { assert.equal(findProposal(comments, "abc1234", undefined, US)?.workflow, "fix") }) - /** Proposals written before the marker carried a job stay approvable. */ + // Proposals written before the marker carried a job stay approvable. test("a marker with no workflow still matches, and reports none", () => { const old = [ { user: { type: "Bot", login: US }, body: "plan\n\n" }, @@ -771,7 +656,7 @@ describe("checkRequestFor — one check per commit, not three", () => { assert.deepEqual(checkRequestFor("abc", made), { method: "PATCH", path: "check-runs/99" }) }) - /** A run can check the head early and the reported commit later. */ + // A run can check the head early and the reported commit later. test("a different commit gets its own check", () => { const made = new Map([["abc", 99]]) assert.deepEqual(checkRequestFor("def", made), { method: "POST", path: "check-runs" }) @@ -783,20 +668,6 @@ describe("checkRequestFor — one check per commit, not three", () => { }) describe("the two marker readers must not drift apart", () => { - /** - * They already had. findProposal learned about `workflow=`; - * findSupersededProposal, 200 lines away, did not — and its pattern - * demanded ` -->` right after the optional `by=`, so it matched NO marker - * this branch writes. - * - * The consequence was the exact failure commit 0c2125b68f fixed: reply - * `go` after the branch moves, the superseded lookup finds nothing, and - * you are told "No @browserstack mention" again. - * - * The old tests missed it because the superseded block hand-wrote its - * fixture while the new marker tests only ever fed proposalMarkerFor - * output to findProposal. Nothing crossed the two. These do. - */ const bot = (body, id = 1) => ({ user: { type: "Bot", login: US }, id, body: `a plan\n\n${body}` }) test("a marker written by proposalMarkerFor is found by BOTH readers", () => { @@ -854,16 +725,7 @@ describe("inFlightCheckId — the sweep must finish what another run started", ( }) describe("a proposal for a job that writes no plan file", () => { - /** - * Only `test-pr` has --propose-only and --plan-file, so only it produces a - * separate plan. For `rca`, `fix` and `verdict` the proposal run is a - * PREVIEW run — it does the work and writes nothing — and its report is - * what the reviewer approves. - * - * The body said "This run did not produce a plan — there is nothing here - * to approve", which is wrong for those jobs and tells the reader to - * re-run something that worked. - */ + // Only `test-pr` has --propose-only and --plan-file, so only it produces a separate plan. test("says the report IS the plan, rather than that there is nothing to approve", () => { const out = renderProposalBody({ summary: "## Test Companion — Succeeded\n\nit went fine", plan: "" }) assert.doesNotMatch(out, /nothing here to approve/) @@ -882,16 +744,6 @@ describe("a proposal for a job that writes no plan file", () => { }) }) -/** - * Seen on a live run. Someone asked "@browserstack explain why the - * contract check is failing". The agent answered — "CI failure analysis from - * a run link is not enabled for your account yet. Please reach out to support - * to get access." — and the pull request comment showed a table saying - * "Succeeded" and not one word of it. - * - * For `rca` and `verdict` the answer is the whole product. A reply that does - * not contain the reply is not a reply. - */ describe("renderJobSummary — the answer is the report, for jobs that only answer", () => { const base = { workflow: "rca", outcome: "Succeeded", summary: { event: "pr_comment", duration: "28s" } } @@ -923,11 +775,6 @@ describe("renderJobSummary — the answer is the report, for jobs that only answ }) }) -/** - * Building `new RegExp(shaPattern)` was safe only because every caller - * remembered to escape. These pin that the escaping is now the helper's job, - * which is what makes the answer stay true. - */ describe("a proposal marker search is built from literals, never from raw input", () => { test("a sha with regex metacharacters matches itself and nothing else", () => { const nasty = "a.c*d+" @@ -942,7 +789,6 @@ describe("a proposal marker search is built from literals, never from raw input" }) test("a long run of repeats does not hang the search", () => { - // The shape ReDoS reports are about. Linear here, so it returns at once. const started = Date.now() findProposal( [{ user: { type: "Bot", login: US }, body: `` const them = (body, id = 9) => ({ @@ -1044,19 +869,6 @@ describe("only OUR OWN comments count as ours", () => { }) }) -/** - * Approving a proposal left no trace that it had been approved. Three things - * followed from that one gap: - * - * - a second `go` found the same proposal and ran the job again; - * - the proposal's in-progress check was never adopted, so it stayed pending - * beside the real run's check; - * - a day later the sweep said "that proposal went a day without a reply" - * about a proposal that had been approved and had already run. - * - * And `stop` had the same gap from the other side: the comment told people - * "say stop to drop it", and stop cancelled runs without dropping anything. - */ describe("a proposal that has been used up is not waiting any more", () => { const US = "test-companion[bot]" const SHA = "abc1234" @@ -1091,16 +903,6 @@ describe("a proposal that has been used up is not waiting any more", () => { }) }) -/** - * The consumed marker worked — a second `go` did not rerun the job. But the - * REPLY was wrong: it said "that plan was for an earlier commit", because - * with nothing waiting the code reaches for a superseded proposal, and an - * older one from before the branch moved was still in the thread. - * - * The commit had not changed. The plan had been approved and run. Telling - * someone their branch moved on when it did not sends them looking for a - * problem that is not there. - */ describe("a plan that was already used says so, rather than blaming the commit", () => { const US = "test-companion[bot]" const SHA = "97e02ad" @@ -1135,15 +937,6 @@ describe("a plan that was already used says so, rather than blaming the commit", }) }) -/** - * : now that EVERY job's answer reaches the comment, the comment can - * be too big to post. - * - * GitHub refuses a body over 65,536 characters outright — the call fails and - * the run reports nothing at all, which is worse than the missing answer this - * change set out to fix. `generate` is the realistic case: thirty test cases - * described in prose. - */ describe("renderComment keeps the body postable", () => { const view = (answer) => ({ workflow: "fix", @@ -1184,14 +977,9 @@ describe("renderComment keeps the body postable", () => { test("the table still renders above the cut answer", () => { const body = renderComment(view("word ".repeat(40_000))) assert.ok(body.includes("| Job | fix |")) - // The heading carries a mark in front of the words now, so match the - // shape rather than the exact string — see outcome-emoji.test.mjs. assert.match(body, /## Test Companion — .*Succeeded/u) }) - // Not the answer's fault. The listed failures are capped, but a test NAME - // comes from the run's own output and nothing bounds it, so the guarantee - // has to hold without an answer to cut. test("a body that is too big even without the answer is still postable", () => { const failedTests = [{ name: "deeply/nested/".repeat(6000) + "test_name" }] const body = renderComment({ ...view(undefined), failedTests }) @@ -1207,17 +995,6 @@ describe("renderComment keeps the body postable", () => { }) }) -/** - * The answer and the failing-test names are the only text in the comment that - * comes from OUTSIDE: agent prose quoting build logs and repository files, and - * test names out of the run's own report. On a fork's pull request all of that - * is attacker-influenceable. - * - * Markers are how this Action finds its own comment and tracks whether a - * proposal is still open, and they are HTML comments. One arriving inside that - * borrowed text — planted, or quoted from a log by accident — gets read back - * as our own control data. - */ describe("renderComment does not let borrowed text forge a marker", () => { const view = (over) => ({ workflow: "rca", @@ -1261,7 +1038,6 @@ describe("renderComment does not let borrowed text forge a marker", () => { }) // Every field of the report is borrowed, not just the two obvious ones. - // `line` sat beside `file` and `name` and went in raw (review finding). test("a line number cannot carry a marker, and is a number or nothing", () => { const body = renderComment(view({ failedTests: [{ name: "t", file: "a.js", line: `1 --> ${COMMENT_MARKER}` }] })) assert.equal(body.split(COMMENT_MARKER).length - 1, 1) @@ -1292,9 +1068,7 @@ describe("renderComment does not let borrowed text forge a marker", () => { }) } - // Neutralising makes the text LONGER, so the size cap has to measure what - // actually goes out. Measuring the raw answer let one full of markers - // overrun the limit and the comment would not post at all. + // Neutralising makes the text longer, so the size cap measures the final text. test("an answer full of markers still fits once neutralised", () => { const body = renderComment(view({ answer: "" const spent = "" @@ -1466,10 +1209,6 @@ describe("renderComment neutralises markers in every borrowed field", () => { }) }) -/** - * The plan in a proposal is written by the agent. A hidden marker in it, above - * ours, used to be the one a `go` read, so it could choose which job ran. - */ describe("a plan cannot choose which job its approval runs", () => { const SHA = "abc1234" const forged = "Looks fine.\n\nMore text." @@ -1485,7 +1224,7 @@ describe("a plan cannot choose which job its approval runs", () => { }) test("our own marker, the last one, is the one read", () => { - // Even if a planted one got through unescaped, it sits ABOVE ours. + // Even if a planted one got through unescaped, it sits above ours. const comment = { id: 1, user: { type: "Bot", login: US }, diff --git a/test-companion/test/runs.test.mjs b/test-companion/test/runs.test.mjs index 7721237..6a9e155 100644 --- a/test-companion/test/runs.test.mjs +++ b/test-companion/test/runs.test.mjs @@ -19,12 +19,6 @@ describe("chooseRunsToCancel", () => { assert.deepEqual(chooseRunsToCancel([inProgress(1, { status: "queued" })], { selfRunId: "99", headSha: "abc123" }), [1]) }) - /** - * The one that would hurt. The run handling the "stop" comment is itself an - * in-flight run on that commit, so a naive filter cancels the very job doing - * the cancelling — and the person is told nothing, because the process that - * owed them a reply is gone. - */ test("never cancels itself", () => { assert.deepEqual(chooseRunsToCancel([inProgress(99)], { selfRunId: "99", headSha: "abc123" }), []) }) @@ -43,10 +37,7 @@ describe("chooseRunsToCancel", () => { assert.deepEqual(chooseRunsToCancel([inProgress(1, { head_sha: "deadbee" })], { selfRunId: "99", headSha: "abc123" }), []) }) - /** - * "Stop" means stop Test Companion, not stop the repository's own build. A - * comment must never be a way to cancel someone else's CI. - */ + // "Stop" means stop Test Companion, not stop the repository's own build. test("leaves other workflows alone when we know our own name", () => { const theirs = inProgress(1, { name: "Build and deploy" }) assert.deepEqual(chooseRunsToCancel([theirs], { selfRunId: "99", headSha: "abc123", workflowName: "Test Companion" }), []) @@ -64,23 +55,6 @@ describe("chooseRunsToCancel", () => { }) describe("runIdsFromChecks — finding OUR in-flight runs on a commit", () => { - /** - * The defect this replaces. cancelRunsOn asked GitHub for - * `actions/runs?head_sha=`, but a run started by - * an ISSUE_COMMENT carries the DEFAULT BRANCH's sha — that is what GitHub - * checks out for that event. The query matched nothing, so `stop` replied - * "nothing is running" with a run plainly in flight, and closing a pull - * request did not stop its run either. Both verified live on 2026-10-29. - * - * A run's own `pull_requests` field is NOT the answer: for an - * issue_comment run it reports the pull request whose HEAD is the - * checked-out branch, which is a different pull request entirely. Observed: - * a run for reporting number 18. - * - * So the handle is our own check run. It is posted on the pull request's - * head commit, it carries the run id in external_id, and nobody else - * writes it. - */ const check = (id, status, name = "Test Companion") => ({ name, status, external_id: String(id) }) test("returns the run ids of our in-flight checks", () => { @@ -97,7 +71,6 @@ describe("runIdsFromChecks — finding OUR in-flight runs on a commit", () => { assert.deepEqual(runIdsFromChecks([check(111, "completed")], { selfRunId: "999" }), []) }) - /** "Stop" means stop Test Companion, never the repository's own build. */ test("somebody else's check is left alone", () => { assert.deepEqual(runIdsFromChecks([check(111, "in_progress", "CI / build")], { selfRunId: "999" }), []) }) @@ -118,17 +91,7 @@ describe("runIdsFromChecks — finding OUR in-flight runs on a commit", () => { }) describe("latestFailedRun — which build 'explain the failure' means", () => { - /** - * `rca`, `fix` and `verdict` need to know WHICH build failed. On a - * build_failed trigger GitHub hands us the run and the Action passes - * `run.html_url` straight through. From a pull request COMMENT nobody - * hands us one, so those three failed on the first comment with "Could - * not run" — while CAPABILITIES advertises all three by name. - * - * Nothing about that needed deciding: the same answer is a lookup. The - * newest failed workflow run on the pull request's head commit is the - * build a person means when they say "explain the failure". - */ + // `rca`, `fix` and `verdict` need to know which build failed. const run = (id, conclusion, name = "CI", status = "completed") => ({ id, name, @@ -148,7 +111,6 @@ describe("latestFailedRun — which build 'explain the failure' means", () => { assert.equal(latestFailedRun(undefined, {}), undefined) }) - /** Explaining our OWN failure is a mirror, not an answer. */ test("never picks a Test Companion run", () => { const got = latestFailedRun([run(9, "failure", "Test Companion"), run(3, "failure", "CI")], { selfWorkflowName: "Test Companion", @@ -161,7 +123,6 @@ describe("latestFailedRun — which build 'explain the failure' means", () => { assert.equal(got, "https://github.com/acme/shop/actions/runs/3") }) - /** A run still going has not failed yet. */ test("ignores runs that have not finished", () => { const got = latestFailedRun([run(9, null, "CI", "in_progress"), run(3, "failure")], {}) assert.equal(got, "https://github.com/acme/shop/actions/runs/3") diff --git a/test-companion/test/sandbox-policy.test.mjs b/test-companion/test/sandbox-policy.test.mjs index ea185fa..723cae2 100644 --- a/test-companion/test/sandbox-policy.test.mjs +++ b/test-companion/test/sandbox-policy.test.mjs @@ -18,10 +18,7 @@ import { startRelay } from "../src/relay.mjs" const layout = sandboxLayout() const q = (query) => JSON.stringify({ query, variables: {} }) -/** - * The agent's environment points at its own folders, never ours. Ours hold a - * signed-in gh and the job's temp files. - */ +// The agent's environment points at its own folders, never ours. describe("the agent's environment", () => { const ours = { PATH: "/home/runner/.local/bin:/opt/hostedtoolcache/node/22/x64/bin:/usr/bin", @@ -55,10 +52,7 @@ describe("the agent's environment", () => { }) }) -/** - * sudo logs the command line it runs, and on a self-hosted runner that log - * outlives the job. Nothing secret may be on it. - */ +// sudo logs the command line it runs, and on a self-hosted runner that log outlives the job. describe("the command that starts the agent", () => { const cmd = asAgentCommand({ node: "/usr/local/bin/node", @@ -161,11 +155,6 @@ describe("telling a GraphQL read from a write", () => { } }) - /** - * Found in review: strings were removed before comments, so a `"` inside a - * comment opened a fake string that swallowed the next line, and a - * mutation written there was passed on with the real token. - */ test("a quote inside a comment cannot hide a mutation", () => { const bypass = JSON.stringify({ query: 'query Q { viewer { login } } # "\nmutation M { addComment(input: {}) { clientMutationId } } # "', @@ -202,10 +191,7 @@ describe("telling a GraphQL read from a write", () => { }) }) -/** - * The agent's run writes this file and the agent can change it, so nothing in - * it is taken on trust. - */ +// The agent's run writes this file and the agent can change it, so nothing in it is taken on trust. describe("a pull request handed back by the CLI", () => { const sha = "a1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d0" const good = { workflow: "fix", sha, branch: `tc/fix/${sha}`, base: "main", title: "tc fix", body: "b" } @@ -275,11 +261,6 @@ describe("reading what `tc workflow claim` printed", () => { }) }) -/** - * The relay end to end, with GitHub replaced by a stub: the agent's request - * comes in over the socket with a placeholder token, and only allowed ones - * go out, carrying the real pass. - */ describe("the relay, over its socket", () => { const socketPath = `/tmp/tc-relay-test-${process.pid}.sock` const send = (method, path, body, host = "api.github.com") => diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index 8bcd92c..18dfc98 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -12,8 +12,6 @@ import { import { TC_EVENTS } from "../src/plan-run.mjs" describe("pipelineConfigPath — the same file the CLI would read", () => { - // Checking a different path than the CLI would is worse than not checking: - // the Action would report "no settings file" for a repo that has one. test("defaults to the repository root file", () => { assert.equal(pipelineConfigPath("", {}), "testcompanion-pipeline.yml") }) @@ -32,14 +30,9 @@ describe("pipelineConfigPath — the same file the CLI would read", () => { }) describe("no pipeline file", () => { - // Installing the Action and getting a red build with "open - // testcompanion-pipeline.yml: no such file or directory" is the worst - // possible first run.: a clean pass, and the file they need. test("the report says plainly that nothing is configured yet", () => { const out = renderNoPipelineReport("testcompanion-pipeline.yml") assert.match(out, /testcompanion-pipeline\.yml/) - // Only the prose. The starter itself legitimately mentions failed - // builds, since explaining one is a job you can bind. const prose = out.split("```")[0] assert.ok(!/error|failed|cannot|could not/i.test(prose), prose) }) @@ -54,8 +47,6 @@ describe("no pipeline file", () => { assert.match(STARTER_PIPELINE, /workflow:\s*\S/) }) - // Every `on:` in the starter must be a name the CLI's matcher knows, or the - // file we hand people silently never fires. test("every event the starter binds is a real pipeline event", () => { const known = new Set(Object.values(TC_EVENTS)) const bound = [...STARTER_PIPELINE.matchAll(/^\s*on:\s*(\S+)/gm)].map((m) => m[1]) @@ -63,20 +54,12 @@ describe("no pipeline file", () => { for (const e of bound) assert.ok(known.has(e), `${e} is not a pipeline event`) }) - // It is handed to someone who has not run Test Companion before. Starting - // them on a mode that writes to their repository is the wrong default. test("the starter starts in preview, so a first run writes nothing", () => { assert.match(STARTER_PIPELINE, /mode:\s*preview/) }) }) describe("which branch the settings file is read from", () => { - /** - * The safety fix. Read from the branch under test, a pull request can edit - * its own rules and be obeyed: switch itself from preview to live, delete its - * own spending limit, or name its own Test Management destination.: - * read from the default branch, so an edit takes effect only once it merges. - */ test("the repository's default branch, not the branch under test", () => { assert.equal(settingsRef({ repository: { default_branch: "trunk" } }, {}), "trunk") }) @@ -85,9 +68,6 @@ describe("which branch the settings file is read from", () => { assert.equal(settingsRef({}, { GITHUB_DEFAULT_BRANCH: "develop" }), "develop") }) - // Guessing "main" is better than guessing the pull request's branch: the - // worst case is "no settings file", which is a clean skip, not a run that - // obeys rules the pull request wrote for itself. test("and main when there is nothing to go on", () => { assert.equal(settingsRef({}, {}), "main") }) @@ -99,18 +79,6 @@ describe("which branch the settings file is read from", () => { }) describe("the starter file matches the install defaults", () => { - /** - * "'Test this PR' started by comment, review-before-run, live; every - * other binding in preview." - * - * Live AND gated is the safe pairing, not a contradiction: a gated request - * runs in preview to produce its proposal, and only an approved run writes. - * Before the gate existed this binding had to be preview, which meant a - * repository that copied the starter could never actually write anything. - */ - // The whole point of Release 3. `test` runs the project's EXISTING suite - // and never reads the diff, so a repository copying the starter got "run - // everything" when it asked Test Companion to test a pull request. test("the comment binding runs test-pr, not test", () => { const onRequest = STARTER_PIPELINE.split("- name:").find((b) => b.includes("on: pr_comment")) assert.match(onRequest, /workflow:\s*test-pr/) @@ -169,38 +137,11 @@ describe("the starter reflects the detected framework", () => { describe("the starter workflow points somewhere that exists", () => { const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") - /** - * The starter is the ONE file we can never update: once forty teams have - * merged it, changing it costs forty pull requests. So the path in it has - * to be right the first time. - * - * It said `browserstack/test-companion-action@v1` — a repository that does - * not exist. The action ships from `browserstack/github-actions`, where the - * other BrowserStack actions live, as a sub-directory. - */ test("uses the published action path", () => { assert.match(starter, /uses: browserstack\/github-actions\/test-companion@v\d/) assert.doesNotMatch(starter, /browserstack\/test-companion-action/) }) - /** - * Seen live on 2026-10-29: tc 1.1.9 shipped mid-session and every unpinned - * run self-upgraded, losing features the Action depends on. An unpinned - * customer gets a download in the middle of their pipeline on release day. - */ - /** - * The test this replaces checked that a version was pinned and said nothing - * about the checksum — so it PINNED THE BROKEN STATE. install.sh refuses a - * pinned install with no checksum ("refusing an unverified pinned - * install"), the step runs under `set -euo pipefail`, and every repository - * that copied the starter would have died on its first run with no comment - * and no check. - * - * The rule is a PAIR: an ACTIVE `cli-version:` requires an active - * `cli-sha256:`. Commented-out guidance is fine and is how the starter now - * presents it, since a checksum cannot be written for a version that does - * not exist yet. - */ const activeLines = starter .split("\n") .filter((l) => !l.trim().startsWith("#")) @@ -224,16 +165,6 @@ describe("the starter workflow points somewhere that exists", () => { }) }) -/** - * A closed pull request is a CONTROL event: it exists to act on a run that is - * already going. Sharing a concurrency group with that run, with - * `cancel-in-progress: false`, meant it queued behind the very thing it was - * sent to end and only ran once that had finished on its own. - * - * `stop` used to be the other one. It was removed with the feature; the lane - * stays for the closed pull request, which did not go anywhere. See - * stop-removed.test.mjs. - */ describe("control events do not queue behind the run they act on", () => { const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) @@ -254,13 +185,6 @@ describe("control events do not queue behind the run they act on", () => { }) }) -/** - * Every comment on every pull request used to boot a - * runner, cloned the repository, installed the CLI and signed in to - * BrowserStack — and then decided it had not been addressed. On a busy - * repository that is most of the runs, and each one also joined the - * concurrency group. - */ describe("a comment that is not for us does not start a job", () => { const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") const job = starter.slice(starter.indexOf("jobs:"), starter.indexOf("steps:")) @@ -278,8 +202,6 @@ describe("a comment that is not for us does not start a job", () => { test("every other event is untouched, so pushes and schedules still run", () => { // The filter opens with the non-comment case for exactly this reason. - // Compare against the condition itself, not the first "@browserstack": - // the comment above the filter mentions the name too. const mentionCheck = job.indexOf("contains(github.event.comment.body, '@browserstack')") assert.ok(mentionCheck > 0, "the mention check is in the filter") assert.ok(job.indexOf("!= 'issue_comment'") < mentionCheck) diff --git a/test-companion/test/stop-removed.test.mjs b/test-companion/test/stop-removed.test.mjs index d535e43..7605840 100644 --- a/test-companion/test/stop-removed.test.mjs +++ b/test-companion/test/stop-removed.test.mjs @@ -4,18 +4,6 @@ import { readFileSync } from "node:fs" import { readIntent, INTENT, CAPABILITIES } from "../src/intent.mjs" import { renderProposal, renderStopRemoved, renderUnknownRequest } from "../src/report.mjs" -/** - * `stop` is gone. - * - * It did two things: cancelled a run in flight, and dropped a plan waiting for - * approval. Neither survives. GitHub's own cancel button ends a run, and a plan - * nobody answers expires by itself after a day. - * - * The WORD is still recognised, because proposals already sitting in live - * threads tell people to say it. Answering those with "that did not name - * anything I can do" would be true and useless. - */ - describe("stop no longer does anything", () => { test("the word is still understood", () => { assert.equal(readIntent("stop").kind, INTENT.STOP) @@ -32,15 +20,13 @@ describe("stop no longer does anything", () => { test("the reply says what to do instead", () => { const said = renderStopRemoved() assert.match(said, /cancel/i) - // Both halves of what it used to do, so nobody is left guessing about - // the plan they were trying to drop. assert.match(said, /24 hours|a day/i) }) test("a proposal no longer tells people to say it", () => { const body = renderProposal("the plan", "abc123", "dev", "fix") assert.doesNotMatch(body, /\bstop\b/i) - // What a proposal DOES still say must survive the edit. + // What a proposal does still say must survive the edit. assert.match(body, /\bgo\b/) }) }) @@ -49,32 +35,14 @@ describe("the starter workflow keeps the control lane for closed pull requests", const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) - /** - * Taking `stop` out of its own lane would put it in - * the WORK group, which has `cancel-in-progress: false` and keeps only one - * pending run — so a `stop` comment could evict a queued `go`, and all it - * does now is reply. The lane is cheap; the eviction is not. - */ test("a stop comment keeps its own lane, so it cannot evict a queued run", () => { assert.match(group, /startsWith\(github\.event\.comment\.body, '@browserstack stop'\)/) }) - /** - * It used to be `contains(body, 'stop')`, so "@browserstack test the stop - * button" took the lane too and ran beside the run already going on that - * pull request, which breaks one run per pull request. - */ test("only a comment that starts with the stop request takes the lane", () => { assert.doesNotMatch(group, /contains\(github\.event\.comment\.body, 'stop'\)/) }) - /** - * The feedback said the `-control` suffix "exists only so a stop comment - * lands in a separate lane". It does not: a CLOSED pull request is a - * control event too, and cancelling its run is, which stays. Removing - * the suffix outright would put that cancel back in the queue behind the - * very run it is sent to end. - */ test("a closed pull request still gets one", () => { assert.match(group, /github\.event\.action == 'closed'/) assert.match(group, /-control/) diff --git a/test-companion/test/ticket.test.mjs b/test-companion/test/ticket.test.mjs index 4ccdb2e..03316da 100644 --- a/test-companion/test/ticket.test.mjs +++ b/test-companion/test/ticket.test.mjs @@ -19,11 +19,6 @@ describe("findTicket", () => { assert.equal(findTicket({ title: "SHOP-9: the real one", branch: "feat/SHOP-221-old" }), "SHOP-9") }) - /** - * The trap. A Jira key and a standards reference are the same shape, so a - * naive pattern turns "upgrade to UTF-8" into a run that tests against - * ticket UTF-8 and reports it confidently in the first comment. - */ test("a standards reference is not a ticket", () => { for (const text of [ "switch to UTF-8", @@ -59,9 +54,6 @@ describe("findTicket", () => { assert.equal(findTicket({ title: "SHOP-221 and SHOP-222" }), "SHOP-221") }) - // The lookbehind makes the match start at the X, so this reads as the - // project XSHOP — which is a perfectly good key — rather than quietly - // slicing out of the middle of a word. test("a longer project key is read whole, not sliced", () => { assert.equal(findTicket({ title: "XSHOP-221" }), "XSHOP-221") })