diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..de2daac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# The image needs only the package source, its build metadata and the entrypoint +# script. Keep dist/: Dockerfile.preview copies the preview wheel from it. +.git +.github +.context +.venv +.venv-test +venv +**/__pycache__ +**/*.pyc +.pytest_cache +.coverage* +htmlcov +tests +benchmarks +docs +examples +instructions +workflows +session.md diff --git a/.github/workflows/e2e-image.yml b/.github/workflows/e2e-image.yml new file mode 100644 index 0000000..1daeb22 --- /dev/null +++ b/.github/workflows/e2e-image.yml @@ -0,0 +1,120 @@ +name: E2E Image + +# Runs reachability inside the Docker image built from this commit, so a missing +# or broken toolchain fails here rather than in a user's pipeline. e2e-test.yml +# covers reachability run directly on the runner. +on: + pull_request: + paths: + - "Dockerfile" + - ".dockerignore" + - "scripts/docker-entrypoint.sh" + - "socketsecurity/core/tools/**" + - "tests/e2e/**" + - ".github/workflows/e2e-image.yml" + push: + branches: [main] + paths: + - "Dockerfile" + - ".dockerignore" + - "scripts/docker-entrypoint.sh" + - "socketsecurity/core/tools/**" + - "tests/e2e/**" + - ".github/workflows/e2e-image.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + reachability: + # Needs SOCKET_CLI_API_TOKEN, which fork and Dependabot PRs don't get. + if: >- + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) && + github.event.pull_request.user.login != 'dependabot[bot]' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + fixture: [simple-npm, simple-cargo] + name: e2e-image-reachability (${{ matrix.fixture }}) + env: + IMAGE: socket-python-cli:e2e + FIXTURE: tests/e2e/fixtures/${{ matrix.fixture }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build the image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + load: true + tags: ${{ env.IMAGE }} + build-args: | + USE_LOCAL_INSTALL=true + # Read-only: reuse the release build's toolchain layers without + # writing this commit's layers back. + cache-from: type=registry,ref=socketdev/cli:buildcache + + - name: Run reachability in the image + id: run-cli + env: + SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} + run: | + set -o pipefail + # The checkout belongs to the runner user and the container runs as + # root, so git needs the mount marked safe to read repository details. + # The named volume keeps the downloaded reachability engine across + # retries. + in_image() { + docker run --rm -v "$PWD:/repo" -w /repo \ + -v socket-cache:/root/.cache/socket \ + -e SOCKET_SECURITY_API_KEY \ + -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/repo \ + "$@" + } + # Same retry policy as e2e-test.yml's reachability job: the backend + # intermittently returns nothing to analyze while the CLI reports + # success, so retry while the facts file has no alerted components + # and let validation classify a result that stays empty. + max_attempts=3 + attempt=1 + while :; do + in_image "$IMAGE" --target-path "$FIXTURE" --reach --disable-blocking --enable-debug 2>&1 | tee /tmp/e2e-output.log + if in_image --entrypoint python "$IMAGE" tests/e2e/validate-image-reachability.py "$FIXTURE" --probe; then + break + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "::warning title=e2e-image incomplete results::no alerted components after ${max_attempts} attempts; letting validation classify the result" + echo "diagnostics=true" >> "$GITHUB_OUTPUT" + break + fi + echo "::warning title=e2e-image transient retry::attempt ${attempt} produced no alerted components (suspected backend transient); retrying" + attempt=$((attempt+1)) + sleep 30 + done + + - name: Validate results + run: >- + docker run --rm -v "$PWD:/repo" -v /tmp/e2e-output.log:/tmp/e2e-output.log:ro -w /repo + --entrypoint python "$IMAGE" + tests/e2e/validate-image-reachability.py "$FIXTURE" --log /tmp/e2e-output.log + + - name: Upload diagnostics + if: failure() || steps.run-cli.outputs.diagnostics == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-image-${{ matrix.fixture }}-diagnostics-attempt${{ github.run_attempt }} + path: | + /tmp/e2e-output.log + ${{ env.FIXTURE }}/.socket.facts.json + if-no-files-found: ignore + include-hidden-files: true + retention-days: 14 diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 1a681e1..bfde20a 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -13,6 +13,7 @@ on: - "tests/core/**/*.py" - "pyproject.toml" - "uv.lock" + - "Dockerfile" - ".github/workflows/python-tests.yml" # Unfiltered: a required check behind a paths filter never reports on a # pull request that misses the filter, which blocks the merge. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef33417..15dbc5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -117,8 +117,13 @@ jobs: with: push: true platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max + # A registry cache rather than type=gha: Actions caches are scoped to + # the ref that wrote them, and each release builds on its own tag, so + # no release can read the one before it. Reusing cached layers is + # what keeps their digests stable, so users upgrading pull only the + # layers that changed. + cache-from: type=registry,ref=socketdev/cli:buildcache + cache-to: type=registry,ref=socketdev/cli:buildcache,mode=max,image-manifest=true,oci-mediatypes=true tags: | socketdev/cli:latest socketdev/cli:${{ env.VERSION }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bfe78cc..1efdc08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2.11.12 + +### Changed: smaller Docker image, and upgrades reuse its toolchain layers + +- Upgrading `socketdev/cli` from one release to the next now downloads only the + layers that changed. Previously each release rebuilt every layer, so an upgrade + downloaded nearly the whole image. +- Rust is installed with rustup's minimal profile (`rustc`, `cargo`, `rust-std`). + The image no longer includes Rust's offline HTML documentation, `rustfmt`, or + `clippy`. +- Removed `strace` from the image, and the npm and pip download caches. Run + `apk add strace` in a derived image if you need it. + ## 2.11.11 ### Changed: use lowercase Python CLI User-Agent diff --git a/Dockerfile b/Dockerfile index 4450c6a..23654b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,20 +11,13 @@ ARG GO_VERSION=system ARG JAVA_VERSION=17 ARG DOTNET_VERSION=8 -# CLI and SDK arguments -ARG CLI_VERSION -ARG SDK_VERSION -ARG PIP_INDEX_URL=https://pypi.org/simple -ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple -ARG USE_LOCAL_INSTALL=false - # Install base packages first # ca-certificates must be 20261020-r0 or newer, which drops the roots the # compromised eMudhra intermediates chain to. The base image ships an older # one, and `apk add` does not upgrade installed packages, so upgrade it here. -RUN apk update && apk add --no-cache \ +RUN apk add --no-cache \ git curl wget \ - ruby ruby-dev build-base strace && \ + ruby ruby-dev build-base && \ apk upgrade --no-cache ca-certificates && \ [ "$(apk version -t "$(apk list -I ca-certificates | awk '{print $1}' | sed 's/^ca-certificates-//')" 20261020-r0)" != "<" ] @@ -104,11 +97,14 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ fi # Install additional tools +# Reachability only runs cargo, so install the minimal Rust profile (rustc, +# cargo, rust-std). The default profile adds about 900 MB of offline HTML docs. RUN npm install socket -g && \ + npm cache clean --force && \ gem install bundler && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal && \ . ~/.cargo/env && \ - rustup component add rustfmt clippy + cargo --version # Set environment paths ENV PATH="/usr/local/go/bin:/usr/lib/go/bin:/root/.cargo/bin:${PATH}" @@ -148,10 +144,21 @@ COPY pyproject.toml uv.lock /tmp/socket-cli-lock/ # at TestPyPI don't leak into the locked dependency install via pip's env vars. RUN uv export --directory /tmp/socket-cli-lock --frozen --no-dev --no-emit-project \ --format requirements-txt -o /tmp/socket-cli-lock/requirements.txt && \ - pip install --require-hashes --no-deps \ + pip install --no-cache-dir --require-hashes --no-deps \ --index-url https://pypi.org/simple --extra-index-url https://pypi.org/simple \ -r /tmp/socket-cli-lock/requirements.txt +# CLI and SDK arguments +# Declared here rather than at the top: every RUN after an ARG is cache-keyed on +# its value, and CLI_VERSION changes on every release. Declared earlier, it +# would rebuild every toolchain layer above, so each release would be a full +# re-pull for anyone upgrading. +ARG CLI_VERSION +ARG SDK_VERSION +ARG PIP_INDEX_URL=https://pypi.org/simple +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple +ARG USE_LOCAL_INSTALL=false + # Install CLI based on build mode RUN if [ "$USE_LOCAL_INSTALL" = "true" ]; then \ echo "Using local development install"; \ @@ -159,7 +166,7 @@ RUN if [ "$USE_LOCAL_INSTALL" = "true" ]; then \ cli_installed=false; \ for i in $(seq 1 10); do \ echo "Attempt $i/10: Installing socketsecurity==$CLI_VERSION"; \ - if pip install --no-deps --index-url ${PIP_INDEX_URL} --extra-index-url ${PIP_EXTRA_INDEX_URL} socketsecurity==$CLI_VERSION; then \ + if pip install --no-cache-dir --no-deps --index-url ${PIP_INDEX_URL} --extra-index-url ${PIP_EXTRA_INDEX_URL} socketsecurity==$CLI_VERSION; then \ cli_installed=true; \ break; \ fi; \ @@ -174,7 +181,7 @@ RUN if [ "$USE_LOCAL_INSTALL" = "true" ]; then \ fi; \ pip check || exit 1; \ if [ ! -z "$SDK_VERSION" ]; then \ - pip install --index-url ${PIP_INDEX_URL} --extra-index-url ${PIP_EXTRA_INDEX_URL} socketdev==${SDK_VERSION}; \ + pip install --no-cache-dir --index-url ${PIP_INDEX_URL} --extra-index-url ${PIP_EXTRA_INDEX_URL} socketdev==${SDK_VERSION}; \ fi; \ fi @@ -182,7 +189,7 @@ RUN if [ "$USE_LOCAL_INSTALL" = "true" ]; then \ COPY . /app WORKDIR /app RUN if [ "$USE_LOCAL_INSTALL" = "true" ]; then \ - pip install --no-deps -e . && pip check; \ + pip install --no-cache-dir --no-deps -e . && pip check; \ fi # Create workspace directory with proper permissions diff --git a/pyproject.toml b/pyproject.toml index 966fee1..183008b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.11.11" +version = "2.11.12" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 4a07834..590de20 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,4 +1,4 @@ __author__ = 'socket.dev' -__version__ = '2.11.11' +__version__ = '2.11.12' USER_AGENT_NAME = 'socket-python-cli' USER_AGENT = f'{USER_AGENT_NAME}/{__version__}' diff --git a/tests/e2e/fixtures/simple-cargo/Cargo.lock b/tests/e2e/fixtures/simple-cargo/Cargo.lock new file mode 100644 index 0000000..68595f5 --- /dev/null +++ b/tests/e2e/fixtures/simple-cargo/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "simple-cargo" +version = "0.1.0" +dependencies = [ + "smallvec", +] + +[[package]] +name = "smallvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a55ca5f3b68e41c979bf8c46a6f1da892ca4db8f94023ce0bd32407573b1ac0" diff --git a/tests/e2e/fixtures/simple-cargo/Cargo.toml b/tests/e2e/fixtures/simple-cargo/Cargo.toml new file mode 100644 index 0000000..4508814 --- /dev/null +++ b/tests/e2e/fixtures/simple-cargo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "simple-cargo" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +# Pinned to a release affected by GHSA-43w2-9j62-hq99 (insert_many buffer +# overflow), so reachability has a known vulnerability to analyze. +smallvec = "=1.6.0" diff --git a/tests/e2e/fixtures/simple-cargo/src/main.rs b/tests/e2e/fixtures/simple-cargo/src/main.rs new file mode 100644 index 0000000..3ef8f23 --- /dev/null +++ b/tests/e2e/fixtures/simple-cargo/src/main.rs @@ -0,0 +1,7 @@ +use smallvec::SmallVec; + +fn main() { + let mut values: SmallVec<[u8; 4]> = SmallVec::new(); + values.insert_many(0, vec![1, 2, 3]); + println!("{:?}", values); +} diff --git a/tests/e2e/validate-image-reachability.py b/tests/e2e/validate-image-reachability.py new file mode 100644 index 0000000..cfb6843 --- /dev/null +++ b/tests/e2e/validate-image-reachability.py @@ -0,0 +1,88 @@ +"""Checks a ``--reach`` run made inside the Docker image. + +Run with the image's Python, from the repository root: + + validate-image-reachability.py --probe + Exits 0 when the fixture's facts file has alerted components. The workflow + retries a run that fails this, because the reachability backend intermittently + returns nothing to analyze. + + validate-image-reachability.py --log + Exits 0 when Coana analyzed at least one vulnerability. That needs the + ecosystem's toolchain in the image to work, which is what this test is for. + Whether a given vulnerability is reachable depends on Coana and the backend, + not on the image, so the verdict itself is not asserted. +""" + +import argparse +import sys +from collections import Counter +from pathlib import Path + +from socketsecurity.core.alert_selection import load_components_with_alerts + +FACTS_FILE = ".socket.facts.json" +ANALYZED = ("reachable", "unreachable") +# Coana's log line when the backend returned no vulnerable components to analyze. +NOTHING_TO_ANALYZE = "Found 0 projects across 0 ecosystems to analyze" + + +def alert_states(fixture: str) -> Counter: + components = load_components_with_alerts(fixture, FACTS_FILE) or [] + return Counter( + alert["props"]["reachability"] + for component in components + for alert in component["alerts"] + ) + + +def validate(fixture: str, log: str) -> int: + if "Reachability analysis completed successfully" not in log: + print("FAIL: reachability analysis did not complete") + return 1 + if not (Path(fixture) / FACTS_FILE).is_file(): + print(f"FAIL: no {FACTS_FILE} in {fixture}") + return 1 + + states = alert_states(fixture) + if not states: + if NOTHING_TO_ANALYZE in log: + print( + f"::warning title=e2e-image inconclusive::the backend returned nothing to analyze for {fixture} " + "after retries, so the image's toolchain was not exercised" + ) + return 0 + print( + f"FAIL: {FACTS_FILE} has no alerted components, and the log lacks the backend's nothing-to-analyze line" + ) + return 1 + + print(f"Reachability states for {fixture}: {dict(states)}") + if not any(states[state] for state in ANALYZED): + print( + "FAIL: Coana analyzed no vulnerability; check the toolchain errors in the CLI output" + ) + return 1 + if states["error"]: + print( + f"::warning title=e2e-image analysis errors::{states['error']} vulnerabilities in {fixture} ended in error" + ) + print("PASS: Coana analyzed vulnerabilities with the image's toolchain") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("fixture") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--probe", action="store_true") + mode.add_argument("--log", type=Path) + args = parser.parse_args() + + if args.probe: + return 0 if alert_states(args.fixture) else 1 + return validate(args.fixture, args.log.read_text()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_dockerfile.py b/tests/unit/test_dockerfile.py new file mode 100644 index 0000000..bdbac8e --- /dev/null +++ b/tests/unit/test_dockerfile.py @@ -0,0 +1,57 @@ +"""Keeps the Dockerfile's release ARGs below its toolchain layers. + +The comment above ``ARG CLI_VERSION`` in the Dockerfile explains why. A misplaced ARG fails +no build; it only turns every upgrade into a full re-pull, so it is checked here. +""" + +from pathlib import Path + +DOCKERFILE = Path(__file__).resolve().parents[2] / "Dockerfile" +RELEASE_ARGS = { + "CLI_VERSION", + "SDK_VERSION", + "PIP_INDEX_URL", + "PIP_EXTRA_INDEX_URL", + "USE_LOCAL_INSTALL", +} +TOOLCHAIN_MARKERS = ("openjdk", "dotnet", "pypy", "rustup", "pyenv") + + +def _instructions() -> list[str]: + """The Dockerfile's instructions, with continuation lines joined and comments dropped.""" + instructions: list[str] = [] + current = "" + for line in DOCKERFILE.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + current += " " + stripped.removesuffix("\\") + if not stripped.endswith("\\"): + instructions.append(current.strip()) + current = "" + return instructions + + +def test_release_args_are_declared_after_the_toolchain_layers(): + instructions = _instructions() + release_args = [ + i + for i, instruction in enumerate(instructions) + if instruction.startswith("ARG ") + and instruction.split()[1].split("=")[0] in RELEASE_ARGS + ] + toolchain_layers = [ + i + for i, instruction in enumerate(instructions) + if instruction.startswith("RUN ") + and any(marker in instruction for marker in TOOLCHAIN_MARKERS) + ] + assert len(release_args) == len(RELEASE_ARGS), ( + "Dockerfile no longer declares every release ARG" + ) + assert len(toolchain_layers) >= len(TOOLCHAIN_MARKERS), ( + "Dockerfile no longer installs every toolchain" + ) + assert min(release_args) > max(toolchain_layers), ( + "A release ARG is declared above a toolchain layer, so every release would rebuild it" + ) diff --git a/uv.lock b/uv.lock index 0a14aad..f9e7a03 100644 --- a/uv.lock +++ b/uv.lock @@ -1332,7 +1332,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.11.11" +version = "2.11.12" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },