From 1a039e32ab24e080ecd0b0b99cc6ed28415f9b7a Mon Sep 17 00:00:00 2001 From: latte Date: Wed, 14 Oct 2026 15:58:12 +0200 Subject: [PATCH] ci: single Gitea Actions pipeline, uv-based image, scoped docker cleanup Replace lint/test/docker workflows with ci.yml (gitleaks, lint, test-unit, test-integration, test-mcp, build-and-push, docs-check, security). Rebuild the Dockerfile as a digest-pinned multi-stage uv build running as uid 1000, add .dockerignore, a hardened compose file and a gitleaks config. The integration harness now falls back to /etc/hostname and /proc/self to find its own container. --- .dockerignore | 35 +++ .gitea/workflows/ci.yml | 341 ++++++++++++++++++++++++++++ .gitea/workflows/docker.yml | 125 ---------- .gitea/workflows/lint.yml | 34 --- .gitea/workflows/publish.yml | 42 ++-- .gitea/workflows/test.yml | 83 ------- .gitleaks.toml | 37 +++ docker-compose.yml | 93 +++----- docker/Dockerfile | 85 ++++--- docker/docker-compose.yml | 51 ----- scripts/ci/cleanup_job_resources.sh | 51 +++++ scripts/ci/install_gitleaks.sh | 46 ++++ scripts/ci/pip_audit.sh | 25 ++ security/pip-audit-ignore.txt | 4 + tests/integration/harness.py | 62 ++++- tests/integration/test_harness.py | 19 ++ 16 files changed, 729 insertions(+), 404 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitea/workflows/ci.yml delete mode 100644 .gitea/workflows/docker.yml delete mode 100644 .gitea/workflows/lint.yml delete mode 100644 .gitea/workflows/test.yml create mode 100644 .gitleaks.toml delete mode 100644 docker/docker-compose.yml create mode 100755 scripts/ci/cleanup_job_resources.sh create mode 100755 scripts/ci/install_gitleaks.sh create mode 100755 scripts/ci/pip_audit.sh create mode 100644 security/pip-audit-ignore.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..906338e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,35 @@ +# Keep the build context small and free of secrets. The Dockerfile only needs +# pyproject.toml, uv.lock, README.md, LICENSE and src/. +.git +.gitea +.github +.claude +.venv +venv +env + +.env +.env.* +!.env.example + +tests +docs +spec +scripts +docker +docker-compose.yml +security + +__pycache__ +*.py[cod] +.pytest_cache +.mypy_cache +.ruff_cache +.coverage +.coverage.* +htmlcov +dist +build +*.egg-info + +*.log diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..38226c0 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,341 @@ +name: ci + +# One pipeline, eight jobs: +# +# gitleaks secret scan over the full history +# lint ruff check, ruff format --check, mypy +# test-unit pytest (coverage gate from pyproject.toml) +# test-integration pytest against a throwaway Gitea in Docker +# test-mcp MCP smoke tests, including against the freshly built image +# build-and-push build the image; push it on push events (needs lint + test-unit) +# docs-check docs drift check +# security security test suite, pip-audit, ruff bandit rules +# +# Docker hygiene on the shared runner: every resource this workflow creates has +# a per-run name (aegis-ci--...) and is removed by an `if: always()` +# step that deletes exactly those names. Nothing here ever prunes. + +on: + push: + branches: + - '**' + pull_request: + branches: + - dev + - main + +env: + UV_PYTHON_DOWNLOADS: never + UV_NO_PROGRESS: "1" + +jobs: + # --------------------------------------------------------------------------- + # 1. Secret scan: pinned gitleaks binary (sha256 verified), full history. + # --------------------------------------------------------------------------- + gitleaks: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install gitleaks + shell: bash + run: | + bin_dir="${RUNNER_TEMP:-/tmp}/gitleaks-bin" + bash scripts/ci/install_gitleaks.sh "${bin_dir}" + echo "${bin_dir}" >> "$GITHUB_PATH" + + - name: Scan history + shell: bash + run: gitleaks git --config .gitleaks.toml --redact --no-banner --verbose --exit-code 1 . + + # --------------------------------------------------------------------------- + # 2. Lint + # --------------------------------------------------------------------------- + lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false + - name: Install dependencies + run: uv sync --locked + - name: Ruff check + run: uv run ruff check . + - name: Ruff format + run: uv run ruff format --check . + - name: Mypy + run: uv run mypy src + + # --------------------------------------------------------------------------- + # 3. Unit tests (integration tests are skipped without AEGIS_INTEGRATION=1). + # --------------------------------------------------------------------------- + test-unit: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false + - name: Install dependencies + run: uv sync --locked + - name: Run unit tests + run: uv run pytest --cov-report=term-missing --cov-report=xml:coverage.xml + - name: Upload coverage report + # Best effort: some act_runner versions do not support the v4 artifact + # backend, and the coverage gate itself already ran above. + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ github.run_id }} + path: coverage.xml + + # --------------------------------------------------------------------------- + # 4. Integration tests against a throwaway Gitea 28.1.0 in Docker. The job + # container is joined to a per-run network (container mode), see + # tests/integration/README.md. The harness removes what it creates; the + # cleanup step is the safety net for cancelled runs. + # --------------------------------------------------------------------------- + test-integration: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false + - name: Install dependencies + run: uv sync --locked + - name: Run integration tests + env: + AEGIS_INTEGRATION: "1" + AEGIS_IT_NETWORK_MODE: container + run: uv run pytest tests/integration -m integration --no-cov + - name: Cleanup + if: always() + shell: bash + run: bash scripts/ci/cleanup_job_resources.sh + + # --------------------------------------------------------------------------- + # 5. MCP smoke tests. Builds its own copy of the image (same Dockerfile, same + # commit) so the tests can also exercise the container, and passes the + # reference through AEGIS_MCP_IMAGE. The image cannot be shared with + # build-and-push: pull requests never push, and jobs may land on different + # runners. + # --------------------------------------------------------------------------- + test-mcp: + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + MCP_IMAGE: aegis-ci-${{ github.run_id }}-mcp:test + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false + - name: Install dependencies + run: uv sync --locked + - name: Build image + run: docker build -f docker/Dockerfile -t "${MCP_IMAGE}" . + - name: Run MCP smoke tests + env: + AEGIS_MCP_IMAGE: ${{ env.MCP_IMAGE }} + # Smoke tests that start a throwaway Gitea use the same harness as the + # integration tests. + AEGIS_INTEGRATION: "1" + AEGIS_IT_NETWORK_MODE: container + run: uv run pytest tests/mcp -m mcp_smoke --no-cov + - name: Cleanup + if: always() + shell: bash + run: bash scripts/ci/cleanup_job_resources.sh "${MCP_IMAGE}" + + # --------------------------------------------------------------------------- + # 6. Build the image from the checked-out commit. On push events it is also + # pushed: + # :sha-<12> every branch (immutable, traceable to the commit; lets any + # feature branch image be pulled and tested) + # :dev additionally on dev + # :latest and :main additionally on main + # Pull requests only build. The pushed digest is recorded in the job + # outputs and the job summary. + # --------------------------------------------------------------------------- + build-and-push: + needs: [lint, test-unit] + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + image: ${{ steps.meta.outputs.sha_tag }} + digest: ${{ steps.digest.outputs.digest }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute image name and tags + id: meta + shell: bash + run: | + IMAGE="git.hiddenden.cafe/${GITHUB_REPOSITORY,,}" + SHA_TAG="${IMAGE}:sha-${GITHUB_SHA::12}" + EXTRA_TAGS="" + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then + if [ "${GITHUB_REF_NAME}" = "main" ]; then + EXTRA_TAGS="${IMAGE}:latest ${IMAGE}:main" + elif [ "${GITHUB_REF_NAME}" = "dev" ]; then + EXTRA_TAGS="${IMAGE}:dev" + fi + fi + { + echo "image=${IMAGE}" + echo "sha_tag=${SHA_TAG}" + echo "extra_tags=${EXTRA_TAGS}" + } >> "$GITHUB_OUTPUT" + + - name: Build image + shell: bash + run: | + docker build -f docker/Dockerfile \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label "org.opencontainers.image.source=https://git.hiddenden.cafe/${GITHUB_REPOSITORY}" \ + -t "${{ steps.meta.outputs.sha_tag }}" . + + - name: Smoke-test image + shell: bash + run: docker run --rm "${{ steps.meta.outputs.sha_tag }}" version + + - name: Log in to the registry + if: github.event_name == 'push' + uses: docker/login-action@v3 + with: + registry: git.hiddenden.cafe + username: ${{ github.actor }} + # PAT with write:package scope. The built-in token lacks package-write + # on this instance. + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Tag and push + if: github.event_name == 'push' + shell: bash + run: | + for tag in ${{ steps.meta.outputs.extra_tags }}; do + docker tag "${{ steps.meta.outputs.sha_tag }}" "${tag}" + done + for tag in ${{ steps.meta.outputs.sha_tag }} ${{ steps.meta.outputs.extra_tags }}; do + docker push "${tag}" + done + + - name: Record digest + id: digest + shell: bash + run: | + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then + DIGEST="$(docker inspect --format '{{index .RepoDigests 0}}' "${{ steps.meta.outputs.sha_tag }}")" + else + DIGEST="not pushed (pull request build)" + fi + echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT" + { + echo "### Image" + echo + echo "- tag: \`${{ steps.meta.outputs.sha_tag }}\`" + echo "- digest: \`${DIGEST}\`" + if [ -n "${{ steps.meta.outputs.extra_tags }}" ]; then + echo "- also tagged: \`${{ steps.meta.outputs.extra_tags }}\`" + fi + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + echo "image digest: ${DIGEST}" + + # Removes exactly what this run created, even on failure. No prune. + - name: Cleanup + if: always() + shell: bash + run: | + docker logout git.hiddenden.cafe || true + bash scripts/ci/cleanup_job_resources.sh \ + "${{ steps.meta.outputs.sha_tag }}" ${{ steps.meta.outputs.extra_tags }} + + # --------------------------------------------------------------------------- + # 7. Docs drift check + # --------------------------------------------------------------------------- + docs-check: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false + - name: Install dependencies + run: uv sync --locked + - name: Check docs against code and spec + run: uv run python scripts/check_docs.py + + # --------------------------------------------------------------------------- + # 8. Security: test suite, dependency audit, bandit rules via ruff. + # Waivers live in docs/security.md; advisory ignores in + # security/pip-audit-ignore.txt. + # --------------------------------------------------------------------------- + security: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false + - name: Install dependencies + run: uv sync --locked + - name: Security tests + run: uv run pytest tests/security --no-cov + - name: Dependency audit + run: bash scripts/ci/pip_audit.sh + - name: Ruff security rules + run: uv run ruff check --select S . diff --git a/.gitea/workflows/docker.yml b/.gitea/workflows/docker.yml deleted file mode 100644 index 0e17a26..0000000 --- a/.gitea/workflows/docker.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: docker - -on: - # Test on every branch push; registry push is gated per-step to main/dev. - push: - branches: - - '**' - pull_request: - branches: - - main - - dev - -jobs: - # --------------------------------------------------------------------------- - # 1. Lint: ruff + black + mypy. - # --------------------------------------------------------------------------- - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - name: Run lint - run: | - ruff check src tests - ruff format --check src tests - black --check src tests - mypy src - - # --------------------------------------------------------------------------- - # 2. Test: pytest with coverage gate. - # --------------------------------------------------------------------------- - test: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - name: Run tests - run: pytest --cov=aegis_gitea_mcp --cov-report=term-missing --cov-fail-under=80 - - # --------------------------------------------------------------------------- - # 3. Build the Docker image, smoke-test it, push to Gitea (push events to - # main/dev only), then clean up so nothing lingers on the self-hosted - # runner. - # --------------------------------------------------------------------------- - docker: - needs: [lint, test] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Compute image name & tags - id: meta - shell: bash - run: | - IMAGE="git.hiddenden.cafe/${GITHUB_REPOSITORY,,}" - echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" - echo "sha_tag=${IMAGE}:sha-${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" - if [ "${GITHUB_REF_NAME}" = "main" ]; then - # Production: stable :latest + :main - echo "branch_tags=${IMAGE}:latest ${IMAGE}:main" >> "$GITHUB_OUTPUT" - else - # dev (and any other branch): tag with the branch name - echo "branch_tags=${IMAGE}:${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" - fi - - - name: Build image - shell: bash - run: docker build -f docker/Dockerfile -t "${{ steps.meta.outputs.sha_tag }}" . - - - name: Smoke-test image - shell: bash - run: | - docker run --rm --entrypoint python "${{ steps.meta.outputs.sha_tag }}" \ - -c "import aegis_gitea_mcp" - echo "Image imports cleanly." - - - name: Log in to Gitea Container Registry - if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'dev') - uses: docker/login-action@v3 - with: - registry: git.hiddenden.cafe - username: ${{ github.actor }} - # PAT with write:package scope, stored as the REGISTRY_TOKEN secret. - # The auto-provided GITEA_TOKEN lacks package-write permission on - # this instance, so we use a dedicated token here. - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Tag & push - if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'dev') - shell: bash - run: | - for tag in ${{ steps.meta.outputs.branch_tags }} ${{ steps.meta.outputs.sha_tag }}; do - docker tag "${{ steps.meta.outputs.sha_tag }}" "$tag" - docker push "$tag" - echo "Pushed $tag" - done - - # Always runs — removes exactly what this run created, even on failure. - # Scoped on purpose: if the runner shares the host Docker daemon, a global - # prune would also wipe other homelab services. We never create volumes - # here, so only dangling images + build cache are swept. - - name: Cleanup - if: always() - shell: bash - run: | - docker rmi -f ${{ steps.meta.outputs.sha_tag }} ${{ steps.meta.outputs.branch_tags }} || true - docker image prune -f || true - docker builder prune -f || true diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml deleted file mode 100644 index 16bcff0..0000000 --- a/.gitea/workflows/lint.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: lint - -on: - push: - pull_request: - -jobs: - lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Run Ruff (fail on any diagnostics) - run: ruff check src tests - - - name: Enforce formatting - run: | - ruff format --check src tests - black --check src tests - - - name: Run mypy - run: mypy src diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index 8ba0ef5..43a8f33 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -3,7 +3,7 @@ name: publish # Build the Python package with uv and publish it to the self-hosted Gitea PyPI # registry on merge. dev -> a dev package (aegis-gitea-mcp-dev, .devN versions); # main -> the stable package (aegis-gitea-mcp). Gated on lint + tests so a release -# can never ship red. Reuses the REGISTRY_TOKEN package secret (same one docker.yml +# can never ship red. Reuses the REGISTRY_TOKEN package secret (same one ci.yml # uses); if it is absent the job fails loudly instead of publishing anonymously. on: push: @@ -13,7 +13,7 @@ on: jobs: # --------------------------------------------------------------------------- - # 1. Lint: ruff + black + mypy (same gate as the other workflows). + # 1. Lint (same gate as ci.yml, via uv and the lockfile). # --------------------------------------------------------------------------- lint: runs-on: ubuntu-latest @@ -24,19 +24,21 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + run: uv sync --locked - name: Run lint run: | - ruff check src tests - ruff format --check src tests - black --check src tests - mypy src + uv run ruff check . + uv run ruff format --check . + uv run mypy src # --------------------------------------------------------------------------- - # 2. Test: pytest with coverage gate. + # 2. Test: unit tests with the coverage gate from pyproject.toml. # --------------------------------------------------------------------------- test: runs-on: ubuntu-latest @@ -47,12 +49,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + run: uv sync --locked - name: Run tests - run: pytest --cov=aegis_gitea_mcp --cov-report=term-missing --cov-fail-under=80 + run: uv run pytest # --------------------------------------------------------------------------- # 3. Build with uv and publish to the Gitea PyPI registry. @@ -78,6 +83,9 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v5 + with: + version: "0.12.5" + enable-cache: false - name: Require publish credentials shell: bash @@ -100,7 +108,9 @@ jobs: CHANNEL="stable" else PKG_NAME="aegis-gitea-mcp-dev" - PKG_VERSION="${BASE}.dev${GITHUB_RUN_NUMBER}" + # Drop an existing .devN suffix (the committed version may be a + # development version) so the result has exactly one dev segment. + PKG_VERSION="${BASE%%.dev*}.dev${GITHUB_RUN_NUMBER}" CHANNEL="dev" fi echo "pkg_name=${PKG_NAME}" >> "$GITHUB_OUTPUT" @@ -133,7 +143,7 @@ jobs: - name: Publish to Gitea PyPI registry shell: bash env: - # Reuse the existing package secret (same one docker.yml uses). The + # Reuse the existing package secret (same one ci.yml uses). The # token authenticates as its owning Gitea user, so GITHUB_ACTOR is the # username and the token is the password. REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml deleted file mode 100644 index d3620da..0000000 --- a/.gitea/workflows/test.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: test - -on: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Run lint - run: | - ruff check src tests - ruff format --check src tests - black --check src tests - - - name: Run tests with coverage gate - run: | - pytest --cov=aegis_gitea_mcp --cov-report=term-missing --cov-fail-under=80 - - # --------------------------------------------------------------------------- - # Package: build with uv and smoke-test both install profiles so packaging - # regressions (broken console scripts, dependency split) are caught in CI. - # --------------------------------------------------------------------------- - package: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@v5 - - - name: Build sdist + wheel - run: uv build - - - name: Smoke-test the core (stdio) install - shell: bash - run: | - WHEEL="$(echo dist/*.whl)" - python -m venv /tmp/core - /tmp/core/bin/pip install --quiet "$WHEEL" - # The core install must NOT pull in the web stack. - if /tmp/core/bin/python -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('fastapi') else 1)"; then - echo "::error::core install unexpectedly includes fastapi" >&2 - exit 1 - fi - # The stdio console script exists and exits 2 with a clear error when - # required env vars are missing (no traceback). - set +e - GITEA_URL= GITEA_TOKEN= /tmp/core/bin/aegis-gitea-mcp >/dev/null 2>/tmp/core_err.txt - rc=$? - set -e - test "$rc" = "2" || { echo "::error::stdio entry exit $rc (expected 2)"; cat /tmp/core_err.txt; exit 1; } - grep -q "GITEA_URL" /tmp/core_err.txt - echo "core stdio entry OK (exit 2, no fastapi)" - - - name: Smoke-test the [server] install - shell: bash - run: | - WHEEL="$(echo dist/*.whl)" - python -m venv /tmp/server - /tmp/server/bin/pip install --quiet "${WHEEL}[server]" - /tmp/server/bin/python -c "import fastapi, uvicorn, aegis_gitea_mcp.server_entry; print('server extra import OK')" diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..7bca784 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,37 @@ +# Gitleaks configuration. Extends the default ruleset; only the findings below +# are allowlisted. Every one of them was reviewed and is a fake or a placeholder +# (documentation examples, test fixtures, and a variable name that matched the +# generic key rule). Real credential rules (cloud keys, private keys, provider +# token formats) stay active everywhere. +title = "AegisGitea-MCP gitleaks config" + +[extend] +useDefault = true + +# Fake tokens in tests and placeholder values in docs. Only the two noisy +# generic rules are silenced, and only in these directories. +[[allowlists]] +description = "fake fixtures in tests and placeholder examples in docs" +targetRules = ["generic-api-key", "curl-auth-header"] +paths = [ + '''^tests/''', + '''^docs/''', +] + +# Legacy top-level guides that no longer exist in the tree (they live only in +# history). They contained placeholder keys such as YOUR_API_KEY and +# a1b2c3d4e5f6.... +[[allowlists]] +description = "removed legacy docs with placeholder keys" +targetRules = ["generic-api-key", "curl-auth-header"] +paths = [ + '''^(AUTH_SETUP|AUTHENTICATION_IMPLEMENTATION_SUMMARY|CHATGPT_SETUP|DEPLOYMENT|KEY_ROTATION|TESTING)\.md$''', +] + +# Removed module: a cache-size constant name matched the generic key rule. +[[allowlists]] +description = "false positive in a removed source file" +targetRules = ["generic-api-key"] +paths = [ + '''^src/aegis_gitea_mcp/server\.py$''', +] diff --git a/docker-compose.yml b/docker-compose.yml index 451dafb..0796200 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,82 +1,59 @@ +# Production deployment of the HTTP transport. +# +# cp .env.example .env # set GITEA_URL, MCP_BEARER_TOKEN and/or OAuth settings +# docker compose up -d +# +# The container binds 0.0.0.0 internally; the server refuses to start unless +# authentication is configured, and the port is published on loopback only. +# Put a TLS reverse proxy in front of 127.0.0.1:8080 (and set PUBLIC_BASE_URL). services: aegis-mcp: - profiles: ["prod"] - build: - context: . - dockerfile: docker/Dockerfile + image: git.hiddenden.cafe/hiddenden/aegisgitea-mcp:${AEGIS_MCP_TAG:-latest} + # To build locally instead of pulling the published image: + # build: + # context: . + # dockerfile: docker/Dockerfile container_name: aegis-gitea-mcp restart: unless-stopped env_file: - .env environment: - ENVIRONMENT: production - MCP_HOST: ${MCP_HOST:-127.0.0.1} - ALLOW_INSECURE_BIND: ${ALLOW_INSECURE_BIND:-false} - expose: - - "8080" + # Fixed by the image layout; do not override from .env. + MCP_TRANSPORT: http + MCP_HOST: 0.0.0.0 + MCP_PORT: "8080" + DATA_DIR: /var/lib/aegis-mcp + ports: + - "127.0.0.1:8080:8080" volumes: - - aegis-mcp-logs:/var/log/aegis-mcp - aegis-mcp-data:/var/lib/aegis-mcp - - ./policy.yaml:/app/policy.yaml:ro read_only: true tmpfs: - /tmp - security_opt: - - no-new-privileges:true cap_drop: - ALL + security_opt: + - no-new-privileges:true user: "1000:1000" - networks: - - proxy healthcheck: test: - [ - "CMD", - "python", - "-c", - "import httpx; httpx.get('http://127.0.0.1:8080/health', timeout=5)", - ] + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=5)" interval: 30s timeout: 10s retries: 3 - start_period: 10s - - aegis-mcp-dev: - profiles: ["dev"] - build: - context: . - dockerfile: docker/Dockerfile - container_name: aegis-gitea-mcp-dev - restart: unless-stopped - env_file: - - .env - environment: - ENVIRONMENT: development - MCP_HOST: 127.0.0.1 - ALLOW_INSECURE_BIND: false - LOG_LEVEL: DEBUG - EXPOSE_ERROR_DETAILS: true - ports: - - "127.0.0.1:${MCP_PORT:-8080}:8080" - volumes: - - ./src:/app/src:ro - - ./policy.yaml:/app/policy.yaml:ro - - aegis-mcp-logs:/var/log/aegis-mcp - - aegis-mcp-data:/var/lib/aegis-mcp - security_opt: - - no-new-privileges:true - cap_drop: - - ALL - user: "1000:1000" - networks: - - proxy + start_period: 15s + # Optional: join an external reverse proxy network instead of publishing a + # port. Uncomment this and the `networks:` block at the bottom, and drop + # the `ports:` entry above. + # networks: + # - proxy volumes: - aegis-mcp-logs: - driver: local aegis-mcp-data: - driver: local -networks: - proxy: - external: true +# networks: +# proxy: +# external: true diff --git a/docker/Dockerfile b/docker/Dockerfile index ff2dea8..fa5ffe0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,51 +1,68 @@ # syntax=docker/dockerfile:1 -# Build stage -FROM python:3.12-slim AS builder +# Both stages use the same python:3.12-slim image, pinned by digest so a build +# is reproducible. Refresh the digest deliberately (docker pull, then +# docker inspect --format '{{index .RepoDigests 0}}' python:3.12-slim). +ARG PYTHON_IMAGE=python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +FROM ${UV_IMAGE} AS uv + +# --------------------------------------------------------------------------- +# Builder: resolve the locked, non-dev dependencies and install the project +# (non-editable) into /app/.venv. Nothing from this stage except the venv is +# copied into the runtime image. +# --------------------------------------------------------------------------- +FROM ${PYTHON_IMAGE} AS builder + +COPY --from=uv /uv /usr/local/bin/uv + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never \ + UV_PROJECT_ENVIRONMENT=/app/.venv WORKDIR /app -RUN apt-get update \ - && apt-get install -y --no-install-recommends gcc \ - && rm -rf /var/lib/apt/lists/* +COPY pyproject.toml uv.lock README.md LICENSE ./ +COPY src ./src +RUN uv sync --locked --no-dev --no-editable -COPY requirements.txt ./ -RUN pip install --no-cache-dir --user -r requirements.txt +# --------------------------------------------------------------------------- +# Runtime: slim image, only the venv, non-root user. +# --------------------------------------------------------------------------- +FROM ${PYTHON_IMAGE} -# Runtime stage -FROM python:3.12-slim +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/app/.venv/bin:$PATH -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV PIP_DISABLE_PIP_VERSION_CHECK=1 -ENV NODE_ENV=production -ENV ENVIRONMENT=production -ENV PATH=/home/aegis/.local/bin:$PATH -ENV PYTHONPATH=/app/src:$PYTHONPATH +# Non-root runtime user (uid 1000). DATA_DIR holds the token store and the +# audit log; mount a persistent volume there. +RUN useradd --create-home --uid 1000 --shell /usr/sbin/nologin aegis \ + && mkdir -p /var/lib/aegis-mcp \ + && chown aegis:aegis /var/lib/aegis-mcp -WORKDIR /app +COPY --from=builder --chown=root:root /app/.venv /app/.venv -# Non-root runtime user -RUN useradd -m -u 1000 -s /usr/sbin/nologin aegis - -COPY --from=builder --chown=aegis:aegis /root/.local /home/aegis/.local -COPY --chown=aegis:aegis src/ ./src/ -COPY --chown=aegis:aegis scripts/ ./scripts/ - -# /var/log/aegis-mcp -> audit log (mount a writable volume) -# /var/lib/aegis-mcp -> dynamic client registration store (mount a writable, persistent volume) -RUN mkdir -p /var/log/aegis-mcp /var/lib/aegis-mcp /tmp/aegis-mcp \ - && chown -R aegis:aegis /var/log/aegis-mcp /var/lib/aegis-mcp /tmp/aegis-mcp +# Binding 0.0.0.0 is required for the published port to work inside a +# container. It is safe by default: the server refuses to start (exit code 2) +# on a non-loopback bind unless authentication is configured (MCP_BEARER_TOKEN +# or OAUTH_ENABLED). Still publish the port only on 127.0.0.1 or behind a +# reverse proxy (see docker-compose.yml). +ENV DATA_DIR=/var/lib/aegis-mcp \ + MCP_TRANSPORT=http \ + MCP_HOST=0.0.0.0 \ + MCP_PORT=8080 USER aegis +WORKDIR /home/aegis EXPOSE 8080 -HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ - CMD python -c "import httpx; httpx.get('http://127.0.0.1:8080/health', timeout=5)" || exit 1 +# Python stdlib only (the slim image has no curl). Probes loopback on MCP_PORT. +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health' % os.environ.get('MCP_PORT', '8080'), timeout=5)"] -CMD ["python", "-m", "aegis_gitea_mcp.server"] +ENTRYPOINT ["aegis-gitea-mcp"] +CMD ["serve"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 3b73e76..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,51 +0,0 @@ -services: - aegis-mcp: - build: - context: .. - dockerfile: docker/Dockerfile - container_name: aegis-gitea-mcp - restart: unless-stopped - env_file: - - ../.env - environment: - ENVIRONMENT: production - MCP_HOST: ${MCP_HOST:-127.0.0.1} - ALLOW_INSECURE_BIND: ${ALLOW_INSECURE_BIND:-false} - ports: - - "127.0.0.1:${MCP_PORT:-8080}:8080" - volumes: - - aegis-mcp-logs:/var/log/aegis-mcp - - aegis-mcp-data:/var/lib/aegis-mcp - - ../policy.yaml:/app/policy.yaml:ro - read_only: true - tmpfs: - - /tmp - security_opt: - - no-new-privileges:true - cap_drop: - - ALL - user: "1000:1000" - networks: - - proxy - healthcheck: - test: - [ - "CMD", - "python", - "-c", - "import httpx; httpx.get('http://127.0.0.1:8080/health', timeout=5)", - ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - -volumes: - aegis-mcp-logs: - driver: local - aegis-mcp-data: - driver: local - -networks: - proxy: - external: true diff --git a/scripts/ci/cleanup_job_resources.sh b/scripts/ci/cleanup_job_resources.sh new file mode 100755 index 0000000..e4b3cbd --- /dev/null +++ b/scripts/ci/cleanup_job_resources.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Remove the Docker resources that THIS job created, and nothing else. +# +# Usage: scripts/ci/cleanup_job_resources.sh [image-ref ...] +# +# The runner shares one Docker daemon with other jobs and other services, so +# this script never prunes. It only removes: +# * per-run networks named aegis-it-* that the job container is attached to +# (the integration harness joins the job container to its own per-run +# network only), plus the containers still attached to those networks; +# * containers started from the exact image refs given as arguments; +# * the exact image refs given as arguments. +# Everything is best effort (a failing removal never fails the job). +set -uo pipefail + +job_ref="" +if command -v uv > /dev/null 2>&1 && [ -f tests/integration/harness.py ]; then + job_ref="$(uv run python -c \ + 'from tests.integration.harness import job_container_ref; print(job_container_ref())' \ + 2> /dev/null || true)" +fi +if [ -z "${job_ref}" ]; then + job_ref="${HOSTNAME:-}" +fi + +if [ -n "${job_ref}" ] && docker container inspect "${job_ref}" > /dev/null 2>&1; then + job_id="$(docker container inspect --format '{{.Id}}' "${job_ref}")" + networks="$(docker container inspect \ + --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{"\n"}}{{end}}' \ + "${job_ref}" | grep -E '^aegis-it-[0-9a-f]+$' || true)" + for net in ${networks}; do + for cid in $(docker network inspect --format '{{range $id, $_ := .Containers}}{{$id}}{{"\n"}}{{end}}' "${net}" 2> /dev/null); do + if [ "${cid}" != "${job_id}" ]; then + docker rm -f -v "${cid}" > /dev/null 2>&1 || true + fi + done + docker network disconnect -f "${net}" "${job_id}" > /dev/null 2>&1 || true + docker network rm "${net}" > /dev/null 2>&1 || true + echo "removed network ${net}" + done +fi + +for image in "$@"; do + for cid in $(docker ps -aq --filter "ancestor=${image}" 2> /dev/null); do + docker rm -f -v "${cid}" > /dev/null 2>&1 || true + done + docker rmi "${image}" > /dev/null 2>&1 || true + echo "removed image ${image}" +done + +exit 0 diff --git a/scripts/ci/install_gitleaks.sh b/scripts/ci/install_gitleaks.sh new file mode 100755 index 0000000..865ff1b --- /dev/null +++ b/scripts/ci/install_gitleaks.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Install a pinned gitleaks release into a directory, verifying its sha256. +# +# Usage: scripts/ci/install_gitleaks.sh +# +# The binary ends up at /gitleaks. To upgrade, bump GITLEAKS_VERSION +# and both digests from the release's gitleaks__checksums.txt. +set -euo pipefail + +GITLEAKS_VERSION="8.30.1" +SHA256_X64="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" +SHA256_ARM64="e4a487ee7ccd7d3a7f7ec08657610aa3606637dab924210b3aee62570fb4b080" + +dest="${1:?usage: install_gitleaks.sh }" + +case "$(uname -m)" in + x86_64 | amd64) + arch="x64" + expected="${SHA256_X64}" + ;; + aarch64 | arm64) + arch="arm64" + expected="${SHA256_ARM64}" + ;; + *) + echo "unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +archive="gitleaks_${GITLEAKS_VERSION}_linux_${arch}.tar.gz" +url="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${archive}" + +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +curl --fail --silent --show-error --location --retry 3 --output "${work}/${archive}" "${url}" +echo "${expected} ${work}/${archive}" | sha256sum --check --status || { + echo "sha256 mismatch for ${archive}" >&2 + exit 1 +} + +mkdir -p "${dest}" +tar -xzf "${work}/${archive}" -C "${dest}" gitleaks +chmod 0755 "${dest}/gitleaks" +"${dest}/gitleaks" version diff --git a/scripts/ci/pip_audit.sh b/scripts/ci/pip_audit.sh new file mode 100755 index 0000000..c07b280 --- /dev/null +++ b/scripts/ci/pip_audit.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Audit the locked dependency set with pip-audit. +# +# Ignored advisories (documented waivers, see docs/security.md) are read from +# security/pip-audit-ignore.txt: one advisory id per line, '#' starts a comment. +set -euo pipefail + +ignore_file="security/pip-audit-ignore.txt" +req="$(mktemp)" +trap 'rm -f "${req}"' EXIT + +uv export --locked --no-hashes --no-emit-project --format requirements-txt > "${req}" + +args=(-r "${req}" --strict --no-deps --disable-pip) +if [ -f "${ignore_file}" ]; then + while IFS= read -r line; do + id="${line%%#*}" + id="$(echo "${id}" | tr -d '[:space:]')" + if [ -n "${id}" ]; then + args+=(--ignore-vuln "${id}") + fi + done < "${ignore_file}" +fi + +uv run pip-audit "${args[@]}" diff --git a/security/pip-audit-ignore.txt b/security/pip-audit-ignore.txt new file mode 100644 index 0000000..c6ab6f1 --- /dev/null +++ b/security/pip-audit-ignore.txt @@ -0,0 +1,4 @@ +# Advisories ignored by `scripts/ci/pip_audit.sh`. +# One vulnerability id per line (GHSA-..., PYSEC-..., CVE-...). '#' starts a comment. +# Every entry needs a justification and an expiry here and a matching waiver in +# docs/security.md. Keep this file empty unless a waiver is documented. diff --git a/tests/integration/harness.py b/tests/integration/harness.py index 83c8a69..c0cae5e 100644 --- a/tests/integration/harness.py +++ b/tests/integration/harness.py @@ -14,6 +14,7 @@ from __future__ import annotations import atexit import contextlib import os +import re import secrets import signal import socket @@ -21,7 +22,9 @@ import subprocess import sys import threading import time +from collections.abc import Mapping from dataclasses import dataclass, field +from pathlib import Path from types import FrameType from typing import Any @@ -76,6 +79,61 @@ def run_docker( return proc +_MOUNTINFO_ID_RE = re.compile(r"/containers/([0-9a-f]{64})/") +_CGROUP_ID_RE = re.compile(r"(? list[str]: + """Return references that may name the container this process runs in. + + Order: ``$HOSTNAME``, ``/etc/hostname``, then 64-hex container ids found in + ``/proc/self/mountinfo`` and ``/proc/self/cgroup``. On act_runner the hostname + is normally the container id, but a runner may set another hostname or use host + networking settings, so the other sources act as fallbacks. + """ + env = os.environ if environ is None else environ + candidates: list[str] = [] + + def add(value: str) -> None: + value = value.strip() + if value and value not in candidates: + candidates.append(value) + + add(env.get("HOSTNAME", "")) + for rel in ("etc/hostname",): + with contextlib.suppress(OSError): + add((proc_root / rel).read_text(encoding="utf-8")) + for rel, pattern in ( + ("proc/self/mountinfo", _MOUNTINFO_ID_RE), + ("proc/self/cgroup", _CGROUP_ID_RE), + ): + with contextlib.suppress(OSError): + for match in pattern.finditer((proc_root / rel).read_text(encoding="utf-8")): + add(match.group(1)) + return candidates + + +def job_container_ref() -> str: + """Return a reference docker accepts for the container running the tests. + + Each candidate from ``container_id_candidates`` is checked with + ``docker inspect``; the first one docker resolves wins. + """ + for candidate in container_id_candidates(): + proc = run_docker( + "container", "inspect", "--format", "{{.Id}}", candidate, check=False, timeout=30 + ) + if proc.returncode == 0 and proc.stdout.strip(): + return candidate + raise HarnessError( + "AEGIS_IT_NETWORK_MODE=container: could not identify the container running the " + "tests ($HOSTNAME, /etc/hostname and /proc/self did not resolve to a container)" + ) + + def require_docker() -> None: """Fail with a clear message when the docker daemon is not usable.""" try: @@ -417,9 +475,7 @@ def start_gitea() -> GiteaInstance: try: base_url = _start_container(res, container_mode) if container_mode: - job_container = os.environ.get("HOSTNAME") - if not job_container: - raise HarnessError("AEGIS_IT_NETWORK_MODE=container requires $HOSTNAME") + job_container = job_container_ref() run_docker("network", "connect", res.network, job_container) res.attached_container = job_container _wait_ready(base_url, res.container) diff --git a/tests/integration/test_harness.py b/tests/integration/test_harness.py index 9ce3c2e..3c6bfc3 100644 --- a/tests/integration/test_harness.py +++ b/tests/integration/test_harness.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import pytest from tests.integration import harness @@ -138,3 +140,20 @@ def test_missing_docker_cli_gives_clear_error(monkeypatch: pytest.MonkeyPatch) - monkeypatch.setenv("PATH", "") with pytest.raises(HarnessError, match="Docker is unavailable"): harness.require_docker() + + +def test_container_id_candidates_fall_back_to_proc(tmp_path: Path) -> None: + cid = "ab" * 32 + (tmp_path / "etc").mkdir() + (tmp_path / "proc/self").mkdir(parents=True) + (tmp_path / "etc/hostname").write_text("custom-host\n") + (tmp_path / "proc/self/mountinfo").write_text( + f"1 2 0:1 /var/lib/docker/containers/{cid}/hostname /etc/hostname rw - ext4 /dev/x rw\n" + ) + (tmp_path / "proc/self/cgroup").write_text(f"0::/docker/{cid}\n") + candidates = harness.container_id_candidates({"HOSTNAME": "job-host"}, proc_root=tmp_path) + assert candidates == ["job-host", "custom-host", cid] + + +def test_container_id_candidates_tolerate_missing_files(tmp_path: Path) -> None: + assert harness.container_id_candidates({}, proc_root=tmp_path) == []