ci: harden GHCR release publishing (#2573)

.github/scripts/release-workflow-contract_test.py

@@ -175,10 +175,14 @@ class ReleaseWorkflowContractTest(unittest.TestCase):
         direct_dependencies = {
             "build-desktop": ("prepare", "build-bundles"),
             "docker-amd64": ("prepare", "build-bundles"),
-            "docker-arm64": ("prepare", "build-bundles"),
+            "docker-arm64": ("prepare", "build-bundles", "docker-amd64"),
             "docker-manifest": ("prepare", "docker-amd64", "docker-arm64"),
             "docker-universal-amd64": ("prepare", "docker-manifest"),
-            "docker-universal-arm64": ("prepare", "docker-manifest"),
+            "docker-universal-arm64": (
+                "prepare",
+                "docker-manifest",
+                "docker-universal-amd64",
+            ),
             "docker-universal-manifest": (
                 "prepare",
                 "docker-universal-amd64",
@@ -691,6 +695,63 @@ class ReleaseWorkflowContractTest(unittest.TestCase):
         self.assertIn('cp "$bundle_root/dmg/bundle_dmg.sh"', DIAGNOSTICS)
         self.assertIn("|| true", DIAGNOSTICS)
 
+    def test_ghcr_builds_retry_transient_failures_and_publish_digests(self) -> None:
+        for job, name in (
+            ("docker-amd64", "Build and push (amd64 staging tag)"),
+            ("docker-arm64", "Build and push (arm64 staging tag)"),
+            (
+                "docker-universal-amd64",
+                "Build and push (universal amd64 staging tag)",
+            ),
+            (
+                "docker-universal-arm64",
+                "Build and push (universal arm64 staging tag)",
+            ),
+        ):
+            build = step_block(name)
+            job_text = job_block(job)
+            with self.subTest(step=name):
+                runtime_setup = "crazy-max/ghaction-github-runtime@"
+                self.assertIn(runtime_setup, job_text)
+                self.assertLess(
+                    job_text.index(runtime_setup),
+                    job_text.index(f"- name: {name}"),
+                )
+                self.assertIn("bash scripts/release/retry-ghcr-command.sh", build)
+                self.assertIn("docker buildx build", build)
+                self.assertIn("--metadata-file", build)
+                self.assertIn("containerimage.digest", build)
+                self.assertIn('echo "digest=$digest" >> "$GITHUB_OUTPUT"', build)
+                self.assertNotIn("docker/build-push-action@", build)
+
+    def test_ghcr_manifest_mutations_retry_and_can_load_shared_helper(self) -> None:
+        for job, step in (
+            ("docker-manifest", "Combine arch images into multi-arch tags"),
+            ("docker-universal-manifest", "Promote staging to final tags"),
+        ):
+            block = job_block(job)
+            mutation = step_block(step)
+            with self.subTest(job=job):
+                self.assertIn("actions/checkout@", block)
+                self.assertIn("ref: ${{ needs.prepare.outputs.ref }}", block)
+                self.assertIn("bash scripts/release/retry-ghcr-command.sh", mutation)
+                self.assertIn("docker buildx imagetools create", mutation)
+
+    def test_ghcr_architecture_jobs_are_serialized(self) -> None:
+        self.assertIn(
+            "needs: [prepare, build-bundles, docker-amd64]",
+            job_block("docker-arm64"),
+        )
+        self.assertIn(
+            "needs: [prepare, docker-manifest, docker-universal-amd64]",
+            job_block("docker-universal-arm64"),
+        )
+
+    def test_release_validation_runs_ghcr_retry_regression(self) -> None:
+        self.assertIn("scripts/release/retry-ghcr-command.test.sh", LINT_WORKFLOW)
+        makefile = (REPO_ROOT / "Makefile").read_text()
+        self.assertIn("bash scripts/release/retry-ghcr-command.test.sh", makefile)
+
 
 if __name__ == "__main__":
     unittest.main()

.github/workflows/lint-action-pinning.yml

@@ -28,6 +28,8 @@ on:
       - "scripts/release/package-npm-runtime.sh"
       - "scripts/release/publish-npm.sh"
       - "scripts/release/publish-npm.test.mjs"
+      - "scripts/release/retry-ghcr-command.sh"
+      - "scripts/release/retry-ghcr-command.test.sh"
   pull_request:
     branches: [main]
     paths:
@@ -55,6 +57,8 @@ on:
       - "scripts/release/package-npm-runtime.sh"
       - "scripts/release/publish-npm.sh"
       - "scripts/release/publish-npm.test.mjs"
+      - "scripts/release/retry-ghcr-command.sh"
+      - "scripts/release/retry-ghcr-command.test.sh"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -95,6 +99,9 @@ jobs:
       - name: Test runtime bundle contract
         run: bash scripts/release/runtime-bundle.test.sh
 
+      - name: Test GHCR retry helper
+        run: bash scripts/release/retry-ghcr-command.test.sh
+
       - name: Test npm release helpers
         run: node --test scripts/release/nightly-version.test.mjs scripts/release/nightly-release.test.mjs scripts/release/npm-view-version.test.mjs scripts/release/publish-npm.test.mjs
 

.github/workflows/release.yml

@@ -1447,29 +1447,44 @@ jobs:
         with:
           images: ghcr.io/kdlbs/kandev
 
+      # The inline Buildx command uses the GHA cache backend directly. This
+      # helper exposes ACTIONS_RUNTIME_TOKEN and ACTIONS_CACHE_URL, which
+      # docker/build-push-action normally prepares for its own invocation.
+      - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
+
       - name: Build and push (amd64 staging tag)
         id: build
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
-        with:
-          context: ./ctx
-          file: ./Dockerfile
-          push: true
-          platforms: linux/amd64
-          tags: ${{ env.STAGING_IMAGE }}
-          labels: ${{ steps.meta.outputs.labels }}
-          # provenance/sbom would attach an extra attestation manifest, which
-          # turns `imagetools inspect --raw` into an OCI index and breaks the
-          # size gate's `.layers[].size` query. The final multi-arch tags
-          # don't preserve attestations through `imagetools create` anyway,
-          # so disabling here doesn't change user-facing image content.
-          provenance: false
-          sbom: false
-          cache-from: type=gha,scope=base-amd64
-          cache-to: type=gha,mode=max,scope=base-amd64
+        env:
+          DOCKER_LABELS: ${{ steps.meta.outputs.labels }}
+        run: |
+          set -euo pipefail
+          metadata_file="$RUNNER_TEMP/docker-metadata.json"
+          label_args=()
+          while IFS= read -r label; do
+            [ -z "$label" ] && continue
+            label_args+=(--label "$label")
+          done <<<"$DOCKER_LABELS"
+
+          bash scripts/release/retry-ghcr-command.sh \
+            docker buildx build \
+            --file ./Dockerfile \
+            --platform linux/amd64 \
+            --push \
+            --tag "$STAGING_IMAGE" \
+            "${label_args[@]}" \
+            --provenance=false \
+            --sbom=false \
+            --cache-from type=gha,scope=base-amd64 \
+            --cache-to type=gha,mode=max,scope=base-amd64 \
+            --metadata-file "$metadata_file" \
+            ./ctx
+
+          digest="$(jq -er '."containerimage.digest"' "$metadata_file")"
+          echo "digest=$digest" >> "$GITHUB_OUTPUT"
 
   docker-arm64:
     name: Build & push container image (arm64)
-    needs: [prepare, build-bundles]
+    needs: [prepare, build-bundles, docker-amd64]
     if: >-
       ${{ !cancelled() &&
         github.event_name == 'workflow_dispatch' &&
@@ -1477,7 +1492,8 @@ jobs:
         !inputs.dry_run &&
         !inputs.desktop_validation_only &&
         needs.prepare.result == 'success' &&
-        needs.build-bundles.result == 'success'
+        needs.build-bundles.result == 'success' &&
+        needs.docker-amd64.result == 'success'
       }}
     runs-on: ubuntu-24.04-arm
     permissions:
@@ -1520,20 +1536,37 @@ jobs:
         with:
           images: ghcr.io/kdlbs/kandev
 
+      - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
+
       - name: Build and push (arm64 staging tag)
         id: build
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
-        with:
-          context: ./ctx
-          file: ./Dockerfile
-          push: true
-          platforms: linux/arm64
-          tags: ${{ env.STAGING_IMAGE }}
-          labels: ${{ steps.meta.outputs.labels }}
-          provenance: false
-          sbom: false
-          cache-from: type=gha,scope=base-arm64
-          cache-to: type=gha,mode=max,scope=base-arm64
+        env:
+          DOCKER_LABELS: ${{ steps.meta.outputs.labels }}
+        run: |
+          set -euo pipefail
+          metadata_file="$RUNNER_TEMP/docker-metadata.json"
+          label_args=()
+          while IFS= read -r label; do
+            [ -z "$label" ] && continue
+            label_args+=(--label "$label")
+          done <<<"$DOCKER_LABELS"
+
+          bash scripts/release/retry-ghcr-command.sh \
+            docker buildx build \
+            --file ./Dockerfile \
+            --platform linux/arm64 \
+            --push \
+            --tag "$STAGING_IMAGE" \
+            "${label_args[@]}" \
+            --provenance=false \
+            --sbom=false \
+            --cache-from type=gha,scope=base-arm64 \
+            --cache-to type=gha,mode=max,scope=base-arm64 \
+            --metadata-file "$metadata_file" \
+            ./ctx
+
+          digest="$(jq -er '."containerimage.digest"' "$metadata_file")"
+          echo "digest=$digest" >> "$GITHUB_OUTPUT"
 
   docker-manifest:
     name: Assemble multi-arch base manifest
@@ -1553,6 +1586,11 @@ jobs:
       contents: read
       packages: write
     steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ needs.prepare.outputs.ref }}
+          persist-credentials: false
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
@@ -1591,7 +1629,9 @@ jobs:
             [ -z "$tag" ] && continue
             tag_args+=(--tag "$tag")
           done <<<"$TAGS"
-          docker buildx imagetools create "${tag_args[@]}" "$AMD64_REF" "$ARM64_REF"
+          bash scripts/release/retry-ghcr-command.sh \
+            docker buildx imagetools create \
+            "${tag_args[@]}" "$AMD64_REF" "$ARM64_REF"
 
   # ---------------------------------------------------------------------------
   # Universal image — same per-arch native split, on top of the base image
@@ -1638,29 +1678,42 @@ jobs:
         with:
           images: ghcr.io/kdlbs/kandev
 
+      - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
+
       - name: Build and push (universal amd64 staging tag)
         id: build
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
-        with:
-          context: .
-          file: Dockerfile.universal
-          push: true
-          platforms: linux/amd64
-          build-args: |
-            BASE_IMAGE=ghcr.io/kdlbs/kandev:${{ needs.prepare.outputs.tag }}
-          tags: ${{ env.STAGING_IMAGE }}
-          labels: ${{ steps.meta.outputs.labels }}
-          # provenance/sbom off so `imagetools inspect --raw` on the staging
-          # tag returns the plain image manifest, not an OCI index — required
-          # by the size gate in docker-universal-manifest.
-          provenance: false
-          sbom: false
-          cache-from: type=gha,scope=universal-amd64
-          cache-to: type=gha,mode=max,scope=universal-amd64
+        env:
+          DOCKER_LABELS: ${{ steps.meta.outputs.labels }}
+        run: |
+          set -euo pipefail
+          metadata_file="$RUNNER_TEMP/docker-metadata.json"
+          label_args=()
+          while IFS= read -r label; do
+            [ -z "$label" ] && continue
+            label_args+=(--label "$label")
+          done <<<"$DOCKER_LABELS"
+
+          bash scripts/release/retry-ghcr-command.sh \
+            docker buildx build \
+            --file Dockerfile.universal \
+            --platform linux/amd64 \
+            --push \
+            --build-arg BASE_IMAGE="ghcr.io/kdlbs/kandev:${{ needs.prepare.outputs.tag }}" \
+            --tag "$STAGING_IMAGE" \
+            "${label_args[@]}" \
+            --provenance=false \
+            --sbom=false \
+            --cache-from type=gha,scope=universal-amd64 \
+            --cache-to type=gha,mode=max,scope=universal-amd64 \
+            --metadata-file "$metadata_file" \
+            .
+
+          digest="$(jq -er '."containerimage.digest"' "$metadata_file")"
+          echo "digest=$digest" >> "$GITHUB_OUTPUT"
 
   docker-universal-arm64:
     name: Build & push universal image (arm64)
-    needs: [prepare, docker-manifest]
+    needs: [prepare, docker-manifest, docker-universal-amd64]
     if: >-
       ${{ !cancelled() &&
         github.event_name == 'workflow_dispatch' &&
@@ -1668,7 +1721,8 @@ jobs:
         !inputs.dry_run &&
         !inputs.desktop_validation_only &&
         needs.prepare.result == 'success' &&
-        needs.docker-manifest.result == 'success'
+        needs.docker-manifest.result == 'success' &&
+        needs.docker-universal-amd64.result == 'success'
       }}
     runs-on: ubuntu-24.04-arm
     permissions:
@@ -1699,22 +1753,38 @@ jobs:
         with:
           images: ghcr.io/kdlbs/kandev
 
+      - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
+
       - name: Build and push (universal arm64 staging tag)
         id: build
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
-        with:
-          context: .
-          file: Dockerfile.universal
-          push: true
-          platforms: linux/arm64
-          build-args: |
-            BASE_IMAGE=ghcr.io/kdlbs/kandev:${{ needs.prepare.outputs.tag }}
-          tags: ${{ env.STAGING_IMAGE }}
-          labels: ${{ steps.meta.outputs.labels }}
-          provenance: false
-          sbom: false
-          cache-from: type=gha,scope=universal-arm64
-          cache-to: type=gha,mode=max,scope=universal-arm64
+        env:
+          DOCKER_LABELS: ${{ steps.meta.outputs.labels }}
+        run: |
+          set -euo pipefail
+          metadata_file="$RUNNER_TEMP/docker-metadata.json"
+          label_args=()
+          while IFS= read -r label; do
+            [ -z "$label" ] && continue
+            label_args+=(--label "$label")
+          done <<<"$DOCKER_LABELS"
+
+          bash scripts/release/retry-ghcr-command.sh \
+            docker buildx build \
+            --file Dockerfile.universal \
+            --platform linux/arm64 \
+            --push \
+            --build-arg BASE_IMAGE="ghcr.io/kdlbs/kandev:${{ needs.prepare.outputs.tag }}" \
+            --tag "$STAGING_IMAGE" \
+            "${label_args[@]}" \
+            --provenance=false \
+            --sbom=false \
+            --cache-from type=gha,scope=universal-arm64 \
+            --cache-to type=gha,mode=max,scope=universal-arm64 \
+            --metadata-file "$metadata_file" \
+            .
+
+          digest="$(jq -er '."containerimage.digest"' "$metadata_file")"
+          echo "digest=$digest" >> "$GITHUB_OUTPUT"
 
   docker-universal-manifest:
     name: Gate size + promote universal manifest
@@ -1739,6 +1809,11 @@ jobs:
       AMD64_REF: ghcr.io/kdlbs/kandev@${{ needs.docker-universal-amd64.outputs.digest }}
       ARM64_REF: ghcr.io/kdlbs/kandev@${{ needs.docker-universal-arm64.outputs.digest }}
     steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ needs.prepare.outputs.ref }}
+          persist-credentials: false
+
       - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
 
       - name: Login to GHCR
@@ -1778,7 +1853,8 @@ jobs:
       - name: Promote staging to final tags
         run: |
           set -euo pipefail
-          docker buildx imagetools create \
+          bash scripts/release/retry-ghcr-command.sh \
+            docker buildx imagetools create \
             --tag "ghcr.io/kdlbs/kandev:${{ needs.prepare.outputs.version }}-universal" \
             --tag "ghcr.io/kdlbs/kandev:${{ needs.prepare.outputs.tag }}-universal" \
             --tag "ghcr.io/kdlbs/kandev:universal" \

Makefile

@@ -547,6 +547,7 @@ test-scripts:
 	@python3 scripts/playwright-blob-audit.test.py
 	@bash scripts/release-desktop.test.sh
 	@bash scripts/release/runtime-bundle.test.sh
+	@bash scripts/release/retry-ghcr-command.test.sh
 	@node --test apps/desktop/e2e/desktop-launch-smoke.test.mjs
 	@python3 .github/scripts/release-workflow-contract_test.py
 	@node --test scripts/release/nightly-version.test.mjs scripts/release/nightly-release.test.mjs scripts/release/npm-view-version.test.mjs scripts/release/publish-npm.test.mjs

apps/backend/internal/mcp/handlers/step_complete_test.go

@@ -112,11 +112,16 @@ type stepCompleteSessionReadBarrier struct {
 }
 
 func (r *stepCompleteSessionReadBarrier) GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) {
+	// Capture the session before signaling readiness. Otherwise one caller can
+	// pass the barrier, claim the signal, and let the other caller read the
+	// already-populated bag after release, turning the concurrency test into a
+	// publish-retry test.
+	session, err := r.Repository.GetTaskSession(ctx, id)
 	if r.reads.Add(1) <= 2 {
 		r.ready <- struct{}{}
 		<-r.release
 	}
-	return r.Repository.GetTaskSession(ctx, id)
+	return session, err
 }
 
 type concurrentStepCompleteEventBus struct {

docs/plans/release-ghcr-secondary-limit/plan.md

@@ -0,0 +1,72 @@
+---
+spec: docs/specs/release-ghcr-secondary-limit/spec.md
+created: 2026-08-12
+status: complete
+---
+
+# Implementation Plan: Resilient GHCR release publishing
+
+## Overview
+
+Run #132 confirmed that the ARM64 `docker/build-push-action` can receive a 403 secondary-rate-limit response from GHCR after the image layers are built. The repair adds one tested local retry helper, uses it for GHCR image and manifest mutations, and serializes the two architecture publish stages so concurrent registry writes are reduced. The release recovery guide will document the safe rerun and backfill paths.
+
+## Release workflow
+
+- Add `scripts/release/retry-ghcr-command.sh`, a command wrapper with four total attempts, 60/120/240 second exponential waits, streamed output, transient-response matching, and preserved exit status.
+- Add a shell regression test with a fake command that covers transient-then-success, persistent transient failure, and immediate non-transient failure. Override delays in the test so it never sleeps for production intervals.
+- Replace the four per-architecture `docker/build-push-action` invocations in `.github/workflows/release.yml` with equivalent `docker buildx build` commands wrapped by the helper. Use BuildKit metadata output to preserve each job's `digest` output.
+- Run the pinned GitHub runtime helper before each inline Buildx build so the existing `type=gha` cache settings receive their runtime token and URL.
+- Wrap the base manifest `docker buildx imagetools create` and universal promotion command with the same helper.
+- Make the base ARM64 job depend on the successful base AMD64 job, and make the universal ARM64 job depend on the successful universal AMD64 job. Update their conditions and the contract test's dependency map.
+- Add the new helper and test to the release validation paths in `.github/workflows/lint-action-pinning.yml` and `Makefile`.
+
+## Tests
+
+- **What:** transient registry errors retry with the specified bounded backoff, permanent errors fail immediately, and final status is preserved.
+  **File:** `scripts/release/retry-ghcr-command.test.sh`
+  **How:** execute the real helper against deterministic fake commands with test-only delay overrides.
+- **What:** all GHCR mutating workflow steps use the helper, retain digest outputs, serialize architecture stages, and keep downstream success gates.
+  **File:** `.github/scripts/release-workflow-contract_test.py`
+  **How:** extend the existing workflow contract tests with exact helper, output, dependency, and condition assertions.
+- **Targeted checks:**
+  ```bash
+  bash scripts/release/retry-ghcr-command.test.sh
+  bash -n scripts/release/retry-ghcr-command.sh
+  python3 .github/scripts/release-workflow-contract_test.py
+  python3 .github/scripts/lint-action-pinning.py
+  ```
+
+## Documentation
+
+Update `docs/public/release-process.md` in the partial-release recovery section with the secondary-limit procedure: wait for the throttle, rerun failed jobs, avoid a new normal bump after the tag exists, and use `backfill_tag` only for the existing latest tag.
+
+## Verification Results
+
+- `bash scripts/release/retry-ghcr-command.test.sh` — passed.
+- `bash -n scripts/release/retry-ghcr-command.sh scripts/release/retry-ghcr-command.test.sh` — passed.
+- `python3 .github/scripts/release-workflow-contract_test.py` — passed, 28 tests.
+- `python3 .github/scripts/lint-action-pinning.py` — passed, all 18 workflow files use SHA-pinned action refs.
+- `python3 .github/scripts/lint-action-pinning_test.py` — passed, 9 tests.
+- `node --test scripts/validate-public-docs.test.mjs` — passed, 60 tests.
+- `node scripts/validate-public-docs.mjs` — passed, 41 published docs pages.
+- `zizmor .github/workflows` — audit completed with pre-existing findings outside this repair; no unrelated workflow changes were made.
+
+## Implementation Waves And Parallel Candidates
+
+Execute sequentially in the primary session:
+
+### Wave 1
+
+- [x] [task-01-retry-helper](task-01-retry-helper.md)
+
+### Wave 2
+
+- [x] [task-02-workflow-wiring](task-02-workflow-wiring.md)
+
+### Wave 3
+
+- [x] [task-03-release-recovery-docs](task-03-release-recovery-docs.md)
+
+## Open Questions
+
+None.

docs/plans/release-ghcr-secondary-limit/task-01-retry-helper.md

@@ -0,0 +1,55 @@
+---
+id: "01-retry-helper"
+title: "Add GHCR retry helper"
+status: done
+wave: 1
+depends_on: []
+plan: "plan.md"
+spec: "../../specs/release-ghcr-secondary-limit/spec.md"
+---
+
+# Task 01: Add GHCR retry helper
+
+## Acceptance
+
+- The helper retries only recognized secondary-rate-limit or equivalent throttling output, using four total attempts and 60/120/240 second waits by default.
+- Non-throttling command failures return immediately with the wrapped command's status; retry logs include attempt and delay information without secrets.
+- The shell regression test proves transient recovery, persistent failure, and non-transient failure without real waits.
+
+## Verification
+
+```bash
+bash scripts/release/retry-ghcr-command.test.sh
+bash -n scripts/release/retry-ghcr-command.sh
+```
+
+## Files likely touched
+
+- `scripts/release/retry-ghcr-command.sh`
+- `scripts/release/retry-ghcr-command.test.sh`
+
+## Dependencies
+
+None.
+
+## Parallelism
+
+sequential
+
+## Inputs
+
+- `docs/specs/release-ghcr-secondary-limit/spec.md`, especially What, Failure modes, and Scenarios.
+- `docs/plans/release-ghcr-secondary-limit/plan.md`, Wave 1.
+- Existing retry conventions in `scripts/release/npm-view-version.sh` and its test.
+
+## Output contract
+
+Report the helper behavior, files changed, exact test results, any blocker, and the updated task/plan status in the primary session.
+
+## Results
+
+Implemented `scripts/release/retry-ghcr-command.sh` and its regression test.
+
+- `bash scripts/release/retry-ghcr-command.test.sh` — passed.
+- `bash -n scripts/release/retry-ghcr-command.sh scripts/release/retry-ghcr-command.test.sh` — passed.
+- No external side effects; the test uses disposable commands and a temporary directory.

docs/plans/release-ghcr-secondary-limit/task-02-workflow-wiring.md

@@ -0,0 +1,62 @@
+---
+id: "02-workflow-wiring"
+title: "Wire retry into releases"
+status: done
+wave: 2
+depends_on: ["01-retry-helper"]
+plan: "plan.md"
+spec: "../../specs/release-ghcr-secondary-limit/spec.md"
+---
+
+# Task 02: Wire retry into releases
+
+## Acceptance
+
+- Every base/universal GHCR image build and manifest mutation uses the retry helper and preserves the digest output consumed by downstream jobs.
+- Base and universal architecture jobs are sequential, and all downstream manifest and publication conditions require successful dependencies.
+- Contract, action-pinning, and script-test workflow paths validate the new helper.
+
+## Verification
+
+```bash
+python3 .github/scripts/release-workflow-contract_test.py
+python3 .github/scripts/lint-action-pinning.py
+bash scripts/release/retry-ghcr-command.test.sh
+bash -n scripts/release/retry-ghcr-command.sh
+```
+
+## Files likely touched
+
+- `.github/workflows/release.yml`
+- `.github/scripts/release-workflow-contract_test.py`
+- `.github/workflows/lint-action-pinning.yml`
+- `Makefile`
+
+## Dependencies
+
+Task 01.
+
+## Parallelism
+
+sequential
+
+## Inputs
+
+- `docs/specs/release-ghcr-secondary-limit/spec.md`, especially What and Scenarios.
+- `docs/plans/release-ghcr-secondary-limit/plan.md`, Wave 2.
+- Existing Docker jobs and dependency assertions in `.github/workflows/release.yml` and `.github/scripts/release-workflow-contract_test.py`.
+
+## Output contract
+
+Report the workflow and contract changes, files changed, exact test results, any blocker, and the updated task/plan status in the primary session.
+
+## Results
+
+Replaced the four GHCR build action calls with retry-wrapped Buildx CLI commands, preserved BuildKit digest outputs, wrapped manifest mutations, and serialized each architecture pair.
+Added the pinned GitHub runtime helper before each inline build so the existing GHA cache backend receives its runtime credentials.
+
+- `python3 .github/scripts/release-workflow-contract_test.py` — passed, 28 tests.
+- `python3 .github/scripts/lint-action-pinning.py` — passed, all 18 workflow files use SHA-pinned action refs.
+- `python3 .github/scripts/lint-action-pinning_test.py` — passed, 9 tests.
+- `bash scripts/release/retry-ghcr-command.test.sh` — passed.
+- `bash -n scripts/release/retry-ghcr-command.sh` — passed.

docs/plans/release-ghcr-secondary-limit/task-03-release-recovery-docs.md

@@ -0,0 +1,53 @@
+---
+id: "03-release-recovery-docs"
+title: "Document release recovery"
+status: done
+wave: 3
+depends_on: ["02-workflow-wiring"]
+plan: "plan.md"
+spec: "../../specs/release-ghcr-secondary-limit/spec.md"
+---
+
+# Task 03: Document release recovery
+
+## Acceptance
+
+- The public release guide explains how to handle a GHCR secondary-rate-limit failure.
+- The guide tells maintainers to rerun failed jobs after the throttle clears and not to start a new normal version bump after the signed tag exists.
+- The guide points maintainers to `backfill_tag` for a recoverable partial release and retains the existing partial-release verification requirements.
+
+## Verification
+
+```bash
+node --test scripts/validate-public-docs.test.mjs
+node scripts/validate-public-docs.mjs
+```
+
+## Files likely touched
+
+- `docs/public/release-process.md`
+
+## Dependencies
+
+Task 02.
+
+## Parallelism
+
+sequential
+
+## Inputs
+
+- `docs/specs/release-ghcr-secondary-limit/spec.md`, especially Why, What, Failure modes, and Out of scope.
+- `docs/plans/release-ghcr-secondary-limit/plan.md`, Documentation section.
+- Existing partial-release recovery guidance in `docs/public/release-process.md`.
+
+## Output contract
+
+Report the documentation change, exact validation results, docs-impact classification, and the updated task/plan status in the primary session.
+
+## Results
+
+Added GHCR secondary-rate-limit recovery guidance to `docs/public/release-process.md`, including bounded workflow retries, rerunning the failed job after the throttle clears, avoiding a new version bump after the signed tag exists, and using `backfill_tag` for a recoverable partial release.
+
+- `node --test scripts/validate-public-docs.test.mjs` — passed, 60 tests.
+- `node scripts/validate-public-docs.mjs` — passed, 41 published docs pages.

docs/public/release-process.md

@@ -138,6 +138,8 @@ npm audit signatures
 
 First identify exactly which immutable and mutable channels succeeded. Preserve workflow logs, checksums, package versions, image digests, and signing output.
 
+If a GHCR build or manifest step returns a recognized registry-throttling response, such as HTTP 403 with a secondary-rate-limit message, HTTP 429, `TOOMANYREQUESTS`, or `too many requests`, treat it as transient registry throttling. The workflow retries these GHCR mutations with bounded 60/120/240-second backoff and publishes the two architectures sequentially. If the job still fails, wait a few minutes for the throttle to clear, then rerun the failed job in the same workflow run. Do not start a new normal version bump after the signed release tag exists. If the run remains partial, use `backfill_tag=vX.Y.Z` for the existing latest tag after confirming which channels already succeeded.
+
 If both tag push attempts fail after the release PR merges, the workflow logs the exact release merge commit. First confirm that `refs/tags/vX.Y.Z` is absent from the remote. Then, from an audited recovery context authorized by the tag ruleset and using the configured release signing identity, recreate and verify the tag at that exact commit:
 
 ```bash

docs/specs/release-ghcr-secondary-limit/spec.md

@@ -0,0 +1,44 @@
+---
+status: building
+created: 2026-08-12
+owner: Kandev maintainers
+---
+
+# Resilient GHCR release publishing
+
+## Why
+
+Stable release jobs can hit a transient GitHub Container Registry secondary rate limit after a valid image build has completed. The release should recover from that service response without asking a maintainer to start a second version bump or blindly repeat unrelated release work.
+
+## What
+
+- Per-architecture GHCR staging publishes retry recognized secondary-rate-limit failures with bounded exponential backoff: four total attempts with waits of 60, 120, and 240 seconds.
+- GHCR manifest creation and promotion retry the same recognized transient failures.
+- Non-rate-limit failures, including build errors and authentication or permission failures, fail immediately and preserve their original exit status.
+- Base-image architecture publishes run sequentially, and universal-image architecture publishes run sequentially. A manifest job runs only after both architecture jobs succeed.
+- A successful retry exposes the same image digest contract to downstream manifest jobs as a first-attempt success.
+- Every retry logs the attempt number and wait duration without printing credentials or tokens.
+- When all retry attempts fail, the workflow fails closed. It does not promote the affected staging image or run downstream release publication jobs.
+- Existing recovery semantics remain unchanged: maintainers rerun failed jobs after the throttle clears, or use `backfill_tag` for a partial release whose signed tag already exists.
+
+## Failure modes
+
+- A GHCR response containing a secondary-rate-limit or equivalent throttling signal causes the bounded retry sequence.
+- A persistent throttling response fails after the final attempt and blocks manifest promotion and release publication.
+- A build, authentication, permission, or other non-throttling error fails without waiting through the retry sequence.
+- A retry after a partially accepted staging push is safe because staging tags are unique to the workflow run and architecture.
+
+## Scenarios
+
+- **GIVEN** a per-architecture image command fails with a secondary-rate-limit response on its first attempt and succeeds on its second attempt, **WHEN** the release job runs, **THEN** the job waits according to the backoff policy, succeeds, and publishes the resulting digest for downstream use.
+- **GIVEN** a per-architecture image command fails with a non-throttling build or permission error, **WHEN** the release job runs, **THEN** the command fails without a retry delay and the dependent manifest job does not run.
+- **GIVEN** a per-architecture image command returns a secondary-rate-limit response on every attempt, **WHEN** all four attempts complete, **THEN** the job fails, no user-facing manifest is promoted, and GitHub Release, npm, and Homebrew publication do not run.
+- **GIVEN** a stable release reaches the base-image stage, **WHEN** the architecture jobs publish their staging images, **THEN** only one architecture publish is active at a time and the manifest job waits for both successful digests.
+- **GIVEN** a stable release reaches the universal-image stage, **WHEN** the architecture jobs publish their staging images, **THEN** only one universal architecture publish is active at a time and the promotion job waits for both successful digests.
+
+## Out of scope
+
+- Moving Kandev images to another registry.
+- Changing GHCR credentials, package visibility, or repository permissions.
+- Increasing GitHub primary API quotas or changing Docker Hub behavior.
+- Changing release versioning, tag immutability, or the `backfill_tag` contract.

scripts/release/retry-ghcr-command.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Retry a GHCR mutation only when the registry reports transient throttling.
+set -euo pipefail
+
+if (( $# == 0 )); then
+  echo "Usage: $0 <command> [args...]" >&2
+  exit 2
+fi
+
+MAX_ATTEMPTS="${RETRY_GHCR_MAX_ATTEMPTS:-4}"
+INITIAL_DELAY_SECONDS="${RETRY_GHCR_INITIAL_DELAY_SECONDS:-60}"
+SLEEP_COMMAND="${RETRY_GHCR_SLEEP_COMMAND:-sleep}"
+
+if [[ ! "$MAX_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then
+  echo "RETRY_GHCR_MAX_ATTEMPTS must be a positive integer" >&2
+  exit 2
+fi
+if [[ ! "$INITIAL_DELAY_SECONDS" =~ ^[0-9]+$ ]]; then
+  echo "RETRY_GHCR_INITIAL_DELAY_SECONDS must be a non-negative integer" >&2
+  exit 2
+fi
+
+OUTPUT_FILE="$(mktemp)"
+trap 'rm -f "$OUTPUT_FILE"' EXIT
+
+retryable_failure() {
+  grep -Eiq \
+    'secondary[[:space:]-]+rate[[:space:]-]*limit|too many requests|status code 429|TOOMANYREQUESTS' \
+    "$OUTPUT_FILE"
+}
+
+retry_delay() {
+  local delay="$INITIAL_DELAY_SECONDS"
+  local step
+
+  for ((step = 1; step < $1; step++)); do
+    delay=$((delay * 2))
+  done
+  printf '%s\n' "$delay"
+}
+
+for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do
+  : >"$OUTPUT_FILE"
+  set +e
+  "$@" 2>&1 | tee "$OUTPUT_FILE"
+  status="${PIPESTATUS[0]}"
+  set -e
+
+  if (( status == 0 )); then
+    exit 0
+  fi
+
+  if (( attempt == MAX_ATTEMPTS )) || ! retryable_failure; then
+    exit "$status"
+  fi
+
+  delay="$(retry_delay "$attempt")"
+  echo "::warning::GHCR command attempt $attempt/$MAX_ATTEMPTS hit a transient rate limit; retrying in ${delay}s." >&2
+  if (( delay > 0 )); then
+    "$SLEEP_COMMAND" "$delay"
+  fi
+done

scripts/release/retry-ghcr-command.test.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SCRIPT="$ROOT_DIR/scripts/release/retry-ghcr-command.sh"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+fail() {
+  echo "FAIL: $*" >&2
+  exit 1
+}
+
+cat >"$TMP_DIR/transient-command" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+COUNT_FILE="${COUNT_FILE:?}"
+count=0
+if [[ -f "$COUNT_FILE" ]]; then
+  count="$(<"$COUNT_FILE")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" >"$COUNT_FILE"
+
+if ((count < 3)); then
+  echo 'denied: Error from intermediary with HTTP status code 403 "Forbidden"'
+  echo 'You have exceeded a secondary rate limit.'
+  exit 17
+fi
+
+echo 'success'
+EOF
+
+cat >"$TMP_DIR/permanent-command" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+echo 'permission_denied: insufficient package permission'
+exit 23
+EOF
+
+cat >"$TMP_DIR/persistent-command" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+echo 'You have exceeded a secondary rate limit.'
+exit 31
+EOF
+
+cat >"$TMP_DIR/sleep" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "$1" >>"${SLEEP_LOG:?}"
+EOF
+
+chmod +x \
+  "$TMP_DIR/transient-command" \
+  "$TMP_DIR/permanent-command" \
+  "$TMP_DIR/persistent-command" \
+  "$TMP_DIR/sleep"
+
+COUNT_FILE="$TMP_DIR/transient-count" \
+SLEEP_LOG="$TMP_DIR/transient-sleeps" \
+RETRY_GHCR_SLEEP_COMMAND="$TMP_DIR/sleep" \
+RETRY_GHCR_MAX_ATTEMPTS=4 \
+RETRY_GHCR_INITIAL_DELAY_SECONDS=60 \
+  bash "$SCRIPT" "$TMP_DIR/transient-command" >"$TMP_DIR/transient-output"
+
+grep -q '^success$' "$TMP_DIR/transient-output" || \
+  fail "transient failure was not retried to success"
+[[ "$(<"$TMP_DIR/transient-count")" == "3" ]] || \
+  fail "expected two failed attempts before success"
+diff -u <(printf '60\n120\n') "$TMP_DIR/transient-sleeps" || \
+  fail "retry delays did not use exponential backoff"
+
+set +e
+RETRY_GHCR_MAX_ATTEMPTS=4 \
+RETRY_GHCR_INITIAL_DELAY_SECONDS=0 \
+  bash "$SCRIPT" "$TMP_DIR/persistent-command" >"$TMP_DIR/persistent-output" 2>"$TMP_DIR/persistent-error"
+persistent_status="$?"
+set -e
+[[ "$persistent_status" -eq 31 ]] || \
+  fail "persistent transient failure did not preserve the command status"
+[[ "$(wc -l <"$TMP_DIR/persistent-output")" -eq 4 ]] || \
+  fail "persistent transient failure did not use all four attempts"
+
+set +e
+RETRY_GHCR_SLEEP_COMMAND="$TMP_DIR/sleep" \
+RETRY_GHCR_MAX_ATTEMPTS=4 \
+RETRY_GHCR_INITIAL_DELAY_SECONDS=0 \
+  bash "$SCRIPT" "$TMP_DIR/permanent-command" >"$TMP_DIR/permanent-output" 2>"$TMP_DIR/permanent-error"
+permanent_status="$?"
+set -e
+if [[ "$permanent_status" -ne 23 ]]; then
+  fail "permanent failure did not preserve the command status"
+fi
+
+if grep -q 'retrying' "$TMP_DIR/permanent-error"; then
+  fail "permanent failure was retried but should have failed immediately"
+fi
+
+echo "PASS: GHCR retry helper retries secondary limits with bounded backoff"