diff --git a/.agents/skills/release/SKILL.md b/.agents/skills/release/SKILL.md index 5985528a92..e1a29c4065 100644 --- a/.agents/skills/release/SKILL.md +++ b/.agents/skills/release/SKILL.md @@ -63,6 +63,15 @@ complete only after `publish-release`, `publish-npm`, `update-homebrew-tap`, and `update-scoop-bucket` each succeed and their artifacts are verified—an aggregate green run can hide skipped publication jobs. +Required web, runtime, and desktop artifact uploads attempt up to three times, +with waits of 30 seconds and 60 seconds between attempts. They fail explicitly +when an expected file is missing. Desktop matrix targets use `fail-fast: false` +so a transient upload failure does not cancel sibling targets, but publication +still requires the complete matrix to succeed. Rerun a failed producer job in +the same workflow run after a transient failure. If the signed tag already +exists and the run remains partial, use `backfill_tag` for that tag after +checking which channels already succeeded. + Stable has no local release driver; the entire Stable flow runs in GHA. The Nightly metadata and publication revalidation state machine lives in `scripts/release/nightly-release.sh`, which GHA invokes for scheduled and manual Nightly runs. diff --git a/.github/scripts/release-workflow-contract_test.py b/.github/scripts/release-workflow-contract_test.py index 0b291c633a..9f4db14fd0 100644 --- a/.github/scripts/release-workflow-contract_test.py +++ b/.github/scripts/release-workflow-contract_test.py @@ -172,6 +172,48 @@ class ReleaseWorkflowContractTest(unittest.TestCase): self.assertIn(archive, package) self.assertLess(package.index(validation), package.index(archive)) + def assert_required_artifact_upload_retries( + self, job_name: str, step_prefix: str + ) -> None: + block = job_block(job_name) + for attempt in (1, 2, 3): + step_id = f"{step_prefix}_{attempt}" + self.assertEqual(block.count(f"id: {step_id}"), 1) + if attempt > 1: + self.assertIn( + f"if: steps.{step_prefix}_{attempt - 1}.outcome == 'failure'", + block, + ) + + self.assertEqual(block.count("if-no-files-found: error"), 3) + self.assertEqual(block.count("overwrite: true"), 2) + self.assertIn("sleep 30", block) + self.assertIn("sleep 60", block) + self.assertIn("attempt 1/3 failed; retrying in 30 seconds.", block) + self.assertIn("attempt 2/3 failed; retrying in 60 seconds.", block) + normalized = " ".join(block.split()) + self.assertIn( + f"if: >- steps.{step_prefix}_1.outcome == 'failure' && " + f"steps.{step_prefix}_2.outcome == 'failure' && " + f"steps.{step_prefix}_3.outcome == 'failure'", + normalized, + ) + + def test_required_artifact_uploads_retry_and_desktop_matrix_isolated(self) -> None: + self.assert_required_artifact_upload_retries("build-web", "upload_web_bundle") + self.assert_required_artifact_upload_retries( + "build-bundles", "upload_runtime_bundle" + ) + self.assert_required_artifact_upload_retries( + "build-desktop", "upload_desktop_artifacts" + ) + + desktop = job_block("build-desktop") + self.assertIn("fail-fast: false", desktop) + self.assertIn( + "needs.build-desktop.result == 'success'", job_block("publish-release") + ) + def test_stable_jobs_continue_past_skipped_nightly_branch_only_after_successful_needs( self, ) -> None: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 18c4790d3a..c1328dabe6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -745,12 +745,61 @@ jobs: VITE_KANDEV_API_PORT: "" VITE_KANDEV_DEBUG: "" run: pnpm -C apps --filter @kandev/web build - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Upload web bundle (attempt 1) + id: upload_web_bundle_1 + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: web-bundle path: apps/web/dist + if-no-files-found: error include-hidden-files: true + - name: Wait before web bundle upload retry (attempt 2) + if: steps.upload_web_bundle_1.outcome == 'failure' + run: | + echo "::warning::web-bundle upload attempt 1/3 failed; retrying in 30 seconds." + sleep 30 + + - name: Upload web bundle (attempt 2) + id: upload_web_bundle_2 + if: steps.upload_web_bundle_1.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: web-bundle + path: apps/web/dist + if-no-files-found: error + include-hidden-files: true + overwrite: true + + - name: Wait before web bundle upload retry (attempt 3) + if: steps.upload_web_bundle_2.outcome == 'failure' + run: | + echo "::warning::web-bundle upload attempt 2/3 failed; retrying in 60 seconds." + sleep 60 + + - name: Upload web bundle (attempt 3) + id: upload_web_bundle_3 + if: steps.upload_web_bundle_2.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: web-bundle + path: apps/web/dist + if-no-files-found: error + include-hidden-files: true + overwrite: true + + - name: Fail after web bundle upload retries + if: >- + steps.upload_web_bundle_1.outcome == 'failure' && + steps.upload_web_bundle_2.outcome == 'failure' && + steps.upload_web_bundle_3.outcome == 'failure' + run: | + echo "::error::web-bundle upload failed after 3 attempts." >&2 + exit 1 + build-bundles: name: Build platform bundle (${{ matrix.platform }}) needs: [prepare, nightly-prepare, build-web] @@ -874,17 +923,70 @@ jobs: shasum -a 256 "kandev-${{ matrix.platform }}.zip" > "kandev-${{ matrix.platform }}.zip.sha256" fi - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Upload runtime bundle (attempt 1) + id: upload_runtime_bundle_1 + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bundle-${{ matrix.platform }} # The zip entries match nothing on the four non-Windows platforms. - # if-no-files-found is left at its default so a missing tarball is - # still an error. path: | dist/kandev-${{ matrix.platform }}.tar.gz dist/kandev-${{ matrix.platform }}.tar.gz.sha256 dist/kandev-${{ matrix.platform }}.zip dist/kandev-${{ matrix.platform }}.zip.sha256 + if-no-files-found: error + + - name: Wait before runtime bundle upload retry (attempt 2) + if: steps.upload_runtime_bundle_1.outcome == 'failure' + run: | + echo "::warning::${{ matrix.platform }} runtime bundle upload attempt 1/3 failed; retrying in 30 seconds." + sleep 30 + + - name: Upload runtime bundle (attempt 2) + id: upload_runtime_bundle_2 + if: steps.upload_runtime_bundle_1.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bundle-${{ matrix.platform }} + path: | + dist/kandev-${{ matrix.platform }}.tar.gz + dist/kandev-${{ matrix.platform }}.tar.gz.sha256 + dist/kandev-${{ matrix.platform }}.zip + dist/kandev-${{ matrix.platform }}.zip.sha256 + if-no-files-found: error + overwrite: true + + - name: Wait before runtime bundle upload retry (attempt 3) + if: steps.upload_runtime_bundle_2.outcome == 'failure' + run: | + echo "::warning::${{ matrix.platform }} runtime bundle upload attempt 2/3 failed; retrying in 60 seconds." + sleep 60 + + - name: Upload runtime bundle (attempt 3) + id: upload_runtime_bundle_3 + if: steps.upload_runtime_bundle_2.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bundle-${{ matrix.platform }} + path: | + dist/kandev-${{ matrix.platform }}.tar.gz + dist/kandev-${{ matrix.platform }}.tar.gz.sha256 + dist/kandev-${{ matrix.platform }}.zip + dist/kandev-${{ matrix.platform }}.zip.sha256 + if-no-files-found: error + overwrite: true + + - name: Fail after runtime bundle upload retries + if: >- + steps.upload_runtime_bundle_1.outcome == 'failure' && + steps.upload_runtime_bundle_2.outcome == 'failure' && + steps.upload_runtime_bundle_3.outcome == 'failure' + run: | + echo "::error::${{ matrix.platform }} runtime bundle upload failed after 3 attempts." >&2 + exit 1 build-desktop: name: Build desktop app (${{ matrix.platform }}) @@ -898,6 +1000,7 @@ jobs: needs.build-bundles.result == 'success' }} strategy: + fail-fast: false matrix: include: - platform: macos-arm64 @@ -1428,10 +1531,57 @@ jobs: fi ls -lh "$out_dir" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Upload desktop artifacts (attempt 1) + id: upload_desktop_artifacts_1 + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: desktop-${{ matrix.platform }} path: dist/desktop-artifacts/* + if-no-files-found: error + + - name: Wait before desktop artifact upload retry (attempt 2) + if: steps.upload_desktop_artifacts_1.outcome == 'failure' + run: | + echo "::warning::${{ matrix.platform }} desktop artifact upload attempt 1/3 failed; retrying in 30 seconds." + sleep 30 + + - name: Upload desktop artifacts (attempt 2) + id: upload_desktop_artifacts_2 + if: steps.upload_desktop_artifacts_1.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: desktop-${{ matrix.platform }} + path: dist/desktop-artifacts/* + if-no-files-found: error + overwrite: true + + - name: Wait before desktop artifact upload retry (attempt 3) + if: steps.upload_desktop_artifacts_2.outcome == 'failure' + run: | + echo "::warning::${{ matrix.platform }} desktop artifact upload attempt 2/3 failed; retrying in 60 seconds." + sleep 60 + + - name: Upload desktop artifacts (attempt 3) + id: upload_desktop_artifacts_3 + if: steps.upload_desktop_artifacts_2.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: desktop-${{ matrix.platform }} + path: dist/desktop-artifacts/* + if-no-files-found: error + overwrite: true + + - name: Fail after desktop artifact upload retries + if: >- + steps.upload_desktop_artifacts_1.outcome == 'failure' && + steps.upload_desktop_artifacts_2.outcome == 'failure' && + steps.upload_desktop_artifacts_3.outcome == 'failure' + run: | + echo "::error::${{ matrix.platform }} desktop artifact upload failed after 3 attempts." >&2 + exit 1 # --------------------------------------------------------------------------- # Docker base image — split per arch onto native runners, then stitched into diff --git a/AGENTS.md b/AGENTS.md index 63c9d83e95..f14087d87d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,7 +73,7 @@ The commitlint hook caps the header at **100 characters** (`type(scope): descrip ### Release & Versioning -Stable Kandev releases use one **SemVer** `X.Y.Z` across npm, Homebrew, Scoop, GitHub Releases, Desktop, and containers. Scheduled npm-only Nightlies use `X.Y.(Z+1)-nightly.sha<12-hex>` without moving any Stable channel. Both flows run in `.github/workflows/release.yml`. A normal Stable release uses the protected `RELEASE_PR_BYPASS_TOKEN` environment secret only for its administrator PR merge. A Stable release is complete only after the GitHub Release, npm, Homebrew, and Scoop publications succeed and are verified. Full details are in the `/release` skill — load it when cutting a release, changing version channels, or debugging release artifacts. +Stable Kandev releases use one **SemVer** `X.Y.Z` across npm, Homebrew, Scoop, GitHub Releases, Desktop, and containers. Scheduled npm-only Nightlies use `X.Y.(Z+1)-nightly.sha<12-hex>` without moving any Stable channel. Both flows run in `.github/workflows/release.yml`. Required web, runtime, and desktop artifact uploads retry up to three total attempts with 30-second and 60-second waits; desktop matrix jobs continue independently, while publication still requires every target to succeed. A normal Stable release uses the protected `RELEASE_PR_BYPASS_TOKEN` environment secret only for its administrator PR merge. A Stable release is complete only after the GitHub Release, npm, Homebrew, and Scoop publications succeed and are verified. Full details are in the `/release` skill — load it when cutting a release, changing version channels, or debugging release artifacts. ### Code Quality diff --git a/docs/plans/release-artifact-upload-recovery/plan.md b/docs/plans/release-artifact-upload-recovery/plan.md new file mode 100644 index 0000000000..e2ebe223d7 --- /dev/null +++ b/docs/plans/release-artifact-upload-recovery/plan.md @@ -0,0 +1,104 @@ +--- +created: 2026-09-19 +status: implemented +requirements: + - REQ-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001 +system_design: + - ../../specs/release/system-design/artifact-upload-recovery.md +legacy_specs: [] +--- + +# Implementation Plan: Resilient Release Artifact Uploads + +## Overview + +Harden the release workflow against transient GitHub Actions artifact-service +failures. The implementation adds bounded retries to required build artifact +uploads, makes missing outputs fail explicitly, and lets all desktop matrix +targets finish before the release gate evaluates the result. Publication keeps +its existing all-target success requirement. + +The work is sequential: update the workflow and its contract tests first, then +bring the public and internal release guidance in line with the behavior. + +## Scope + +### In scope + +- Required web, runtime bundle, and desktop artifact uploads. +- Three total upload attempts with 30-second and 60-second waits. +- Explicit missing-file failures for required uploads. +- `build-desktop` matrix isolation with `fail-fast: false`. +- Contract tests, action security checks, and maintainer recovery guidance. + +### Out of scope + +- Artifact download retries. +- GHCR, npm, Homebrew, Scoop, or GitHub Release API retry policies. +- Automatic backfill dispatch or tag/version behavior. +- Changes to artifact contents or publication ordering. + +## Technical approach + +### Workflow and test contract + +Update `.github/workflows/release.yml` with a repeated, explicit upload pattern +for the three required producer paths. Each pattern records the action outcome, +waits only after failure, retries with `overwrite: true`, and fails through a +final gate when all attempts fail. Set `if-no-files-found: error` for required +uploads. Set `strategy.fail-fast: false` for the desktop matrix while retaining +the existing downstream `needs.*.result == 'success'` conditions. + +Extend `.github/scripts/release-workflow-contract_test.py` to assert the retry +budget, wait durations, overwrite behavior, required-file checks, desktop +matrix setting, and fail-closed publication gates. + +### Documentation + +Update `docs/public/release-process.md`, `AGENTS.md`, and +`.agents/skills/release/SKILL.md` with the retry and matrix behavior. Keep the +existing backfill instructions and state that retries do not make an incomplete +release publishable. + +## Tests + +- **Retry contract:** `.github/scripts/release-workflow-contract_test.py` + asserts all required upload sites use the bounded retry and missing-file + contract. +- **Action pinning:** `.github/scripts/lint-action-pinning_test.py` and + `.github/scripts/lint-action-pinning.py` verify every workflow action remains + pinned. +- **Workflow security:** `zizmor .github/workflows` checks the changed workflow + for unsafe GitHub Actions patterns. +- **Public documentation:** `scripts/validate-public-docs.test.mjs` and + `scripts/validate-public-docs.mjs` validate the release guide. +- **Harness documentation:** `scripts/lint-harness-files.test.py` and + `.github/scripts/lint-harness-files.py --all` validate the updated guidance. + +## Work orders + +- [x] [Task 01: Add resilient artifact uploads](task-01-resilient-artifact-uploads.md) +- [x] [Task 02: Update release recovery guidance](task-02-release-recovery-guidance.md) + +## Verification results + +Task 01 and Task 02 are implemented and verified. + +- `python3 .github/scripts/release-workflow-contract_test.py`: 33 tests passed. +- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed. +- `python3 .github/scripts/lint-action-pinning.py`: all 24 workflow files passed. +- `zizmor .github/workflows`: the command completed with the repository's + existing 233 findings and exit code 14. The changed `release.yml` had 27 + findings before and after this change, so the retry blocks added no new + findings. +- Public documentation validators: 62 tests passed and 47 pages validated. +- Harness validators: 19 tests passed and 199 files passed. +- All scoped `git diff --check` commands passed. + +## Risks + +- Each failed upload can add up to 90 seconds before the job fails. +- Repeated YAML patterns can drift if a future required artifact upload is added + without extending the contract test. +- `fail-fast: false` can consume more runner time after a real desktop build + failure, but preserves diagnostics and independent target results. diff --git a/docs/plans/release-artifact-upload-recovery/task-01-resilient-artifact-uploads.md b/docs/plans/release-artifact-upload-recovery/task-01-resilient-artifact-uploads.md new file mode 100644 index 0000000000..b49aa10741 --- /dev/null +++ b/docs/plans/release-artifact-upload-recovery/task-01-resilient-artifact-uploads.md @@ -0,0 +1,106 @@ +--- +id: "01-resilient-artifact-uploads" +title: "Add resilient artifact uploads" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001 +acceptance_criteria: + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.1 + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.2 + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.3 + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.4 + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.5 +system_design: + - ../../specs/release/system-design/artifact-upload-recovery.md +--- + +# Task 01: Add resilient artifact uploads + +## Summary + +Update the Stable and shared runtime release workflow so transient required +artifact upload failures receive two bounded retries. Keep missing outputs and +persistent upload failures fail-closed, and let all desktop matrix targets +finish before the publication gate evaluates the result. + +## In scope + +- Add the three-attempt upload pattern to `build-web`, `build-bundles`, and + `build-desktop` required artifact uploads. +- Set `if-no-files-found: error` on those required uploads. +- Set `build-desktop.strategy.fail-fast: false`. +- Add static contract coverage for the retry, matrix, and publication behavior. + +## Out of scope + +- Artifact download retries. +- External publication retry policies. +- Automatic backfill dispatch or release tag changes. + +## Acceptance + +- Each required upload logs and performs at most three attempts with 30-second + and 60-second waits, using `overwrite: true` on retries. +- Missing required files and three failed attempts make the producer job fail. +- A failed desktop target does not cancel sibling targets, and publication + remains gated on the complete desktop matrix. + +## Verification + +```bash +python3 .github/scripts/release-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +python3 .github/scripts/lint-action-pinning.py +python3 scripts/list-docs.py validate +python3 scripts/lint-spec-files.py --all +zizmor .github/workflows +git diff --check -- .github/workflows/release.yml .github/scripts/release-workflow-contract_test.py +``` + +## Files likely touched + +- `.github/workflows/release.yml` +- `.github/scripts/release-workflow-contract_test.py` + +## Dependencies + +None. + +## Risks + +- A persistent upload failure can add up to 90 seconds to a producer job. +- The repeated workflow pattern must remain synchronized across required upload + sites. + +## Parallelism + +`sequential` + +## Inputs + +- `docs/specs/release/requirements/artifact-upload-recovery.md` +- `docs/specs/release/system-design/artifact-upload-recovery.md` +- The existing release workflow and its contract tests. + +## Results + +Implemented the three-attempt upload pattern for web, runtime, and desktop +artifacts, with explicit missing-file failures, bounded waits, retry logging, +and fail-closed final gates. The desktop matrix now uses `fail-fast: false` +while `publish-release` still requires a successful desktop matrix. + +Verification passed: + +- `python3 .github/scripts/release-workflow-contract_test.py`: 33 tests. +- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests. +- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows. +- `python3 scripts/list-docs.py validate`: 291 decisions and 1034 + specifications. +- `python3 scripts/lint-spec-files.py --all`: all specification files passed. +- `git diff --check -- .github/workflows/release.yml .github/scripts/release-workflow-contract_test.py`. + +`zizmor .github/workflows` completed with its existing repository findings. +The changed `release.yml` had 27 findings before and after this work. diff --git a/docs/plans/release-artifact-upload-recovery/task-02-release-recovery-guidance.md b/docs/plans/release-artifact-upload-recovery/task-02-release-recovery-guidance.md new file mode 100644 index 0000000000..e4fcb8aec7 --- /dev/null +++ b/docs/plans/release-artifact-upload-recovery/task-02-release-recovery-guidance.md @@ -0,0 +1,94 @@ +--- +id: "02-release-recovery-guidance" +title: "Update release recovery guidance" +status: done +wave: 2 +depends_on: + - "01-resilient-artifact-uploads" +plan: "plan.md" +requirements: + - REQ-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001 +acceptance_criteria: + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.4 + - AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.5 +system_design: + - ../../specs/release/system-design/artifact-upload-recovery.md +--- + +# Task 02: Update release recovery guidance + +## Summary + +Document the artifact upload retry budget, desktop matrix behavior, and the +existing backfill path so maintainers can distinguish an exhausted retry budget +from a successful release. Keep the instructions consistent across public +release documentation, repository guidance, and the release skill. + +## In scope + +- Update `docs/public/release-process.md` with the operational retry and + recovery behavior. +- Update the Release and Versioning guidance in `AGENTS.md`. +- Update `.agents/skills/release/SKILL.md` with the same contract. + +## Out of scope + +- New release commands or workflow inputs. +- Changes to public installation or runtime behavior. +- Repeating the full release workflow architecture in the public guide. + +## Acceptance + +- The public release guide states the bounded retry behavior and preserves the + `backfill_tag` guidance for an exhausted or partial release. +- Internal release guidance states that retries remain fail-closed and that all + desktop targets continue independently. +- The documentation validators and harness lint pass. + +## Verification + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +python3 scripts/lint-harness-files.test.py +python3 .github/scripts/lint-harness-files.py --all +git diff --check -- docs/public/release-process.md AGENTS.md .agents/skills/release/SKILL.md +``` + +## Files likely touched + +- `docs/public/release-process.md` +- `AGENTS.md` +- `.agents/skills/release/SKILL.md` + +## Dependencies + +Task 01, so the documentation describes the final workflow contract. + +## Risks + +None. + +## Parallelism + +`sequential` + +## Inputs + +- `docs/specs/release/requirements/artifact-upload-recovery.md` +- `docs/specs/release/system-design/artifact-upload-recovery.md` +- `docs/decisions/0029-release-backfill-and-desktop-diagnostics.md` + +## Results + +Updated the public release process, root engineering guidance, and release +skill with the retry budget, missing-file behavior, desktop matrix isolation, +publication gate, failed-job rerun, and `backfill_tag` recovery guidance. + +Verification passed: + +- `node --test scripts/validate-public-docs.test.mjs`: 62 tests. +- `node scripts/validate-public-docs.mjs`: 47 pages. +- `python3 scripts/lint-harness-files.test.py`: 19 tests. +- `python3 .github/scripts/lint-harness-files.py --all`: 199 files. +- `git diff --check -- docs/public/release-process.md AGENTS.md .agents/skills/release/SKILL.md`. diff --git a/docs/public/release-process.md b/docs/public/release-process.md index bbff8372a4..f2aef8a2dc 100644 --- a/docs/public/release-process.md +++ b/docs/public/release-process.md @@ -177,6 +177,8 @@ npm audit signatures First identify exactly which immutable and mutable channels succeeded. Preserve workflow logs, checksums, package versions, image digests, and signing output. +Required web, runtime, and desktop artifact uploads make up to three total attempts. The workflow waits 30 seconds before the second attempt and 60 seconds before the third attempt. Each required upload also fails when an expected file is missing. Desktop targets run independently, so one upload failure does not cancel the other matrix targets. Publication remains blocked until every required target succeeds. If all attempts fail, rerun the failed job in the same workflow run. If the signed tag already exists and the run remains partial, use `backfill_tag=vX.Y.Z` after confirming which channels already succeeded. + If a GHCR build or manifest step returns a recognized registry-throttling response, such as HTTP 403 with a secondary-rate-limit message, HTTP 429, `TOOMANYREQUESTS`, or `too many requests`, treat it as transient registry throttling. The workflow retries these GHCR mutations with bounded 60/120/240-second backoff and publishes the two architectures sequentially. If the job still fails, wait a few minutes for the throttle to clear, then rerun the failed job in the same workflow run. Do not start a new normal version bump after the signed release tag exists. If the run remains partial, use `backfill_tag=vX.Y.Z` for the existing latest tag after confirming which channels already succeeded. If both tag push attempts fail after the release PR merges, the workflow logs the exact release merge commit. First confirm that `refs/tags/vX.Y.Z` is absent from the remote. Then, from an audited recovery context authorized by the tag ruleset and using the configured release signing identity, recreate and verify the tag at that exact commit: diff --git a/docs/specs/release/requirements/artifact-upload-recovery.md b/docs/specs/release/requirements/artifact-upload-recovery.md new file mode 100644 index 0000000000..55258f3950 --- /dev/null +++ b/docs/specs/release/requirements/artifact-upload-recovery.md @@ -0,0 +1,62 @@ +--- +status: active +system: release +created: 2026-09-19 +owners: + - kandev +--- + +# Stable Release Artifact Upload Recovery Requirements + +## Overview + +The release workflow transfers required build outputs between jobs through +GitHub Actions artifacts. A transient artifact service or network failure must +not immediately discard an otherwise valid platform build, while a persistent +or incomplete upload must still block publication. + +The release system owns this contract because it owns the artifact set, +publication gates, and recovery path for Stable and shared runtime builds. + +## Terminology + +- **Required artifact:** A web bundle, platform runtime bundle, or desktop + artifact consumed by a later release job. +- **Upload attempt:** One invocation of the pinned GitHub artifact upload action + for one required artifact name. +- **Backfill:** A Stable workflow run that reuses the latest existing signed + release tag to repair missing artifacts or publication outputs. + +## Requirements + +### REQ-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001: Recover transient required artifact upload failures + +**Intent:** Preserve valid release builds when GitHub's artifact service has a +transient failure, while keeping incomplete releases from reaching publication. + +#### Acceptance criteria + +- **AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.1:** When a required artifact upload + fails, the workflow shall make no more than three total upload attempts for + that artifact, waiting 30 seconds before the second attempt and 60 seconds + before the third attempt. +- **AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.2:** When a required artifact path + contains no files, the upload step shall fail explicitly; retry behavior shall + not turn a missing build output into a successful artifact. +- **AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.3:** When one Stable desktop matrix + target fails, the other desktop targets shall continue to completion, while + release publication shall still require every desktop target to succeed. +- **AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.4:** When all upload attempts for a + required artifact fail, the producing job shall fail and GitHub Release, npm, + Homebrew, and Scoop publication jobs shall not run from that incomplete run. +- **AC-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001.5:** Each retry shall log its attempt + and wait duration without exposing credentials, tokens, or signing material. + +## Out of scope + +- Retrying artifact downloads or external package-manager publication calls. +- Automatically starting a new workflow run or choosing a `backfill_tag`. +- Changing release versioning, tag signing, artifact contents, or publication + ordering. +- Treating a partial release as complete when a required artifact remains + unavailable. diff --git a/docs/specs/release/system-design/artifact-upload-recovery.md b/docs/specs/release/system-design/artifact-upload-recovery.md new file mode 100644 index 0000000000..186496ed0e --- /dev/null +++ b/docs/specs/release/system-design/artifact-upload-recovery.md @@ -0,0 +1,113 @@ +--- +status: current +system: release +requirements: + - REQ-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001 +--- + +# Stable Release Artifact Upload Recovery System Design + +## Purpose and boundaries + +The release system owns the GitHub Actions artifact handoff between build jobs +and publication jobs. This design covers required artifact uploads in +`build-web`, `build-bundles`, and `build-desktop`, plus the desktop matrix +failure policy. It uses the existing `backfill_tag` recovery contract for a +run that still fails after bounded retries. + +Artifact downloads, GitHub Release asset uploads, GHCR mutations, npm +publication, Homebrew updates, and Scoop updates remain separate contracts. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-RELEASE-ARTIFACT-UPLOAD-RECOVERY-001` | [Control flow](#control-flow), [Failure and recovery](#failure-and-recovery), and [Observability](#observability) | + +## Components and responsibilities + +- **`build-web` in `.github/workflows/release.yml`:** Uploads the shared web + bundle with the bounded retry pattern. +- **`build-bundles` in `.github/workflows/release.yml`:** Uploads each named + platform runtime bundle with the same pattern. The artifact name remains + `bundle-` so existing consumers keep their contract. +- **`build-desktop` in `.github/workflows/release.yml`:** Runs all five desktop + targets with `strategy.fail-fast: false` and uploads each validated desktop + artifact with the bounded retry pattern. +- **Downstream release jobs:** Continue to require successful producer jobs. + `publish-release` therefore remains the gate for npm, Homebrew, and Scoop. +- **`release-workflow-contract_test.py`:** Protects retry count, wait values, + missing-file behavior, matrix isolation, and downstream success conditions. + +The upload retry steps stay in the workflow file. This keeps the control logic +available from `github.workflow_sha` during a backfill even when the selected +application tag predates the resilience change. + +## Data and contracts + +Each required artifact keeps its current name and path contract. The first +upload attempt uses the pinned `actions/upload-artifact` action. Retry attempts +use the same name and path with `overwrite: true`, which permits a rerun to +replace a partially created artifact or an artifact left by an earlier attempt. + +Required uploads set `if-no-files-found: error`. The diagnostics upload remains +best effort because it runs after a failed macOS build and is not consumed by +publication. + +## Control flow + +For each required artifact: + +1. Run the upload action. +2. If its step outcome is `failure`, log the first retry and sleep for 30 + seconds. +3. Run the second upload attempt with `overwrite: true`. +4. If the second attempt fails, log the second retry and sleep for 60 seconds. +5. Run the third upload attempt with `overwrite: true`. +6. Fail the producing job when all three outcomes are failures. + +`continue-on-error` is limited to the individual upload attempts. The final +failure gate converts three failed attempts back into a job failure. A +successful first or retry attempt skips later attempts. + +The desktop matrix does not cancel sibling jobs after one target fails. The +publication job still checks `needs.build-desktop.result == 'success'`, so this +change preserves fail-closed publication while retaining evidence from every +target. + +## Failure and recovery + +A transient artifact-service or network error gets two bounded retry chances +inside the same job. A missing output, invalid path, or persistent service +failure remains a job failure after the retry budget is exhausted. Downstream +publication jobs remain skipped by their existing dependency checks. + +After a signed tag exists, maintainers use the existing failed-job rerun or +`backfill_tag` procedure documented in +[`docs/decisions/0029-release-backfill-and-desktop-diagnostics.md`](../../../decisions/0029-release-backfill-and-desktop-diagnostics.md). +The resilience change does not create or move tags and does not bypass artifact +validation. + +## Persistence + +No repository, release, or package-manager state changes. GitHub Actions +artifacts remain ephemeral run-scoped handoff objects. A retry can replace an +artifact with the same name, but it does not alter the release tag or public +publication channels. + +## Security + +The change keeps the existing SHA-pinned upload action and job permissions. It +adds no credentials, network endpoints, or shell inputs. Retry logs contain +only attempt numbers, wait durations, and the existing artifact action output. + +## Observability + +Each retry emits a warning containing the artifact name, attempt number, and +wait duration. The final failure remains visible in the producing job, while +the desktop matrix records sibling failures independently for diagnosis. +The existing release workflow contract test verifies the static control flow. + +## Related decisions + +- [ADR 0029: Release Backfill and Desktop Diagnostics](../../../decisions/0029-release-backfill-and-desktop-diagnostics.md)