fix(ci): stabilize immutable runtime image resolution (#3397)

Co-authored-by: ayattara <alassane.yattara@savoirfairelinux.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
Alassane
2026-09-05 16:15:04 +00:00
committed by GitHub
parent 545e29311c
commit 0c9b1ec588
4 changed files with 314 additions and 61 deletions
@@ -1,7 +1,11 @@
#!/usr/bin/env python3
"""Contract tests for the prebuilt desktop E2E image path."""
"""Contract tests for the E2E workflow and prebuilt images."""
import hashlib
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
@@ -10,6 +14,16 @@ DOCKERFILE = REPO_ROOT / ".github" / "docker" / "ci-base" / "Dockerfile"
IMAGE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci-base-image.yml"
E2E_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "e2e-tests.yml"
LINT_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "lint-action-pinning.yml"
IMAGE_DIGEST_RESOLVER = REPO_ROOT / ".github" / "scripts" / "resolve-image-digest.sh"
VALID_IMAGE_INDEX = (
b'{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json",'
b'"digest":"sha256:' + b"1" * 64 + b'","size":2}]}'
)
VALID_IMAGE_MANIFEST = (
b'{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json",'
b'"digest":"sha256:' + b"0" * 64 + b'","size":2},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip",'
b'"digest":"sha256:' + b"2" * 64 + b'","size":2}]}'
)
def job_block(workflow: str, job: str, next_job: str) -> str:
@@ -21,7 +35,85 @@ def job_block(workflow: str, job: str, next_job: str) -> str:
return remainder.partition(f"\n {next_job}:\n")[0]
class DesktopE2EWorkflowContractTest(unittest.TestCase):
class E2EWorkflowContractTest(unittest.TestCase):
def run_image_digest_resolver(
self,
*,
manifest: bytes,
failures_before_success: int,
attempt_timeout: str = "5s",
stalls_before_success: int = 0,
invalid_manifest: bytes = b"",
invalid_responses_before_success: int = 0,
) -> tuple[subprocess.CompletedProcess[str], int]:
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
bin_dir = temp_path / "bin"
bin_dir.mkdir()
count_file = temp_path / "docker-count"
manifest_file = temp_path / "manifest.json"
manifest_file.write_bytes(manifest)
invalid_manifest_file = temp_path / "invalid-manifest.json"
invalid_manifest_file.write_bytes(invalid_manifest)
docker = bin_dir / "docker"
docker.write_text(
"""#!/usr/bin/env bash
set -euo pipefail
count=0
if [[ -f "${FAKE_DOCKER_COUNT}" ]]; then
count="$(cat "${FAKE_DOCKER_COUNT}")"
fi
count="$((count + 1))"
printf '%s' "${count}" > "${FAKE_DOCKER_COUNT}"
if (( count <= FAKE_DOCKER_FAILURES )); then
echo "transient registry failure" >&2
exit 1
fi
if (( count <= FAKE_DOCKER_FAILURES + FAKE_DOCKER_STALLS )); then
echo "stalled registry lookup" >&2
/bin/sleep 1
exit 1
fi
if (( count <= FAKE_DOCKER_FAILURES + FAKE_DOCKER_STALLS + FAKE_DOCKER_INVALID_RESPONSES )); then
cat "${FAKE_DOCKER_INVALID_MANIFEST}"
exit 0
fi
cat "${FAKE_DOCKER_MANIFEST}"
""",
encoding="utf-8",
)
docker.chmod(0o755)
sleep = bin_dir / "sleep"
sleep.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8")
sleep.chmod(0o755)
env = os.environ.copy()
env.update(
{
"PATH": f"{bin_dir}:{env['PATH']}",
"FAKE_DOCKER_COUNT": str(count_file),
"FAKE_DOCKER_FAILURES": str(failures_before_success),
"FAKE_DOCKER_STALLS": str(stalls_before_success),
"FAKE_DOCKER_INVALID_MANIFEST": str(invalid_manifest_file),
"FAKE_DOCKER_INVALID_RESPONSES": str(
invalid_responses_before_success
),
"FAKE_DOCKER_MANIFEST": str(manifest_file),
"IMAGE_RESOLVE_ATTEMPT_TIMEOUT": attempt_timeout,
}
)
result = subprocess.run(
["bash", str(IMAGE_DIGEST_RESOLVER), "ghcr.io/kdlbs/kandev-ci:runtime-latest"],
check=False,
capture_output=True,
env=env,
text=True,
)
attempts = int(count_file.read_text(encoding="utf-8")) if count_file.exists() else 0
return result, attempts
def test_desktop_image_contains_pinned_toolchain_and_system_dependencies(self) -> None:
dockerfile = DOCKERFILE.read_text(encoding="utf-8")
@@ -85,13 +177,14 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase):
changes_job = job_block(workflow, "changes", "build")
for pattern in (
".github/docker/ci-base/**",
".github/scripts/resolve-image-digest.sh",
".github/workflows/ci-base-image.yml",
):
self.assertIn(pattern, changes_job)
def test_normal_shard_has_queue_safe_timeout(self) -> None:
workflow = E2E_WORKFLOW.read_text(encoding="utf-8")
normal_job = job_block(workflow, "e2e", "e2e-containers")
normal_job = job_block(workflow, "e2e", "playwright_image")
self.assertIn(
"# 35 min covers the serial count-fallback tail and setup overhead",
@@ -126,7 +219,8 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase):
# @covers AC-PLATFORM-E2E-DURATION-AWARE-SHARDING-002.2
def test_container_job_reuses_verified_browser_cache_with_fallback(self) -> None:
workflow = E2E_WORKFLOW.read_text(encoding="utf-8")
container_job = job_block(workflow, "e2e-containers", "desktop-e2e")
resolver_job = job_block(workflow, "playwright_image", "e2e-containers")
container_job = job_block(workflow, "e2e-containers", "e2e-kubernetes-compatibility")
self.assertIn(
"uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9",
@@ -136,36 +230,30 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase):
"uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9",
container_job,
)
self.assertIn("id: playwright_image", container_job)
self.assertLess(
container_job.index("- name: Log in to GHCR"),
container_job.index("- name: Resolve immutable Playwright runtime image"),
)
self.assertIn("needs: [changes, build, playwright_image]", container_job)
self.assertNotIn("docker buildx imagetools inspect", container_job)
self.assertIn(
"if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository\n uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f",
container_job,
resolver_job,
)
self.assertIn(
"image=ghcr.io/kdlbs/kandev-ci:runtime-latest",
container_job,
".github/scripts/resolve-image-digest.sh \"$image\"",
resolver_job,
)
self.assertIn(
'docker buildx imagetools inspect "$image"',
container_job,
"digest: ${{ steps.resolve.outputs.digest }}",
resolver_job,
)
self.assertIn("for attempt in 1 2 3; do", container_job)
gate_job = workflow.partition(" e2e-gate:\n")[2]
self.assertIn("PLAYWRIGHT_IMAGE_RESULT", gate_job)
self.assertIn('"playwright-image:${PLAYWRIGHT_IMAGE_RESULT}"', gate_job)
self.assertIn(
'if candidate="$(docker buildx imagetools inspect "$image"',
"key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}",
container_job,
)
self.assertIn('sleep "$((attempt * 2))"', container_job)
self.assertIn("path: /tmp/ms-playwright", container_job)
self.assertIn(
"key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}",
container_job,
)
self.assertIn(
"restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-",
"restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-",
container_job,
)
self.assertIn("id: playwright_cache", container_job)
@@ -184,7 +272,7 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase):
container_job,
)
self.assertIn(
"PLAYWRIGHT_RUNTIME_REF: ${{ steps.playwright_image.outputs.ref }}",
"PLAYWRIGHT_RUNTIME_REF: ${{ needs.playwright_image.outputs.ref }}",
container_job,
)
self.assertIn('docker pull "$PLAYWRIGHT_RUNTIME_REF"', container_job)
@@ -201,6 +289,105 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase):
self.assertIn('setup_mode="image-fallback"', container_job)
self.assertGreaterEqual(container_job.count("continue-on-error: true"), 2)
def test_image_digest_resolver_hashes_raw_manifest_bytes(self) -> None:
self.assertTrue(IMAGE_DIGEST_RESOLVER.exists())
resolver = IMAGE_DIGEST_RESOLVER.read_text(encoding="utf-8")
self.assertIn('imagetools inspect --raw "$image"', resolver)
manifest = VALID_IMAGE_MANIFEST
result, attempts = self.run_image_digest_resolver(
manifest=manifest, failures_before_success=0
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
result.stdout.strip(), f"sha256:{hashlib.sha256(manifest).hexdigest()}"
)
self.assertEqual(attempts, 1)
def test_image_digest_resolver_retries_transient_registry_failures(self) -> None:
self.assertTrue(IMAGE_DIGEST_RESOLVER.exists())
manifest = VALID_IMAGE_INDEX
result, attempts = self.run_image_digest_resolver(
manifest=manifest, failures_before_success=2
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(attempts, 3)
self.assertIn("attempt 1/5", result.stderr)
self.assertIn("transient registry failure", result.stderr)
def test_image_digest_resolver_times_out_stalled_registry_lookup(self) -> None:
manifest = VALID_IMAGE_INDEX
result, attempts = self.run_image_digest_resolver(
manifest=manifest,
failures_before_success=0,
attempt_timeout="0.5s",
stalls_before_success=1,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(attempts, 2)
self.assertIn("timed out after 0.5s", result.stderr)
def test_image_digest_resolver_retries_invalid_manifest_then_succeeds(self) -> None:
self.assertTrue(IMAGE_DIGEST_RESOLVER.exists())
manifest = VALID_IMAGE_INDEX
result, attempts = self.run_image_digest_resolver(
manifest=manifest,
failures_before_success=0,
invalid_manifest=b'{"schemaVersion":1}',
invalid_responses_before_success=2,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
result.stdout.strip(), f"sha256:{hashlib.sha256(manifest).hexdigest()}"
)
self.assertEqual(attempts, 3)
self.assertIn("attempt 1/5", result.stderr)
def test_image_digest_resolver_rejects_invalid_manifest_responses(self) -> None:
for name, invalid_manifest in (
("empty", b""),
("malformed", b"not-json"),
("wrong-schema", b'{"schemaVersion":1}'),
("schema-only", b'{"schemaVersion":2}'),
("empty-index", b'{"schemaVersion":2,"manifests":[]}'),
("missing-descriptor-fields", b'{"schemaVersion":2,"manifests":[{}]}'),
(
"empty-layers",
b'{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json",'
b'"digest":"sha256:' + b"0" * 64 + b'","size":2},"layers":[]}',
),
):
with self.subTest(name=name):
result, attempts = self.run_image_digest_resolver(
manifest=VALID_IMAGE_INDEX,
failures_before_success=0,
invalid_manifest=invalid_manifest,
invalid_responses_before_success=5,
)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(attempts, 5)
self.assertEqual(result.stdout, "")
self.assertIn("Could not resolve an immutable digest", result.stderr)
def test_image_digest_resolver_fails_closed_after_bounded_retries(self) -> None:
self.assertTrue(IMAGE_DIGEST_RESOLVER.exists())
result, attempts = self.run_image_digest_resolver(
manifest=VALID_IMAGE_INDEX, failures_before_success=5
)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(attempts, 5)
self.assertIn("Could not resolve an immutable digest", result.stderr)
if __name__ == "__main__":
unittest.main()
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env bash
set -euo pipefail
if (( $# != 1 )); then
echo "usage: $0 <image>" >&2
exit 2
fi
image="$1"
attempt_timeout="${IMAGE_RESOLVE_ATTEMPT_TIMEOUT:-30s}"
manifest_file="$(mktemp)"
trap 'rm -f "$manifest_file"' EXIT
for attempt in 1 2 3 4 5; do
: > "$manifest_file"
inspect_status=0
timeout --kill-after=5s "$attempt_timeout" \
docker buildx imagetools inspect --raw "$image" \
> "$manifest_file" || inspect_status=$?
if (( inspect_status == 124 || inspect_status == 137 )); then
echo "::warning::Registry lookup for $image timed out after $attempt_timeout on attempt $attempt/5" >&2
fi
if (( inspect_status == 0 )) \
&& [[ -s "$manifest_file" ]] \
&& jq -e '
def descriptor:
type == "object"
and (.mediaType? | type == "string")
and (.digest? | if type == "string" then test("^sha256:[0-9a-f]{64}$") else false end)
and (.size? | if type == "number" then . >= 0 and . == floor else false end);
.schemaVersion == 2
and (
(.manifests? | if type == "array" then length > 0 and all(.[]; descriptor) else false end)
or (
(.config? | descriptor)
and (.layers? | if type == "array" then length > 0 and all(.[]; descriptor) else false end)
)
)
' "$manifest_file" > /dev/null; then
printf 'sha256:%s\n' "$(sha256sum "$manifest_file" | cut -d ' ' -f 1)"
exit 0
fi
if (( attempt < 5 )); then
echo "::warning::Could not resolve $image on attempt $attempt/5; retrying" >&2
sleep "$((attempt * 2))"
fi
done
echo "::error::Could not resolve an immutable digest for $image" >&2
exit 1
+49 -36
View File
@@ -68,6 +68,7 @@ jobs:
apps/pnpm-lock.yaml
apps/pnpm-workspace.yaml
.github/docker/ci-base/**
.github/scripts/resolve-image-digest.sh
.github/workflows/ci-base-image.yml
.github/workflows/e2e-tests.yml
.github/scripts/changed-paths.py
@@ -486,6 +487,44 @@ jobs:
path: apps/web/e2e/test-results/
retention-days: 7
# Resolve the runtime tag once before the six container shards start. Fork
# pull requests use the public package anonymously, so sharing one bounded
# lookup avoids multiplying transient GHCR failures across the matrix.
playwright_image:
name: Resolve Playwright runtime image
needs: changes
if: needs.changes.outputs.run == 'true'
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
digest: ${{ steps.resolve.outputs.digest }}
ref: ${{ steps.resolve.outputs.ref }}
steps:
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
persist-credentials: false
- name: Log in to GHCR
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Resolve immutable Playwright runtime image
id: resolve
run: |
set -euo pipefail
image=ghcr.io/kdlbs/kandev-ci:runtime-latest
digest="$(bash .github/scripts/resolve-image-digest.sh "$image")"
{
echo "digest=$digest"
echo "ref=$image@$digest"
} >> "$GITHUB_OUTPUT"
# Real-container E2E: exercises the Docker, SSH, and Kubernetes executors.
# ubuntu-latest ships with Docker preinstalled, so no service container is
# required. Container-bound tests are slow (up to ~3 min each), so we shard across
@@ -497,7 +536,7 @@ jobs:
# Sharded 6 ways — each shard is its own runner. See apps/web/e2e/README.md.
e2e-containers:
name: E2E Containers Shard ${{ matrix.shard }}/6
needs: [changes, build]
needs: [changes, build, playwright_image]
if: needs.changes.outputs.run == 'true'
# This job intentionally runs on the host ubuntu runner (NOT the kandev-ci
# container image): the containers project needs direct access to the
@@ -565,36 +604,6 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Resolve the mutable convenience tag before restoring the cache. The
# digest binds both the cache and the fallback extraction to one image
# identity, while avoiding a full image pull on a verified cache hit.
- name: Resolve immutable Playwright runtime image
id: playwright_image
run: |
set -euo pipefail
image=ghcr.io/kdlbs/kandev-ci:runtime-latest
digest=""
for attempt in 1 2 3; do
candidate=""
if candidate="$(docker buildx imagetools inspect "$image" | awk '$1 == "Digest:" { print $2; exit }')" \
&& [[ "$candidate" =~ ^sha256:[0-9a-f]{64}$ ]]; then
digest="$candidate"
break
fi
if (( attempt < 3 )); then
echo "::warning::Could not resolve $image on attempt $attempt/3; retrying"
sleep "$((attempt * 2))"
fi
done
if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "::error::Could not resolve an immutable digest for $image"
exit 1
fi
{
echo "digest=$digest"
echo "ref=$image@$digest"
} >> "$GITHUB_OUTPUT"
# The runtime image remains the browser source of truth. The cache only
# avoids copying its already-installed browsers on warm host runners.
# Each run has a unique primary key so a repaired fallback can replace a
@@ -606,8 +615,8 @@ jobs:
continue-on-error: true
with:
path: /tmp/ms-playwright
key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-
key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-
- name: Configure Playwright browser path
run: echo "PLAYWRIGHT_BROWSERS_PATH=/tmp/ms-playwright" >> "$GITHUB_ENV"
@@ -649,7 +658,7 @@ jobs:
id: playwright_extract
if: steps.playwright_cache.outputs.cache-hit == '' || steps.playwright_cache_verify.outcome != 'success'
env:
PLAYWRIGHT_RUNTIME_REF: ${{ steps.playwright_image.outputs.ref }}
PLAYWRIGHT_RUNTIME_REF: ${{ needs.playwright_image.outputs.ref }}
run: |
set -euo pipefail
docker pull "$PLAYWRIGHT_RUNTIME_REF"
@@ -702,7 +711,7 @@ jobs:
continue-on-error: true
with:
path: /tmp/ms-playwright
key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}
key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}
- name: Summarize Playwright browser setup
if: always()
@@ -712,7 +721,7 @@ jobs:
CACHE_VERIFY_OUTCOME: ${{ steps.playwright_cache_verify.outcome }}
CACHE_EXTRACT_OUTCOME: ${{ steps.playwright_extract.outcome }}
CACHE_SAVE_OUTCOME: ${{ steps.playwright_cache_save.outcome }}
IMAGE_DIGEST: ${{ steps.playwright_image.outputs.digest }}
IMAGE_DIGEST: ${{ needs.playwright_image.outputs.digest }}
SETUP_STARTED_AT: ${{ steps.browser_setup_timer.outputs.started_at }}
run: |
set -euo pipefail
@@ -1469,6 +1478,7 @@ jobs:
[
changes,
build,
playwright_image,
e2e,
e2e-containers,
e2e-kubernetes-compatibility,
@@ -1482,6 +1492,7 @@ jobs:
env:
CHANGES_RESULT: ${{ needs.changes.result }}
BUILD_RESULT: ${{ needs.build.result }}
PLAYWRIGHT_IMAGE_RESULT: ${{ needs.playwright_image.result }}
E2E_RESULT: ${{ needs.e2e.result }}
E2E_CONTAINERS_RESULT: ${{ needs['e2e-containers'].result }}
E2E_KUBERNETES_COMPATIBILITY_RESULT: ${{ needs['e2e-kubernetes-compatibility'].result }}
@@ -1492,6 +1503,7 @@ jobs:
echo "Change detection: ${CHANGES_RESULT}"
echo "Build: ${BUILD_RESULT}"
echo "Playwright image: ${PLAYWRIGHT_IMAGE_RESULT}"
echo "E2E shards: ${E2E_RESULT}"
echo "E2E container shards: ${E2E_CONTAINERS_RESULT}"
echo "Kubernetes compatibility: ${E2E_KUBERNETES_COMPATIBILITY_RESULT}"
@@ -1515,6 +1527,7 @@ jobs:
# list above against twenty shard names.
for entry in \
"build:${BUILD_RESULT}" \
"playwright-image:${PLAYWRIGHT_IMAGE_RESULT}" \
"e2e:${E2E_RESULT}" \
"e2e-containers:${E2E_CONTAINERS_RESULT}" \
"e2e-kubernetes-compatibility:${E2E_KUBERNETES_COMPATIBILITY_RESULT}" \
+3 -2
View File
@@ -248,8 +248,9 @@ Dispatch the workflow with `fail_on_flaky=true` to set
existing two-retry policy while the summary makes retry groups visible.
Container-backed CI jobs also cache the browser directory used by the host
runner. Each job first resolves the `runtime-latest` convenience tag to a
sha256 image digest. The cache key includes the runner OS, the Playwright
runner. The workflow resolves the `runtime-latest` convenience tag once to a
sha256 image digest and shares that immutable reference with every container
shard. The cache key includes the runner OS, the Playwright
`v1.61.1-noble` browser source, that digest, and a run-specific primary-key
suffix. Its stable restore prefix selects the newest compatible entry without
making a rejected cache entry win forever. A verified exact or prefix match