diff --git a/.github/scripts/e2e-tests-workflow-contract_test.py b/.github/scripts/e2e-tests-workflow-contract_test.py index 23854e6277..c5ff61cb3b 100644 --- a/.github/scripts/e2e-tests-workflow-contract_test.py +++ b/.github/scripts/e2e-tests-workflow-contract_test.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 -"""Contract tests for the prebuilt desktop E2E image path.""" +"""Contract tests for the E2E workflow and prebuilt images.""" +import hashlib +import os from pathlib import Path +import subprocess +import tempfile import unittest @@ -10,6 +14,16 @@ DOCKERFILE = REPO_ROOT / ".github" / "docker" / "ci-base" / "Dockerfile" IMAGE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci-base-image.yml" E2E_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "e2e-tests.yml" LINT_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "lint-action-pinning.yml" +IMAGE_DIGEST_RESOLVER = REPO_ROOT / ".github" / "scripts" / "resolve-image-digest.sh" +VALID_IMAGE_INDEX = ( + b'{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json",' + b'"digest":"sha256:' + b"1" * 64 + b'","size":2}]}' +) +VALID_IMAGE_MANIFEST = ( + b'{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json",' + b'"digest":"sha256:' + b"0" * 64 + b'","size":2},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip",' + b'"digest":"sha256:' + b"2" * 64 + b'","size":2}]}' +) def job_block(workflow: str, job: str, next_job: str) -> str: @@ -21,7 +35,85 @@ def job_block(workflow: str, job: str, next_job: str) -> str: return remainder.partition(f"\n {next_job}:\n")[0] -class DesktopE2EWorkflowContractTest(unittest.TestCase): +class E2EWorkflowContractTest(unittest.TestCase): + def run_image_digest_resolver( + self, + *, + manifest: bytes, + failures_before_success: int, + attempt_timeout: str = "5s", + stalls_before_success: int = 0, + invalid_manifest: bytes = b"", + invalid_responses_before_success: int = 0, + ) -> tuple[subprocess.CompletedProcess[str], int]: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + bin_dir = temp_path / "bin" + bin_dir.mkdir() + count_file = temp_path / "docker-count" + manifest_file = temp_path / "manifest.json" + manifest_file.write_bytes(manifest) + invalid_manifest_file = temp_path / "invalid-manifest.json" + invalid_manifest_file.write_bytes(invalid_manifest) + + docker = bin_dir / "docker" + docker.write_text( + """#!/usr/bin/env bash +set -euo pipefail +count=0 +if [[ -f "${FAKE_DOCKER_COUNT}" ]]; then + count="$(cat "${FAKE_DOCKER_COUNT}")" +fi +count="$((count + 1))" +printf '%s' "${count}" > "${FAKE_DOCKER_COUNT}" +if (( count <= FAKE_DOCKER_FAILURES )); then + echo "transient registry failure" >&2 + exit 1 +fi +if (( count <= FAKE_DOCKER_FAILURES + FAKE_DOCKER_STALLS )); then + echo "stalled registry lookup" >&2 + /bin/sleep 1 + exit 1 +fi +if (( count <= FAKE_DOCKER_FAILURES + FAKE_DOCKER_STALLS + FAKE_DOCKER_INVALID_RESPONSES )); then + cat "${FAKE_DOCKER_INVALID_MANIFEST}" + exit 0 +fi +cat "${FAKE_DOCKER_MANIFEST}" +""", + encoding="utf-8", + ) + docker.chmod(0o755) + + sleep = bin_dir / "sleep" + sleep.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + sleep.chmod(0o755) + + env = os.environ.copy() + env.update( + { + "PATH": f"{bin_dir}:{env['PATH']}", + "FAKE_DOCKER_COUNT": str(count_file), + "FAKE_DOCKER_FAILURES": str(failures_before_success), + "FAKE_DOCKER_STALLS": str(stalls_before_success), + "FAKE_DOCKER_INVALID_MANIFEST": str(invalid_manifest_file), + "FAKE_DOCKER_INVALID_RESPONSES": str( + invalid_responses_before_success + ), + "FAKE_DOCKER_MANIFEST": str(manifest_file), + "IMAGE_RESOLVE_ATTEMPT_TIMEOUT": attempt_timeout, + } + ) + result = subprocess.run( + ["bash", str(IMAGE_DIGEST_RESOLVER), "ghcr.io/kdlbs/kandev-ci:runtime-latest"], + check=False, + capture_output=True, + env=env, + text=True, + ) + attempts = int(count_file.read_text(encoding="utf-8")) if count_file.exists() else 0 + return result, attempts + def test_desktop_image_contains_pinned_toolchain_and_system_dependencies(self) -> None: dockerfile = DOCKERFILE.read_text(encoding="utf-8") @@ -85,13 +177,14 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase): changes_job = job_block(workflow, "changes", "build") for pattern in ( ".github/docker/ci-base/**", + ".github/scripts/resolve-image-digest.sh", ".github/workflows/ci-base-image.yml", ): self.assertIn(pattern, changes_job) def test_normal_shard_has_queue_safe_timeout(self) -> None: workflow = E2E_WORKFLOW.read_text(encoding="utf-8") - normal_job = job_block(workflow, "e2e", "e2e-containers") + normal_job = job_block(workflow, "e2e", "playwright_image") self.assertIn( "# 35 min covers the serial count-fallback tail and setup overhead", @@ -126,7 +219,8 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase): # @covers AC-PLATFORM-E2E-DURATION-AWARE-SHARDING-002.2 def test_container_job_reuses_verified_browser_cache_with_fallback(self) -> None: workflow = E2E_WORKFLOW.read_text(encoding="utf-8") - container_job = job_block(workflow, "e2e-containers", "desktop-e2e") + resolver_job = job_block(workflow, "playwright_image", "e2e-containers") + container_job = job_block(workflow, "e2e-containers", "e2e-kubernetes-compatibility") self.assertIn( "uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", @@ -136,36 +230,30 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase): "uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9", container_job, ) - self.assertIn("id: playwright_image", container_job) - self.assertLess( - container_job.index("- name: Log in to GHCR"), - container_job.index("- name: Resolve immutable Playwright runtime image"), - ) + self.assertIn("needs: [changes, build, playwright_image]", container_job) + self.assertNotIn("docker buildx imagetools inspect", container_job) self.assertIn( "if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository\n uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f", - container_job, + resolver_job, ) self.assertIn( - "image=ghcr.io/kdlbs/kandev-ci:runtime-latest", - container_job, + ".github/scripts/resolve-image-digest.sh \"$image\"", + resolver_job, ) self.assertIn( - 'docker buildx imagetools inspect "$image"', - container_job, + "digest: ${{ steps.resolve.outputs.digest }}", + resolver_job, ) - self.assertIn("for attempt in 1 2 3; do", container_job) + gate_job = workflow.partition(" e2e-gate:\n")[2] + self.assertIn("PLAYWRIGHT_IMAGE_RESULT", gate_job) + self.assertIn('"playwright-image:${PLAYWRIGHT_IMAGE_RESULT}"', gate_job) self.assertIn( - 'if candidate="$(docker buildx imagetools inspect "$image"', + "key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}", container_job, ) - self.assertIn('sleep "$((attempt * 2))"', container_job) self.assertIn("path: /tmp/ms-playwright", container_job) self.assertIn( - "key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }}", - container_job, - ) - self.assertIn( - "restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-", + "restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-", container_job, ) self.assertIn("id: playwright_cache", container_job) @@ -184,7 +272,7 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase): container_job, ) self.assertIn( - "PLAYWRIGHT_RUNTIME_REF: ${{ steps.playwright_image.outputs.ref }}", + "PLAYWRIGHT_RUNTIME_REF: ${{ needs.playwright_image.outputs.ref }}", container_job, ) self.assertIn('docker pull "$PLAYWRIGHT_RUNTIME_REF"', container_job) @@ -201,6 +289,105 @@ class DesktopE2EWorkflowContractTest(unittest.TestCase): self.assertIn('setup_mode="image-fallback"', container_job) self.assertGreaterEqual(container_job.count("continue-on-error: true"), 2) + def test_image_digest_resolver_hashes_raw_manifest_bytes(self) -> None: + self.assertTrue(IMAGE_DIGEST_RESOLVER.exists()) + resolver = IMAGE_DIGEST_RESOLVER.read_text(encoding="utf-8") + self.assertIn('imagetools inspect --raw "$image"', resolver) + manifest = VALID_IMAGE_MANIFEST + + result, attempts = self.run_image_digest_resolver( + manifest=manifest, failures_before_success=0 + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.strip(), f"sha256:{hashlib.sha256(manifest).hexdigest()}" + ) + self.assertEqual(attempts, 1) + + def test_image_digest_resolver_retries_transient_registry_failures(self) -> None: + self.assertTrue(IMAGE_DIGEST_RESOLVER.exists()) + manifest = VALID_IMAGE_INDEX + + result, attempts = self.run_image_digest_resolver( + manifest=manifest, failures_before_success=2 + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(attempts, 3) + self.assertIn("attempt 1/5", result.stderr) + self.assertIn("transient registry failure", result.stderr) + + def test_image_digest_resolver_times_out_stalled_registry_lookup(self) -> None: + manifest = VALID_IMAGE_INDEX + + result, attempts = self.run_image_digest_resolver( + manifest=manifest, + failures_before_success=0, + attempt_timeout="0.5s", + stalls_before_success=1, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(attempts, 2) + self.assertIn("timed out after 0.5s", result.stderr) + + def test_image_digest_resolver_retries_invalid_manifest_then_succeeds(self) -> None: + self.assertTrue(IMAGE_DIGEST_RESOLVER.exists()) + manifest = VALID_IMAGE_INDEX + + result, attempts = self.run_image_digest_resolver( + manifest=manifest, + failures_before_success=0, + invalid_manifest=b'{"schemaVersion":1}', + invalid_responses_before_success=2, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.strip(), f"sha256:{hashlib.sha256(manifest).hexdigest()}" + ) + self.assertEqual(attempts, 3) + self.assertIn("attempt 1/5", result.stderr) + + def test_image_digest_resolver_rejects_invalid_manifest_responses(self) -> None: + for name, invalid_manifest in ( + ("empty", b""), + ("malformed", b"not-json"), + ("wrong-schema", b'{"schemaVersion":1}'), + ("schema-only", b'{"schemaVersion":2}'), + ("empty-index", b'{"schemaVersion":2,"manifests":[]}'), + ("missing-descriptor-fields", b'{"schemaVersion":2,"manifests":[{}]}'), + ( + "empty-layers", + b'{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json",' + b'"digest":"sha256:' + b"0" * 64 + b'","size":2},"layers":[]}', + ), + ): + with self.subTest(name=name): + result, attempts = self.run_image_digest_resolver( + manifest=VALID_IMAGE_INDEX, + failures_before_success=0, + invalid_manifest=invalid_manifest, + invalid_responses_before_success=5, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(attempts, 5) + self.assertEqual(result.stdout, "") + self.assertIn("Could not resolve an immutable digest", result.stderr) + + def test_image_digest_resolver_fails_closed_after_bounded_retries(self) -> None: + self.assertTrue(IMAGE_DIGEST_RESOLVER.exists()) + + result, attempts = self.run_image_digest_resolver( + manifest=VALID_IMAGE_INDEX, failures_before_success=5 + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(attempts, 5) + self.assertIn("Could not resolve an immutable digest", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/.github/scripts/resolve-image-digest.sh b/.github/scripts/resolve-image-digest.sh new file mode 100755 index 0000000000..10f29f4deb --- /dev/null +++ b/.github/scripts/resolve-image-digest.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if (( $# != 1 )); then + echo "usage: $0 " >&2 + exit 2 +fi + +image="$1" +attempt_timeout="${IMAGE_RESOLVE_ATTEMPT_TIMEOUT:-30s}" +manifest_file="$(mktemp)" +trap 'rm -f "$manifest_file"' EXIT + +for attempt in 1 2 3 4 5; do + : > "$manifest_file" + inspect_status=0 + timeout --kill-after=5s "$attempt_timeout" \ + docker buildx imagetools inspect --raw "$image" \ + > "$manifest_file" || inspect_status=$? + if (( inspect_status == 124 || inspect_status == 137 )); then + echo "::warning::Registry lookup for $image timed out after $attempt_timeout on attempt $attempt/5" >&2 + fi + if (( inspect_status == 0 )) \ + && [[ -s "$manifest_file" ]] \ + && jq -e ' + def descriptor: + type == "object" + and (.mediaType? | type == "string") + and (.digest? | if type == "string" then test("^sha256:[0-9a-f]{64}$") else false end) + and (.size? | if type == "number" then . >= 0 and . == floor else false end); + .schemaVersion == 2 + and ( + (.manifests? | if type == "array" then length > 0 and all(.[]; descriptor) else false end) + or ( + (.config? | descriptor) + and (.layers? | if type == "array" then length > 0 and all(.[]; descriptor) else false end) + ) + ) + ' "$manifest_file" > /dev/null; then + printf 'sha256:%s\n' "$(sha256sum "$manifest_file" | cut -d ' ' -f 1)" + exit 0 + fi + + if (( attempt < 5 )); then + echo "::warning::Could not resolve $image on attempt $attempt/5; retrying" >&2 + sleep "$((attempt * 2))" + fi +done + +echo "::error::Could not resolve an immutable digest for $image" >&2 +exit 1 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 4b92d57b5b..f751f09c46 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -68,6 +68,7 @@ jobs: apps/pnpm-lock.yaml apps/pnpm-workspace.yaml .github/docker/ci-base/** + .github/scripts/resolve-image-digest.sh .github/workflows/ci-base-image.yml .github/workflows/e2e-tests.yml .github/scripts/changed-paths.py @@ -486,6 +487,44 @@ jobs: path: apps/web/e2e/test-results/ retention-days: 7 + # Resolve the runtime tag once before the six container shards start. Fork + # pull requests use the public package anonymously, so sharing one bounded + # lookup avoids multiplying transient GHCR failures across the matrix. + playwright_image: + name: Resolve Playwright runtime image + needs: changes + if: needs.changes.outputs.run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + digest: ${{ steps.resolve.outputs.digest }} + ref: ${{ steps.resolve.outputs.ref }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Log in to GHCR + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve immutable Playwright runtime image + id: resolve + run: | + set -euo pipefail + image=ghcr.io/kdlbs/kandev-ci:runtime-latest + digest="$(bash .github/scripts/resolve-image-digest.sh "$image")" + { + echo "digest=$digest" + echo "ref=$image@$digest" + } >> "$GITHUB_OUTPUT" + # Real-container E2E: exercises the Docker, SSH, and Kubernetes executors. # ubuntu-latest ships with Docker preinstalled, so no service container is # required. Container-bound tests are slow (up to ~3 min each), so we shard across @@ -497,7 +536,7 @@ jobs: # Sharded 6 ways — each shard is its own runner. See apps/web/e2e/README.md. e2e-containers: name: E2E Containers Shard ${{ matrix.shard }}/6 - needs: [changes, build] + needs: [changes, build, playwright_image] if: needs.changes.outputs.run == 'true' # This job intentionally runs on the host ubuntu runner (NOT the kandev-ci # container image): the containers project needs direct access to the @@ -565,36 +604,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Resolve the mutable convenience tag before restoring the cache. The - # digest binds both the cache and the fallback extraction to one image - # identity, while avoiding a full image pull on a verified cache hit. - - name: Resolve immutable Playwright runtime image - id: playwright_image - run: | - set -euo pipefail - image=ghcr.io/kdlbs/kandev-ci:runtime-latest - digest="" - for attempt in 1 2 3; do - candidate="" - if candidate="$(docker buildx imagetools inspect "$image" | awk '$1 == "Digest:" { print $2; exit }')" \ - && [[ "$candidate" =~ ^sha256:[0-9a-f]{64}$ ]]; then - digest="$candidate" - break - fi - if (( attempt < 3 )); then - echo "::warning::Could not resolve $image on attempt $attempt/3; retrying" - sleep "$((attempt * 2))" - fi - done - if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "::error::Could not resolve an immutable digest for $image" - exit 1 - fi - { - echo "digest=$digest" - echo "ref=$image@$digest" - } >> "$GITHUB_OUTPUT" - # The runtime image remains the browser source of truth. The cache only # avoids copying its already-installed browsers on warm host runners. # Each run has a unique primary key so a repaired fallback can replace a @@ -606,8 +615,8 @@ jobs: continue-on-error: true with: path: /tmp/ms-playwright - key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}- + key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}- - name: Configure Playwright browser path run: echo "PLAYWRIGHT_BROWSERS_PATH=/tmp/ms-playwright" >> "$GITHUB_ENV" @@ -649,7 +658,7 @@ jobs: id: playwright_extract if: steps.playwright_cache.outputs.cache-hit == '' || steps.playwright_cache_verify.outcome != 'success' env: - PLAYWRIGHT_RUNTIME_REF: ${{ steps.playwright_image.outputs.ref }} + PLAYWRIGHT_RUNTIME_REF: ${{ needs.playwright_image.outputs.ref }} run: | set -euo pipefail docker pull "$PLAYWRIGHT_RUNTIME_REF" @@ -702,7 +711,7 @@ jobs: continue-on-error: true with: path: /tmp/ms-playwright - key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ steps.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }} + key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Summarize Playwright browser setup if: always() @@ -712,7 +721,7 @@ jobs: CACHE_VERIFY_OUTCOME: ${{ steps.playwright_cache_verify.outcome }} CACHE_EXTRACT_OUTCOME: ${{ steps.playwright_extract.outcome }} CACHE_SAVE_OUTCOME: ${{ steps.playwright_cache_save.outcome }} - IMAGE_DIGEST: ${{ steps.playwright_image.outputs.digest }} + IMAGE_DIGEST: ${{ needs.playwright_image.outputs.digest }} SETUP_STARTED_AT: ${{ steps.browser_setup_timer.outputs.started_at }} run: | set -euo pipefail @@ -1469,6 +1478,7 @@ jobs: [ changes, build, + playwright_image, e2e, e2e-containers, e2e-kubernetes-compatibility, @@ -1482,6 +1492,7 @@ jobs: env: CHANGES_RESULT: ${{ needs.changes.result }} BUILD_RESULT: ${{ needs.build.result }} + PLAYWRIGHT_IMAGE_RESULT: ${{ needs.playwright_image.result }} E2E_RESULT: ${{ needs.e2e.result }} E2E_CONTAINERS_RESULT: ${{ needs['e2e-containers'].result }} E2E_KUBERNETES_COMPATIBILITY_RESULT: ${{ needs['e2e-kubernetes-compatibility'].result }} @@ -1492,6 +1503,7 @@ jobs: echo "Change detection: ${CHANGES_RESULT}" echo "Build: ${BUILD_RESULT}" + echo "Playwright image: ${PLAYWRIGHT_IMAGE_RESULT}" echo "E2E shards: ${E2E_RESULT}" echo "E2E container shards: ${E2E_CONTAINERS_RESULT}" echo "Kubernetes compatibility: ${E2E_KUBERNETES_COMPATIBILITY_RESULT}" @@ -1515,6 +1527,7 @@ jobs: # list above against twenty shard names. for entry in \ "build:${BUILD_RESULT}" \ + "playwright-image:${PLAYWRIGHT_IMAGE_RESULT}" \ "e2e:${E2E_RESULT}" \ "e2e-containers:${E2E_CONTAINERS_RESULT}" \ "e2e-kubernetes-compatibility:${E2E_KUBERNETES_COMPATIBILITY_RESULT}" \ diff --git a/apps/web/e2e/README.md b/apps/web/e2e/README.md index 284d69b5b2..09b06d7e95 100644 --- a/apps/web/e2e/README.md +++ b/apps/web/e2e/README.md @@ -248,8 +248,9 @@ Dispatch the workflow with `fail_on_flaky=true` to set existing two-retry policy while the summary makes retry groups visible. Container-backed CI jobs also cache the browser directory used by the host -runner. Each job first resolves the `runtime-latest` convenience tag to a -sha256 image digest. The cache key includes the runner OS, the Playwright +runner. The workflow resolves the `runtime-latest` convenience tag once to a +sha256 image digest and shares that immutable reference with every container +shard. The cache key includes the runner OS, the Playwright `v1.61.1-noble` browser source, that digest, and a run-specific primary-key suffix. Its stable restore prefix selects the newest compatible entry without making a rejected cache entry win forever. A verified exact or prefix match