diff --git a/.github/actions/conformance/expected-failures.2025-11-25.yml b/.github/actions/conformance/expected-failures.2025-11-25.yml new file mode 100644 index 000000000..0ca289738 --- /dev/null +++ b/.github/actions/conformance/expected-failures.2025-11-25.yml @@ -0,0 +1,26 @@ +# Expected failures at the 2025-11-25 wire, read by the `--suite all +# --spec-version 2025-11-25` server and client legs: every scenario the pinned +# harness marks applicable at 2025-11-25, run over the stateful handshake. +# +# That selection is a superset of the frozen 2025-11-25 requirement set (`npx +# $CONFORMANCE_PKG list --requirements 2025-11-25`), so an entry here for a +# scenario in that set means `conformance tier-check` scores the SDK below 100% +# for 2025-11-25 even though CI is green; link the tracking issue next to any +# such entry. +# +# Kept separate from expected-failures.yml even though the bare `--suite all` +# legs currently run these same scenarios at 2025-11-25 too: the harness's +# default wire is due to move (modelcontextprotocol/conformance#426) and this +# file must keep meaning 2025-11-25. Until then a failure here also needs an +# entry there. The file is passed even while empty: run-client.sh keys its solo +# re-verification off the harness's baseline output. +# +# Baseline established against the harness pinned via CONFORMANCE_PKG in +# .github/workflows/conformance.yml. New conformance releases are adopted by +# deliberately bumping that pin and reconciling all three expected-failures +# files in the same change. Entries MUST be removed as they start passing (the +# runner fails on stale entries). + +client: [] + +server: [] diff --git a/.github/actions/conformance/expected-failures.2026-07-28.yml b/.github/actions/conformance/expected-failures.2026-07-28.yml index 504b46385..4d872524b 100644 --- a/.github/actions/conformance/expected-failures.2026-07-28.yml +++ b/.github/actions/conformance/expected-failures.2026-07-28.yml @@ -1,19 +1,23 @@ -# Expected failures for the carried-forward 2026-07-28 legs -# (`--suite all --spec-version 2026-07-28` for both server and client). +# Expected failures at the 2026-07-28 wire, read by the `--suite all +# --spec-version 2026-07-28` server and client legs: every scenario the pinned +# harness marks applicable at 2026-07-28, run stateless with per-request _meta. +# +# That selection is a superset of the frozen 2026-07-28 requirement set (`npx +# $CONFORMANCE_PKG list --requirements 2026-07-28`), so an entry here for a +# scenario in that set means `conformance tier-check` scores the SDK below 100% +# for 2026-07-28 even though CI is green; link the tracking issue next to any +# such entry. # # This baseline is separate from expected-failures.yml because entries are -# keyed by scenario name only: a scenario that passes at its default version -# in the 2025 legs but fails when forced to 2026-07-28 (or vice versa) cannot -# be expressed in a shared file (the passing leg would flag the entry as -# stale). Like expected-failures.yml, this single file covers both -# directions: the client 2026 leg reads the `client:` section and the server -# 2026 leg reads the `server:` section. Both burn down independently of the -# 2025 legs. +# keyed by scenario name only: a scenario that passes at the harness default +# wire in the bare `--suite all` legs but fails when forced to 2026-07-28 (or +# vice versa) cannot be expressed in a shared file. Where the bare legs already +# default a scenario to 2026-07-28, a failure here also needs an entry there. # # Baseline established against the harness pinned via CONFORMANCE_PKG in # .github/workflows/conformance.yml. New conformance releases are adopted by -# deliberately bumping that pin and reconciling both this file and -# expected-failures.yml in the same change. +# deliberately bumping that pin and reconciling all three expected-failures +# files in the same change. # # Entries are grouped by what unblocks them. As each gap closes the # corresponding scenarios start passing and MUST be removed from this list diff --git a/.github/actions/conformance/expected-failures.yml b/.github/actions/conformance/expected-failures.yml index 4379d116d..e60e20105 100644 --- a/.github/actions/conformance/expected-failures.yml +++ b/.github/actions/conformance/expected-failures.yml @@ -1,10 +1,16 @@ -# Conformance scenarios not yet passing against the Python SDK on main. -# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. +# Conformance scenarios not yet passing against the Python SDK on main, read by +# the bare `--suite all` server and client legs (every scenario shipped with the +# pinned harness, each at the harness's default wire for it). CI exits 0 if only +# these fail, exits 1 on unexpected failures or stale entries. +# +# The wire-pinned legs read expected-failures.2025-11-25.yml and +# expected-failures.2026-07-28.yml instead; see the header of +# .github/workflows/conformance.yml for the leg layout. # # Baseline established against the harness pinned via CONFORMANCE_PKG in # .github/workflows/conformance.yml. New conformance releases are adopted by -# deliberately bumping that pin and reconciling both this file and -# expected-failures.2026-07-28.yml in the same change. +# deliberately bumping that pin and reconciling all three expected-failures +# files in the same change. # # Entries are grouped by SEP. As each SEP lands in the SDK the corresponding # scenarios start passing and MUST be removed from this list (the runner fails @@ -30,10 +36,9 @@ client: server: # SEP-2663 (io.modelcontextprotocol/tasks): the SDK does not implement the - # tasks extension yet. These extension-tagged scenarios are selected only by - # the bare `--suite all` leg — extension scenarios never match a - # --spec-version filter and the active/draft suites exclude them — so these - # entries are inert for the other legs that read this file. + # tasks extension yet. The bare `--suite all` leg is the only one that + # reaches these extension-tagged scenarios (they never match a + # --spec-version filter), so these entries live here only. # # `tasks-status-notifications` is intentionally NOT listed: the harness # skips it unconditionally (pending its rewrite against subscriptions/ diff --git a/.github/actions/conformance/run-client.sh b/.github/actions/conformance/run-client.sh index 3c9678877..f64d2de7b 100755 --- a/.github/actions/conformance/run-client.sh +++ b/.github/actions/conformance/run-client.sh @@ -1,6 +1,6 @@ #!/bin/bash # Run a client conformance suite, re-verifying unexpected failures solo. -# Concurrent suite runs on a 2-vCPU runner can push scenarios with real-time +# Concurrent suite runs on a hosted runner can push scenarios with real-time # waits past tolerance; solo, a real failure fails again while a contention # artifact passes. Failures that only reproduce under concurrency are excused. set -uo pipefail @@ -101,4 +101,8 @@ if [ -n "$output_dir" ]; then printf '%s\n' "${scenarios[@]}" > "$output_dir/FLAKE_RESCUED" fi echo "All ${#scenarios[@]} unexpected failure(s) passed when re-run solo; the suite failures were parallel-run contention." +# Surface the rescue on the run summary; the log line above and the +# FLAKE_RESCUED marker in the uploaded results are otherwise easy to miss on a +# green run. +echo "::warning title=Conformance flake rescued::${scenarios[*]} failed under suite concurrency and passed solo${output_dir:+ (see ${output_dir}-solo and FLAKE_RESCUED in the uploaded results)}." exit 0 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 022229688..65b014b63 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -15,13 +15,28 @@ permissions: env: # Pinned conformance harness package spec (passed verbatim to `npx --yes`). - # Bump deliberately and reconcile both + # Bump deliberately and reconcile the three # .github/actions/conformance/expected-failures*.yml files in the same change. CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11" +# Each job runs three `--suite all` legs, one per baseline file. The two +# `--spec-version ` legs run every scenario the pinned harness marks +# applicable at that spec revision, at that revision's wire (stateful +# handshake for 2025-11-25, stateless per-request _meta for 2026-07-28). Each +# is a superset of the frozen requirement set that `conformance tier-check` +# scores for that revision (`npx $CONFORMANCE_PKG list --requirements ` +# prints it), so an expected-failures..yml entry for a scenario in that +# set means tier-check reports the SDK below 100% for . The bare leg runs +# every shipped scenario at whichever wire the harness defaults it to; it is +# the only leg that reaches extension-tagged scenarios (tasks-*, auth/dpop*, +# ...), which never match a --spec-version filter. For everything else it +# overlaps one of the pinned legs, so a failure it also sees needs its own +# expected-failures.yml entry besides the pinned file's. + jobs: server-conformance: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -34,37 +49,21 @@ jobs: with: node-version: 24 - run: uv sync --frozen --all-extras --package mcp-everything-server - - name: Run server conformance (active suite) - run: >- - ./.github/actions/conformance/run-server.sh - --suite active - --expected-failures ./.github/actions/conformance/expected-failures.yml - --output-dir conformance-results/server-active - - name: Run server conformance (draft suite) + - name: Run server conformance (all suite, 2025-11-25 wire) run: >- ./.github/actions/conformance/run-server.sh - --suite draft - --expected-failures ./.github/actions/conformance/expected-failures.yml - --output-dir conformance-results/server-draft - - name: Run server conformance (2026-07-28 wire, all suite) + --suite all + --spec-version 2025-11-25 + --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml + --output-dir conformance-results/server-2025-11-25 + - name: Run server conformance (all suite, 2026-07-28 wire) run: >- ./.github/actions/conformance/run-server.sh --suite all --spec-version 2026-07-28 --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml --output-dir conformance-results/server-2026-07-28 - - name: Run server conformance (all suite, extension scenarios) - # A bare `--suite all` (no --spec-version) selects every scenario - # shipped with the pinned harness — including the extension-tagged - # tasks-* scenarios and pending-listed ones like server-sse-polling, - # which no other leg reaches (extension scenarios never match a - # --spec-version filter, and the pending list keeps them out of the - # active suite). Running the full set keeps unimplemented surfaces - # visible as baselined known failures in expected-failures.yml instead - # of silent exclusions, and stays robust to scenarios moving between - # harness suite lists across pin bumps. `--suite pending` would cover - # the same union slightly faster; the full set is preferred for the - # self-contained run and for parity with typescript-sdk's CI. + - name: Run server conformance (all suite, default wire) run: >- ./.github/actions/conformance/run-server.sh --suite all @@ -72,7 +71,9 @@ jobs: --output-dir conformance-results/server-all - name: Upload conformance results # The log has only summary counts; per-check data is in checks.json. - if: failure() + # always(), not !cancelled(): a timeout-minutes kill is a cancellation, + # and that is when the partial results matter most. + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: server-conformance-results @@ -81,6 +82,7 @@ jobs: client-conformance: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -97,20 +99,21 @@ jobs: - run: uv sync --frozen --all-extras --package mcp --compile-bytecode - name: Pre-compile bytecode (editable sources) run: uv run --frozen python -m compileall -q src .github/actions/conformance - - name: Run client conformance (all suite) - # The harness runs all scenarios via unbounded Promise.all; with 40 - # scenarios on a 2-core runner the slowest one (sse-retry, which has a - # real-time SSE reconnect wait) needs more than the 30s default budget. - # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40 - # concurrent spawns; run-client.sh re-runs unexpected failures solo. + # The harness runs a leg's scenarios via unbounded Promise.all; under + # that concurrency the slowest one (sse-retry, which has a real-time SSE + # reconnect wait) needs more than the 30s default budget. + # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40 + # concurrent spawns; run-client.sh re-runs unexpected failures solo. + - name: Run client conformance (all suite, 2025-11-25 wire) run: >- ./.github/actions/conformance/run-client.sh --command '.venv/bin/python .github/actions/conformance/client.py' --suite all --timeout 60000 - --expected-failures ./.github/actions/conformance/expected-failures.yml - --output-dir conformance-results/client-all - - name: Run client conformance (2026-07-28 wire, all suite) + --spec-version 2025-11-25 + --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml + --output-dir conformance-results/client-2025-11-25 + - name: Run client conformance (all suite, 2026-07-28 wire) run: >- ./.github/actions/conformance/run-client.sh --command '.venv/bin/python .github/actions/conformance/client.py' @@ -119,10 +122,19 @@ jobs: --spec-version 2026-07-28 --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml --output-dir conformance-results/client-2026-07-28 + - name: Run client conformance (all suite, default wire) + run: >- + ./.github/actions/conformance/run-client.sh + --command '.venv/bin/python .github/actions/conformance/client.py' + --suite all + --timeout 60000 + --expected-failures ./.github/actions/conformance/expected-failures.yml + --output-dir conformance-results/client-all - name: Upload conformance results # The log has only summary counts; per-check data is in checks.json. - # Also on FLAKE_RESCUED: rescued-flake evidence is otherwise discarded. - if: failure() || hashFiles('conformance-results/**/FLAKE_RESCUED') != '' + # always(), not !cancelled(): a timeout-minutes kill is a cancellation, + # and that is when the partial results matter most. + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: client-conformance-results