From 78514c2a18a5d5be63df3b219ae5fe57d12b0520 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Fri, 24 Apr 2026 12:25:13 +0100 Subject: [PATCH 1/2] feat: support "draft" as a first-class spec version target - Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next spec release is a one-line change in types.ts. - Accept "draft" as a valid protocolVersion in the initialize check and mock-server response. - --spec-version draft now selects latest-dated scenarios plus draft-tagged ones, so SEP authors can run the full suite against an SDK tracking the in-progress spec without retagging core scenarios. - Forward --spec-version to the client process via MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching protocolVersion. Closes #253 --- README.md | 3 +- src/checks/checks.test.ts | 26 ++++++++++++++ src/checks/client.ts | 17 +++++++--- src/index.ts | 6 ++-- src/runner/client.ts | 21 +++++++++--- src/scenarios/client/initialize.ts | 12 +++++-- src/scenarios/index.ts | 30 +++++++++++++---- src/scenarios/spec-version.test.ts | 54 +++++++++++++++++++++--------- src/types.ts | 34 +++++++++++++++---- 9 files changed, 160 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index dc84f102..5ba49836 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") - `--suite` - Run a suite of tests in parallel (e.g., "auth") +- `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `draft`). `draft` runs the latest dated release plus any draft-only scenarios - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output -The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. +The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, the corresponding wire protocol version is forwarded as `MCP_CONFORMANCE_PROTOCOL_VERSION` (e.g., `--spec-version draft` sets it to the current draft identifier such as `DRAFT-2026-v1`); example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. ### Server Testing diff --git a/src/checks/checks.test.ts b/src/checks/checks.test.ts index a08ba08a..b82f17c9 100644 --- a/src/checks/checks.test.ts +++ b/src/checks/checks.test.ts @@ -1,4 +1,5 @@ import { createClientInitializationCheck } from './client'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; describe('createClientInitializationCheck', () => { it('should return SUCCESS for a valid initialize request', () => { @@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => { expect(check.errorMessage).toContain('Client version missing'); }); + it('should accept the current draft protocol version', () => { + const request = { + protocolVersion: DRAFT_PROTOCOL_VERSION, + clientInfo: { name: 'TestClient', version: '1.0.0' } + }; + + const check = createClientInitializationCheck(request); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + }); + + it.each(['DRAFT-2025-v1', 'draft'])( + 'should reject stale or non-canonical draft version %s', + (protocolVersion) => { + const request = { + protocolVersion, + clientInfo: { name: 'TestClient', version: '1.0.0' } + }; + + const check = createClientInitializationCheck(request); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain('Version mismatch'); + } + ); + it('should support custom expected spec version', () => { const request = { protocolVersion: '2024-11-05', diff --git a/src/checks/client.ts b/src/checks/client.ts index 99834f92..340f603f 100644 --- a/src/checks/client.ts +++ b/src/checks/client.ts @@ -1,4 +1,9 @@ -import { ConformanceCheck, CheckStatus } from '../types'; +import { + ConformanceCheck, + CheckStatus, + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION +} from '../types'; export function createServerInfoCheck(serverInfo: { name: string; @@ -23,12 +28,16 @@ export function createServerInfoCheck(serverInfo: { }; } -// Valid MCP protocol versions -const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25']; +// Protocol versions the mock server will accept on initialize. +const VALID_PROTOCOL_VERSIONS = [ + '2025-06-18', + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION +]; export function createClientInitializationCheck( initializeRequest: any, - expectedSpecVersion: string = '2025-11-25' + expectedSpecVersion: string = LATEST_SPEC_VERSION ): ConformanceCheck { const protocolVersionSent = initializeRequest?.protocolVersion; diff --git a/src/index.ts b/src/index.ts index 38f7701a..64d1eaaa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -152,7 +152,8 @@ program options.command, scenarioName, timeout, - outputDir + outputDir, + specVersionFilter ); return { scenario: scenarioName, @@ -259,7 +260,8 @@ program validated.command, validated.scenario, timeout, - outputDir + outputDir, + specVersionFilter ); const { overallFailure } = printClientResults( diff --git a/src/runner/client.ts b/src/runner/client.ts index 4525416c..571d8f9d 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,7 +1,11 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; -import { ConformanceCheck } from '../types'; +import { + ConformanceCheck, + SpecVersion, + specVersionToProtocolVersion +} from '../types'; import { getScenario } from '../scenarios'; import { createResultDir, formatPrettyChecks } from './utils'; @@ -17,7 +21,8 @@ async function executeClient( scenarioName: string, serverUrl: string, timeout: number = 30000, - context?: Record + context?: Record, + specVersion?: SpecVersion ): Promise { const commandParts = command.split(' '); const executable = commandParts[0]; @@ -34,6 +39,12 @@ async function executeClient( // 3. Semantic separation: scenario identifies "which test", context provides "test data" const env = { ...process.env }; env.MCP_CONFORMANCE_SCENARIO = scenarioName; + const protocolVersion = specVersion + ? specVersionToProtocolVersion(specVersion) + : undefined; + if (protocolVersion) { + env.MCP_CONFORMANCE_PROTOCOL_VERSION = protocolVersion; + } if (context) { // Include scenario name in context for discriminated union parsing env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ @@ -92,7 +103,8 @@ export async function runConformanceTest( clientCommand: string, scenarioName: string, timeout: number = 30000, - outputDir?: string + outputDir?: string, + specVersion?: SpecVersion ): Promise<{ checks: ConformanceCheck[]; clientOutput: ClientExecutionResult; @@ -123,7 +135,8 @@ export async function runConformanceTest( scenarioName, urls.serverUrl, timeout, - urls.context + urls.context, + specVersion ); // Print stdout/stderr if client exited with nonzero code diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index 70fb0d1a..ccf0d127 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -3,7 +3,9 @@ import { Scenario, ScenarioUrls, ConformanceCheck, - SpecVersion + SpecVersion, + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION } from '../../types'; import { clientChecks } from '../../checks/index'; @@ -117,11 +119,15 @@ export class InitializeScenario implements Scenario { this.checks.push(clientChecks.createServerInfoCheck(serverInfo)); // Echo back client's version if valid, otherwise use latest - const VALID_VERSIONS = ['2025-06-18', '2025-11-25']; + const VALID_VERSIONS = [ + '2025-06-18', + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION + ]; const clientVersion = initializeRequest?.protocolVersion; const responseVersion = VALID_VERSIONS.includes(clientVersion) ? clientVersion - : '2025-11-25'; + : LATEST_SPEC_VERSION; const response = { jsonrpc: '2.0', diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 70808490..f364073e 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -2,7 +2,9 @@ import { Scenario, ClientScenario, ClientScenarioForAuthorizationServer, - SpecVersion + SpecVersion, + DATED_SPEC_VERSIONS, + LATEST_SPEC_VERSION } from '../types'; import { InitializeScenario } from './client/initialize'; import { ToolsCallScenario } from './client/tools_call'; @@ -257,9 +259,7 @@ export { listMetadataScenarios }; // All valid spec versions, used by the CLI to validate --spec-version input. export const ALL_SPEC_VERSIONS: SpecVersion[] = [ - '2025-03-26', - '2025-06-18', - '2025-11-25', + ...DATED_SPEC_VERSIONS, 'draft', 'extension' ]; @@ -273,15 +273,31 @@ export function resolveSpecVersion(value: string): SpecVersion { process.exit(1); } +// `draft` selects everything in the latest dated release plus scenarios tagged +// draft-only, so SEP authors can run the full suite against an SDK tracking the +// in-progress spec without retagging core scenarios. +function matchesSpecVersion( + scenario: { specVersions: SpecVersion[] }, + version: SpecVersion +): boolean { + if (version === 'draft') { + return ( + scenario.specVersions.includes('draft') || + scenario.specVersions.includes(LATEST_SPEC_VERSION) + ); + } + return scenario.specVersions.includes(version); +} + export function listScenariosForSpec(version: SpecVersion): string[] { return scenariosList - .filter((s) => s.specVersions.includes(version)) + .filter((s) => matchesSpecVersion(s, version)) .map((s) => s.name); } export function listClientScenariosForSpec(version: SpecVersion): string[] { return allClientScenariosList - .filter((s) => s.specVersions.includes(version)) + .filter((s) => matchesSpecVersion(s, version)) .map((s) => s.name); } @@ -289,7 +305,7 @@ export function listClientScenariosForAuthorizationServerForSpec( version: SpecVersion ): string[] { return allClientScenariosListForAuthorizationServer - .filter((s) => s.specVersions.includes(version)) + .filter((s) => matchesSpecVersion(s, version)) .map((s) => s.name); } diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index 0b8e652f..d875df85 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -3,9 +3,16 @@ import { listScenarios, listClientScenarios, listScenariosForSpec, + listDraftScenarios, getScenarioSpecVersions, ALL_SPEC_VERSIONS } from './index'; +import { + DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION, + LATEST_SPEC_VERSION, + specVersionToProtocolVersion +} from '../types'; describe('specVersions helpers', () => { it('every Scenario has specVersions', () => { @@ -69,26 +76,41 @@ describe('specVersions helpers', () => { } }); - it('draft and extension scenarios are isolated', () => { - const draft = listScenariosForSpec('draft'); - for (const name of draft) { - expect(getScenarioSpecVersions(name)).toContain('draft'); + it('--spec-version draft is a superset of the latest dated release', () => { + const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION)); + const draft = new Set(listScenariosForSpec('draft')); + for (const name of latest) { + expect(draft.has(name)).toBe(true); } - const ext = listScenariosForSpec('extension'); - for (const name of ext) { - expect(getScenarioSpecVersions(name)).toContain('extension'); + for (const name of listDraftScenarios()) { + expect(draft.has(name)).toBe(true); + } + }); + + it('draft-tagged scenarios are not also tagged with a dated version', () => { + for (const name of listDraftScenarios()) { + const versions = getScenarioSpecVersions(name)!; + for (const dated of DATED_SPEC_VERSIONS) { + expect( + versions, + `scenario "${name}" is tagged with both 'draft' and '${dated}'` + ).not.toContain(dated); + } } }); - it('draft scenarios are not in dated versions', () => { - const draft = listScenariosForSpec('draft'); - const dated = new Set([ - ...listScenariosForSpec('2025-03-26'), - ...listScenariosForSpec('2025-06-18'), - ...listScenariosForSpec('2025-11-25') - ]); - for (const name of draft) { - expect(dated.has(name)).toBe(false); + it('specVersionToProtocolVersion maps tags to wire versions', () => { + expect(specVersionToProtocolVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION); + expect(specVersionToProtocolVersion(LATEST_SPEC_VERSION)).toBe( + LATEST_SPEC_VERSION + ); + expect(specVersionToProtocolVersion('extension')).toBeUndefined(); + }); + + it('extension scenarios are isolated', () => { + const ext = listScenariosForSpec('extension'); + for (const name of ext) { + expect(getScenarioSpecVersions(name)).toContain('extension'); } }); }); diff --git a/src/types.ts b/src/types.ts index 193686f6..33ab7b37 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,12 +23,34 @@ export interface ConformanceCheck { logs?: string[]; } -export type SpecVersion = - | '2025-03-26' - | '2025-06-18' - | '2025-11-25' - | 'draft' - | 'extension'; +export const DATED_SPEC_VERSIONS = [ + '2025-03-26', + '2025-06-18', + '2025-11-25' +] as const; + +export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number]; + +export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25'; + +// Mirrors LATEST_PROTOCOL_VERSION in the spec repo's schema/draft/schema.ts. +// Bump when that constant changes. +export const DRAFT_PROTOCOL_VERSION = 'DRAFT-2026-v1'; + +export type SpecVersion = DatedSpecVersion | 'draft' | 'extension'; + +export function specVersionToProtocolVersion( + version: SpecVersion +): string | undefined { + if (version === 'draft') return DRAFT_PROTOCOL_VERSION; + // TODO(#253 follow-up): 'extension' isn't a spec version — it's a scenario + // category that got lumped into SpecVersion so `--spec-version extension` + // could reuse the filter plumbing. It has no corresponding wire + // protocolVersion. Split it out of this type when moving to + // introducedIn/removedIn tagging. + if (version === 'extension') return undefined; + return version; +} export interface ScenarioUrls { serverUrl: string; From 7a87bf7905946c0dd5d4c64ffb8968f72dec63b8 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Fri, 24 Apr 2026 14:16:33 +0100 Subject: [PATCH 2/2] review: collapse SpecVersion to wire strings; dedup NEGOTIABLE_PROTOCOL_VERSIONS; split out 'extension' Addresses review feedback from @felixweinberger and @mikekistler: - SpecVersion is now always a wire protocolVersion string (DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION). The separate 'draft' tag literal is gone from the type system; 'draft' survives only as a CLI input alias in resolveSpecVersion. Removes the tag-vs-wire confusion and the specVersionToProtocolVersion mapping. - NEGOTIABLE_PROTOCOL_VERSIONS in types.ts is the single source for what the mock server accepts on initialize (was duplicated in checks/client.ts and scenarios/client/initialize.ts). - 'extension' moved out of SpecVersion into a separate ScenarioSpecTag type. --spec-version extension is no longer valid; extension scenarios remain reachable via --suite extensions. - Draft-tagged scenarios now use the DRAFT_PROTOCOL_VERSION constant so a draft revision bump is a one-line change in types.ts. --- README.md | 4 +- src/checks/client.ts | 17 +++----- src/runner/client.ts | 13 ++---- .../client/auth/client-credentials.ts | 6 +-- src/scenarios/client/auth/cross-app-access.ts | 4 +- src/scenarios/client/auth/offline-access.ts | 10 +++-- .../client/auth/resource-mismatch.ts | 8 +++- src/scenarios/client/initialize.ts | 9 +--- src/scenarios/index.ts | 28 ++++++++----- src/scenarios/server/resources.ts | 9 +++- src/scenarios/spec-version.test.ts | 41 +++++++++++------- .../checks/test-conformance-results.ts | 16 +++++-- src/tier-check/output.ts | 5 ++- src/tier-check/types.ts | 4 +- src/types.ts | 42 +++++++++++-------- 15 files changed, 121 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 5ba49836..b2b5f0e4 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,12 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") - `--suite` - Run a suite of tests in parallel (e.g., "auth") -- `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `draft`). `draft` runs the latest dated release plus any draft-only scenarios +- `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output -The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, the corresponding wire protocol version is forwarded as `MCP_CONFORMANCE_PROTOCOL_VERSION` (e.g., `--spec-version draft` sets it to the current draft identifier such as `DRAFT-2026-v1`); example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. +The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. ### Server Testing diff --git a/src/checks/client.ts b/src/checks/client.ts index 340f603f..89b11068 100644 --- a/src/checks/client.ts +++ b/src/checks/client.ts @@ -2,7 +2,7 @@ import { ConformanceCheck, CheckStatus, LATEST_SPEC_VERSION, - DRAFT_PROTOCOL_VERSION + NEGOTIABLE_PROTOCOL_VERSIONS } from '../types'; export function createServerInfoCheck(serverInfo: { @@ -28,13 +28,6 @@ export function createServerInfoCheck(serverInfo: { }; } -// Protocol versions the mock server will accept on initialize. -const VALID_PROTOCOL_VERSIONS = [ - '2025-06-18', - LATEST_SPEC_VERSION, - DRAFT_PROTOCOL_VERSION -]; - export function createClientInitializationCheck( initializeRequest: any, expectedSpecVersion: string = LATEST_SPEC_VERSION @@ -42,9 +35,11 @@ export function createClientInitializationCheck( const protocolVersionSent = initializeRequest?.protocolVersion; // Accept known valid versions OR custom expected version (for backward compatibility) - const validVersions = VALID_PROTOCOL_VERSIONS.includes(expectedSpecVersion) - ? VALID_PROTOCOL_VERSIONS - : [...VALID_PROTOCOL_VERSIONS, expectedSpecVersion]; + const validVersions = NEGOTIABLE_PROTOCOL_VERSIONS.includes( + expectedSpecVersion + ) + ? NEGOTIABLE_PROTOCOL_VERSIONS + : [...NEGOTIABLE_PROTOCOL_VERSIONS, expectedSpecVersion]; const versionMatch = validVersions.includes(protocolVersionSent); const errors: string[] = []; diff --git a/src/runner/client.ts b/src/runner/client.ts index 571d8f9d..1bf8c9f6 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,11 +1,7 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; -import { - ConformanceCheck, - SpecVersion, - specVersionToProtocolVersion -} from '../types'; +import { ConformanceCheck, SpecVersion } from '../types'; import { getScenario } from '../scenarios'; import { createResultDir, formatPrettyChecks } from './utils'; @@ -39,11 +35,8 @@ async function executeClient( // 3. Semantic separation: scenario identifies "which test", context provides "test data" const env = { ...process.env }; env.MCP_CONFORMANCE_SCENARIO = scenarioName; - const protocolVersion = specVersion - ? specVersionToProtocolVersion(specVersion) - : undefined; - if (protocolVersion) { - env.MCP_CONFORMANCE_PROTOCOL_VERSION = protocolVersion; + if (specVersion) { + env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion; } if (context) { // Include scenario name in context for discriminated union parsing diff --git a/src/scenarios/client/auth/client-credentials.ts b/src/scenarios/client/auth/client-credentials.ts index 79ab1b2e..5923da22 100644 --- a/src/scenarios/client/auth/client-credentials.ts +++ b/src/scenarios/client/auth/client-credentials.ts @@ -4,7 +4,7 @@ import type { Scenario, ConformanceCheck, ScenarioUrls, - SpecVersion + ScenarioSpecTag } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; @@ -37,7 +37,7 @@ async function generateTestKeypair(): Promise<{ */ export class ClientCredentialsJwtScenario implements Scenario { name = 'auth/client-credentials-jwt'; - specVersions: SpecVersion[] = ['extension']; + specVersions: ScenarioSpecTag[] = ['extension']; description = 'Tests OAuth client_credentials flow with private_key_jwt authentication (SEP-1046)'; @@ -256,7 +256,7 @@ export class ClientCredentialsJwtScenario implements Scenario { */ export class ClientCredentialsBasicScenario implements Scenario { name = 'auth/client-credentials-basic'; - specVersions: SpecVersion[] = ['extension']; + specVersions: ScenarioSpecTag[] = ['extension']; description = 'Tests OAuth client_credentials flow with client_secret_basic authentication'; diff --git a/src/scenarios/client/auth/cross-app-access.ts b/src/scenarios/client/auth/cross-app-access.ts index 05a351ba..3eaff5a2 100644 --- a/src/scenarios/client/auth/cross-app-access.ts +++ b/src/scenarios/client/auth/cross-app-access.ts @@ -5,7 +5,7 @@ import type { Scenario, ConformanceCheck, ScenarioUrls, - SpecVersion + ScenarioSpecTag } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; @@ -60,7 +60,7 @@ async function createIdpIdToken( */ export class CrossAppAccessCompleteFlowScenario implements Scenario { name = 'auth/cross-app-access-complete-flow'; - specVersions: SpecVersion[] = ['extension']; + specVersions: ScenarioSpecTag[] = ['extension']; description = 'Tests complete SEP-990 flow: token exchange + JWT bearer grant (Enterprise Managed OAuth)'; diff --git a/src/scenarios/client/auth/offline-access.ts b/src/scenarios/client/auth/offline-access.ts index e4ed7775..a9029a43 100644 --- a/src/scenarios/client/auth/offline-access.ts +++ b/src/scenarios/client/auth/offline-access.ts @@ -1,5 +1,9 @@ import type { Scenario, ConformanceCheck } from '../../../types'; -import { ScenarioUrls, SpecVersion } from '../../../types'; +import { + ScenarioUrls, + SpecVersion, + DRAFT_PROTOCOL_VERSION +} from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; import { ServerLifecycle } from './helpers/serverLifecycle'; @@ -23,7 +27,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier'; */ export class OfflineAccessScopeScenario implements Scenario { name = 'auth/offline-access-scope'; - specVersions: SpecVersion[] = ['draft']; + specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = 'Tests that a client that wants a refresh token handles offline_access scope and refresh_token grant type when AS supports them (SEP-2207)'; @@ -227,7 +231,7 @@ export class OfflineAccessScopeScenario implements Scenario { */ export class OfflineAccessNotSupportedScenario implements Scenario { name = 'auth/offline-access-not-supported'; - specVersions: SpecVersion[] = ['draft']; + specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = 'Tests that client does not request offline_access when AS does not list it in scopes_supported (SEP-2207)'; diff --git a/src/scenarios/client/auth/resource-mismatch.ts b/src/scenarios/client/auth/resource-mismatch.ts index dd76c68c..32b95253 100644 --- a/src/scenarios/client/auth/resource-mismatch.ts +++ b/src/scenarios/client/auth/resource-mismatch.ts @@ -1,5 +1,9 @@ import type { Scenario, ConformanceCheck } from '../../../types.js'; -import { ScenarioUrls, SpecVersion } from '../../../types.js'; +import { + ScenarioUrls, + SpecVersion, + DRAFT_PROTOCOL_VERSION +} from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; import { createServer } from './helpers/createServer.js'; import { ServerLifecycle } from './helpers/serverLifecycle.js'; @@ -27,7 +31,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier.js'; */ export class ResourceMismatchScenario implements Scenario { name = 'auth/resource-mismatch'; - specVersions: SpecVersion[] = ['draft']; + specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = 'Tests that client rejects when PRM resource does not match server URL'; allowClientError = true; diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index ccf0d127..191ed30c 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -5,7 +5,7 @@ import { ConformanceCheck, SpecVersion, LATEST_SPEC_VERSION, - DRAFT_PROTOCOL_VERSION + NEGOTIABLE_PROTOCOL_VERSIONS } from '../../types'; import { clientChecks } from '../../checks/index'; @@ -119,13 +119,8 @@ export class InitializeScenario implements Scenario { this.checks.push(clientChecks.createServerInfoCheck(serverInfo)); // Echo back client's version if valid, otherwise use latest - const VALID_VERSIONS = [ - '2025-06-18', - LATEST_SPEC_VERSION, - DRAFT_PROTOCOL_VERSION - ]; const clientVersion = initializeRequest?.protocolVersion; - const responseVersion = VALID_VERSIONS.includes(clientVersion) + const responseVersion = NEGOTIABLE_PROTOCOL_VERSIONS.includes(clientVersion) ? clientVersion : LATEST_SPEC_VERSION; diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index f364073e..0e2191aa 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -3,7 +3,9 @@ import { ClientScenario, ClientScenarioForAuthorizationServer, SpecVersion, + ScenarioSpecTag, DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../types'; import { InitializeScenario } from './client/initialize'; @@ -258,31 +260,35 @@ export function listDraftScenarios(): string[] { export { listMetadataScenarios }; // All valid spec versions, used by the CLI to validate --spec-version input. +// 'extension' is intentionally excluded — extension scenarios are off-timeline +// and selected via `--suite extensions`, not `--spec-version`. export const ALL_SPEC_VERSIONS: SpecVersion[] = [ ...DATED_SPEC_VERSIONS, - 'draft', - 'extension' + DRAFT_PROTOCOL_VERSION ]; export function resolveSpecVersion(value: string): SpecVersion { + if (value === 'draft') return DRAFT_PROTOCOL_VERSION; if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) { return value as SpecVersion; } console.error(`Unknown spec version: ${value}`); - console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); + console.error( + `Valid versions: ${ALL_SPEC_VERSIONS.join(', ')} (or 'draft' as an alias for ${DRAFT_PROTOCOL_VERSION})` + ); process.exit(1); } -// `draft` selects everything in the latest dated release plus scenarios tagged -// draft-only, so SEP authors can run the full suite against an SDK tracking the -// in-progress spec without retagging core scenarios. +// The draft version selects everything in the latest dated release plus +// scenarios tagged draft-only, so SEP authors can run the full suite against an +// SDK tracking the in-progress spec without retagging core scenarios. function matchesSpecVersion( - scenario: { specVersions: SpecVersion[] }, + scenario: { specVersions: ScenarioSpecTag[] }, version: SpecVersion ): boolean { - if (version === 'draft') { + if (version === DRAFT_PROTOCOL_VERSION) { return ( - scenario.specVersions.includes('draft') || + scenario.specVersions.includes(DRAFT_PROTOCOL_VERSION) || scenario.specVersions.includes(LATEST_SPEC_VERSION) ); } @@ -311,7 +317,7 @@ export function listClientScenariosForAuthorizationServerForSpec( export function getScenarioSpecVersions( name: string -): SpecVersion[] | undefined { +): ScenarioSpecTag[] | undefined { return ( scenarios.get(name)?.specVersions ?? clientScenarios.get(name)?.specVersions ?? @@ -319,4 +325,4 @@ export function getScenarioSpecVersions( ); } -export type { SpecVersion }; +export type { SpecVersion, ScenarioSpecTag }; diff --git a/src/scenarios/server/resources.ts b/src/scenarios/server/resources.ts index bea27067..9f9d636a 100644 --- a/src/scenarios/server/resources.ts +++ b/src/scenarios/server/resources.ts @@ -2,7 +2,12 @@ * Resources test scenarios for MCP servers */ -import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types'; +import { + ClientScenario, + ConformanceCheck, + SpecVersion, + DRAFT_PROTOCOL_VERSION +} from '../../types'; import { connectToServer } from './client-helper'; import { TextResourceContents, @@ -438,7 +443,7 @@ Example request: export class ResourcesNotFoundErrorScenario implements ClientScenario { name = 'sep-2164-resource-not-found'; - specVersions: SpecVersion[] = ['draft']; + specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test error handling for non-existent resources (SEP-2164). **Server Implementation Requirements:** diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index d875df85..87b79703 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -4,16 +4,23 @@ import { listClientScenarios, listScenariosForSpec, listDraftScenarios, + listExtensionScenarios, getScenarioSpecVersions, + resolveSpecVersion, ALL_SPEC_VERSIONS } from './index'; import { DATED_SPEC_VERSIONS, DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION, - specVersionToProtocolVersion + ScenarioSpecTag } from '../types'; +const ALL_SCENARIO_SPEC_TAGS: ScenarioSpecTag[] = [ + ...ALL_SPEC_VERSIONS, + 'extension' +]; + describe('specVersions helpers', () => { it('every Scenario has specVersions', () => { for (const name of listScenarios()) { @@ -24,7 +31,7 @@ describe('specVersions helpers', () => { ).toBeDefined(); expect(versions!.length).toBeGreaterThan(0); for (const v of versions!) { - expect(ALL_SPEC_VERSIONS).toContain(v); + expect(ALL_SCENARIO_SPEC_TAGS).toContain(v); } } }); @@ -38,7 +45,7 @@ describe('specVersions helpers', () => { ).toBeDefined(); expect(versions!.length).toBeGreaterThan(0); for (const v of versions!) { - expect(ALL_SPEC_VERSIONS).toContain(v); + expect(ALL_SCENARIO_SPEC_TAGS).toContain(v); } } }); @@ -76,9 +83,9 @@ describe('specVersions helpers', () => { } }); - it('--spec-version draft is a superset of the latest dated release', () => { + it('the draft spec version is a superset of the latest dated release', () => { const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION)); - const draft = new Set(listScenariosForSpec('draft')); + const draft = new Set(listScenariosForSpec(DRAFT_PROTOCOL_VERSION)); for (const name of latest) { expect(draft.has(name)).toBe(true); } @@ -93,24 +100,26 @@ describe('specVersions helpers', () => { for (const dated of DATED_SPEC_VERSIONS) { expect( versions, - `scenario "${name}" is tagged with both 'draft' and '${dated}'` + `scenario "${name}" is tagged with both DRAFT_PROTOCOL_VERSION and '${dated}'` ).not.toContain(dated); } } }); - it('specVersionToProtocolVersion maps tags to wire versions', () => { - expect(specVersionToProtocolVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION); - expect(specVersionToProtocolVersion(LATEST_SPEC_VERSION)).toBe( - LATEST_SPEC_VERSION - ); - expect(specVersionToProtocolVersion('extension')).toBeUndefined(); + it("resolveSpecVersion accepts 'draft' as an alias", () => { + expect(resolveSpecVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION); + expect(resolveSpecVersion(LATEST_SPEC_VERSION)).toBe(LATEST_SPEC_VERSION); }); - it('extension scenarios are isolated', () => { - const ext = listScenariosForSpec('extension'); - for (const name of ext) { - expect(getScenarioSpecVersions(name)).toContain('extension'); + it('extension-tagged scenarios are not selected by any --spec-version', () => { + for (const version of ALL_SPEC_VERSIONS) { + const selected = new Set(listScenariosForSpec(version)); + for (const name of listExtensionScenarios()) { + expect( + selected.has(name), + `extension scenario "${name}" was selected by --spec-version ${version}` + ).toBe(false); + } } }); }); diff --git a/src/tier-check/checks/test-conformance-results.ts b/src/tier-check/checks/test-conformance-results.ts index 2a9ce432..ba16be21 100644 --- a/src/tier-check/checks/test-conformance-results.ts +++ b/src/tier-check/checks/test-conformance-results.ts @@ -10,14 +10,22 @@ import { listClientScenariosForSpec, getScenarioSpecVersions } from '../../scenarios'; -import { ConformanceCheck, SpecVersion } from '../../types'; +import { + ConformanceCheck, + DRAFT_PROTOCOL_VERSION, + ScenarioSpecTag, + SpecVersion +} from '../../types'; -const NON_SCORING_VERSIONS: SpecVersion[] = ['draft', 'extension']; +const NON_SCORING_TAGS: ScenarioSpecTag[] = [ + DRAFT_PROTOCOL_VERSION, + 'extension' +]; /** Whether a scenario counts toward tier scoring (has at least one date-versioned spec). */ -function isTierScoring(specVersions?: SpecVersion[]): boolean { +function isTierScoring(specVersions?: ScenarioSpecTag[]): boolean { if (!specVersions || specVersions.length === 0) return true; // unknown = count it - return specVersions.some((v) => !NON_SCORING_VERSIONS.includes(v)); + return specVersions.some((v) => !NON_SCORING_TAGS.includes(v)); } /** diff --git a/src/tier-check/output.ts b/src/tier-check/output.ts index f08a47d6..ab0d2fbe 100644 --- a/src/tier-check/output.ts +++ b/src/tier-check/output.ts @@ -1,4 +1,5 @@ import { TierScorecard, CheckStatus, ConformanceResult } from './types'; +import { DATED_SPEC_VERSIONS, DRAFT_PROTOCOL_VERSION } from '../types'; const COLORS = { RESET: '\x1b[0m', @@ -23,9 +24,9 @@ function statusIcon(status: CheckStatus): string { } } -const TIER_SPEC_VERSIONS = ['2025-03-26', '2025-06-18', '2025-11-25'] as const; +const TIER_SPEC_VERSIONS = DATED_SPEC_VERSIONS; -const INFO_SPEC_VERSIONS = ['draft', 'extension'] as const; +const INFO_SPEC_VERSIONS = [DRAFT_PROTOCOL_VERSION, 'extension'] as const; type Cell = { passed: number; total: number }; diff --git a/src/tier-check/types.ts b/src/tier-check/types.ts index a9830f4c..eacbcbe6 100644 --- a/src/tier-check/types.ts +++ b/src/tier-check/types.ts @@ -1,4 +1,4 @@ -import type { SpecVersion } from '../types'; +import type { ScenarioSpecTag } from '../types'; export type CheckStatus = 'pass' | 'fail' | 'partial' | 'skipped'; @@ -17,7 +17,7 @@ export interface ConformanceResult extends CheckResult { passed: boolean; checks_passed: number; checks_failed: number; - specVersions?: SpecVersion[]; + specVersions?: ScenarioSpecTag[]; }>; } diff --git a/src/types.ts b/src/types.ts index 33ab7b37..51cee73c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,24 +33,30 @@ export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number]; export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25'; -// Mirrors LATEST_PROTOCOL_VERSION in the spec repo's schema/draft/schema.ts. -// Bump when that constant changes. +/** + * Wire `protocolVersion` for the in-progress spec. Mirrors + * `LATEST_PROTOCOL_VERSION` in the spec repo's `schema/draft/schema.ts`; + * bump when that constant changes. + */ export const DRAFT_PROTOCOL_VERSION = 'DRAFT-2026-v1'; -export type SpecVersion = DatedSpecVersion | 'draft' | 'extension'; +// Wire protocolVersion strings the mock server will negotiate on initialize. +export const NEGOTIABLE_PROTOCOL_VERSIONS: readonly string[] = [ + '2025-06-18', + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION +]; -export function specVersionToProtocolVersion( - version: SpecVersion -): string | undefined { - if (version === 'draft') return DRAFT_PROTOCOL_VERSION; - // TODO(#253 follow-up): 'extension' isn't a spec version — it's a scenario - // category that got lumped into SpecVersion so `--spec-version extension` - // could reuse the filter plumbing. It has no corresponding wire - // protocolVersion. Split it out of this type when moving to - // introducedIn/removedIn tagging. - if (version === 'extension') return undefined; - return version; -} +/** + * A spec revision the conformance suite can target via `--spec-version`. + * Always a wire `protocolVersion` string. The CLI also accepts `'draft'` as + * an alias for {@link DRAFT_PROTOCOL_VERSION}. + */ +export type SpecVersion = DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION; + +// Scenarios may also be tagged 'extension' to mark them as off-timeline +// (selectable via --suite extensions, never via --spec-version). See #256. +export type ScenarioSpecTag = SpecVersion | 'extension'; export interface ScenarioUrls { serverUrl: string; @@ -65,7 +71,7 @@ export interface ScenarioUrls { export interface Scenario { name: string; description: string; - specVersions: SpecVersion[]; + specVersions: ScenarioSpecTag[]; /** * If true, a non-zero client exit code is expected and will not cause the test to fail. * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth). @@ -79,13 +85,13 @@ export interface Scenario { export interface ClientScenario { name: string; description: string; - specVersions: SpecVersion[]; + specVersions: ScenarioSpecTag[]; run(serverUrl: string): Promise; } export interface ClientScenarioForAuthorizationServer { name: string; description: string; - specVersions: SpecVersion[]; + specVersions: ScenarioSpecTag[]; run(serverUrl: string): Promise; }