Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen
- `--command` - The command to run your MCP client (can include flags)
- `--scenario` - The test scenario to run (e.g., "initialize")
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
- `--timeout` - Timeout in milliseconds (default: 30000)
- `--verbose` - Show verbose output

The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data.
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it.

### Server Testing

Expand Down
26 changes: 26 additions & 0 deletions src/checks/checks.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { createClientInitializationCheck } from './client';
import { DRAFT_PROTOCOL_VERSION } from '../types';

describe('createClientInitializationCheck', () => {
it('should return SUCCESS for a valid initialize request', () => {
Expand Down Expand Up @@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => {
expect(check.errorMessage).toContain('Client version missing');
});

it('should accept the current draft protocol version', () => {
const request = {
protocolVersion: DRAFT_PROTOCOL_VERSION,
clientInfo: { name: 'TestClient', version: '1.0.0' }
};

const check = createClientInitializationCheck(request);
expect(check.status).toBe('SUCCESS');
expect(check.errorMessage).toBeUndefined();
});

it.each(['DRAFT-2025-v1', 'draft'])(
'should reject stale or non-canonical draft version %s',
(protocolVersion) => {
const request = {
protocolVersion,
clientInfo: { name: 'TestClient', version: '1.0.0' }
};

const check = createClientInitializationCheck(request);
expect(check.status).toBe('FAILURE');
expect(check.errorMessage).toContain('Version mismatch');
}
);

it('should support custom expected spec version', () => {
const request = {
protocolVersion: '2024-11-05',
Expand Down
20 changes: 12 additions & 8 deletions src/checks/client.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { ConformanceCheck, CheckStatus } from '../types';
import {
ConformanceCheck,
CheckStatus,
LATEST_SPEC_VERSION,
NEGOTIABLE_PROTOCOL_VERSIONS
} from '../types';

export function createServerInfoCheck(serverInfo: {
name: string;
Expand All @@ -23,19 +28,18 @@ export function createServerInfoCheck(serverInfo: {
};
}

// Valid MCP protocol versions
const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25'];

export function createClientInitializationCheck(
initializeRequest: any,
expectedSpecVersion: string = '2025-11-25'
expectedSpecVersion: string = LATEST_SPEC_VERSION
): ConformanceCheck {
const protocolVersionSent = initializeRequest?.protocolVersion;

// Accept known valid versions OR custom expected version (for backward compatibility)
const validVersions = VALID_PROTOCOL_VERSIONS.includes(expectedSpecVersion)
? VALID_PROTOCOL_VERSIONS
: [...VALID_PROTOCOL_VERSIONS, expectedSpecVersion];
const validVersions = NEGOTIABLE_PROTOCOL_VERSIONS.includes(
expectedSpecVersion
)
? NEGOTIABLE_PROTOCOL_VERSIONS
: [...NEGOTIABLE_PROTOCOL_VERSIONS, expectedSpecVersion];
const versionMatch = validVersions.includes(protocolVersionSent);

const errors: string[] = [];
Expand Down
6 changes: 4 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ program
options.command,
scenarioName,
timeout,
outputDir
outputDir,
specVersionFilter
);
return {
scenario: scenarioName,
Expand Down Expand Up @@ -259,7 +260,8 @@ program
validated.command,
validated.scenario,
timeout,
outputDir
outputDir,
specVersionFilter
);

const { overallFailure } = printClientResults(
Expand Down
14 changes: 10 additions & 4 deletions src/runner/client.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { spawn } from 'child_process';
import { promises as fs } from 'fs';
import path from 'path';
import { ConformanceCheck } from '../types';
import { ConformanceCheck, SpecVersion } from '../types';
import { getScenario } from '../scenarios';
import { createResultDir, formatPrettyChecks } from './utils';

Expand All @@ -17,7 +17,8 @@ async function executeClient(
scenarioName: string,
serverUrl: string,
timeout: number = 30000,
context?: Record<string, unknown>
context?: Record<string, unknown>,
specVersion?: SpecVersion
): Promise<ClientExecutionResult> {
const commandParts = command.split(' ');
const executable = commandParts[0];
Expand All @@ -34,6 +35,9 @@ async function executeClient(
// 3. Semantic separation: scenario identifies "which test", context provides "test data"
const env = { ...process.env };
env.MCP_CONFORMANCE_SCENARIO = scenarioName;
if (specVersion) {
env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion;
}
if (context) {
// Include scenario name in context for discriminated union parsing
env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
Expand Down Expand Up @@ -92,7 +96,8 @@ export async function runConformanceTest(
clientCommand: string,
scenarioName: string,
timeout: number = 30000,
outputDir?: string
outputDir?: string,
specVersion?: SpecVersion
): Promise<{
checks: ConformanceCheck[];
clientOutput: ClientExecutionResult;
Expand Down Expand Up @@ -123,7 +128,8 @@ export async function runConformanceTest(
scenarioName,
urls.serverUrl,
timeout,
urls.context
urls.context,
specVersion
);

// Print stdout/stderr if client exited with nonzero code
Expand Down
6 changes: 3 additions & 3 deletions src/scenarios/client/auth/client-credentials.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type {
Scenario,
ConformanceCheck,
ScenarioUrls,
SpecVersion
ScenarioSpecTag
} from '../../../types';
import { createAuthServer } from './helpers/createAuthServer';
import { createServer } from './helpers/createServer';
Expand Down Expand Up @@ -37,7 +37,7 @@ async function generateTestKeypair(): Promise<{
*/
export class ClientCredentialsJwtScenario implements Scenario {
name = 'auth/client-credentials-jwt';
specVersions: SpecVersion[] = ['extension'];
specVersions: ScenarioSpecTag[] = ['extension'];
description =
'Tests OAuth client_credentials flow with private_key_jwt authentication (SEP-1046)';

Expand Down Expand Up @@ -256,7 +256,7 @@ export class ClientCredentialsJwtScenario implements Scenario {
*/
export class ClientCredentialsBasicScenario implements Scenario {
name = 'auth/client-credentials-basic';
specVersions: SpecVersion[] = ['extension'];
specVersions: ScenarioSpecTag[] = ['extension'];
description =
'Tests OAuth client_credentials flow with client_secret_basic authentication';

Expand Down
4 changes: 2 additions & 2 deletions src/scenarios/client/auth/cross-app-access.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type {
Scenario,
ConformanceCheck,
ScenarioUrls,
SpecVersion
ScenarioSpecTag
} from '../../../types';
import { createAuthServer } from './helpers/createAuthServer';
import { createServer } from './helpers/createServer';
Expand Down Expand Up @@ -60,7 +60,7 @@ async function createIdpIdToken(
*/
export class CrossAppAccessCompleteFlowScenario implements Scenario {
name = 'auth/cross-app-access-complete-flow';
specVersions: SpecVersion[] = ['extension'];
specVersions: ScenarioSpecTag[] = ['extension'];
description =
'Tests complete SEP-990 flow: token exchange + JWT bearer grant (Enterprise Managed OAuth)';

Expand Down
10 changes: 7 additions & 3 deletions src/scenarios/client/auth/offline-access.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import type { Scenario, ConformanceCheck } from '../../../types';
import { ScenarioUrls, SpecVersion } from '../../../types';
import {
ScenarioUrls,
SpecVersion,
DRAFT_PROTOCOL_VERSION
} from '../../../types';
import { createAuthServer } from './helpers/createAuthServer';
import { createServer } from './helpers/createServer';
import { ServerLifecycle } from './helpers/serverLifecycle';
Expand All @@ -23,7 +27,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier';
*/
export class OfflineAccessScopeScenario implements Scenario {
name = 'auth/offline-access-scope';
specVersions: SpecVersion[] = ['draft'];
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
description =
'Tests that a client that wants a refresh token handles offline_access scope and refresh_token grant type when AS supports them (SEP-2207)';

Expand Down Expand Up @@ -227,7 +231,7 @@ export class OfflineAccessScopeScenario implements Scenario {
*/
export class OfflineAccessNotSupportedScenario implements Scenario {
name = 'auth/offline-access-not-supported';
specVersions: SpecVersion[] = ['draft'];
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
description =
'Tests that client does not request offline_access when AS does not list it in scopes_supported (SEP-2207)';

Expand Down
8 changes: 6 additions & 2 deletions src/scenarios/client/auth/resource-mismatch.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import type { Scenario, ConformanceCheck } from '../../../types.js';
import { ScenarioUrls, SpecVersion } from '../../../types.js';
import {
ScenarioUrls,
SpecVersion,
DRAFT_PROTOCOL_VERSION
} from '../../../types.js';
import { createAuthServer } from './helpers/createAuthServer.js';
import { createServer } from './helpers/createServer.js';
import { ServerLifecycle } from './helpers/serverLifecycle.js';
Expand Down Expand Up @@ -27,7 +31,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier.js';
*/
export class ResourceMismatchScenario implements Scenario {
name = 'auth/resource-mismatch';
specVersions: SpecVersion[] = ['draft'];
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
description =
'Tests that client rejects when PRM resource does not match server URL';
allowClientError = true;
Expand Down
9 changes: 5 additions & 4 deletions src/scenarios/client/initialize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ import {
Scenario,
ScenarioUrls,
ConformanceCheck,
SpecVersion
SpecVersion,
LATEST_SPEC_VERSION,
NEGOTIABLE_PROTOCOL_VERSIONS
} from '../../types';
import { clientChecks } from '../../checks/index';

Expand Down Expand Up @@ -117,11 +119,10 @@ export class InitializeScenario implements Scenario {
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));

// Echo back client's version if valid, otherwise use latest
const VALID_VERSIONS = ['2025-06-18', '2025-11-25'];
const clientVersion = initializeRequest?.protocolVersion;
const responseVersion = VALID_VERSIONS.includes(clientVersion)
const responseVersion = NEGOTIABLE_PROTOCOL_VERSIONS.includes(clientVersion)
? clientVersion
: '2025-11-25';
: LATEST_SPEC_VERSION;

const response = {
jsonrpc: '2.0',
Expand Down
46 changes: 34 additions & 12 deletions src/scenarios/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ import {
Scenario,
ClientScenario,
ClientScenarioForAuthorizationServer,
SpecVersion
SpecVersion,
ScenarioSpecTag,
DATED_SPEC_VERSIONS,
DRAFT_PROTOCOL_VERSION,
LATEST_SPEC_VERSION
} from '../types';
import { InitializeScenario } from './client/initialize';
import { ToolsCallScenario } from './client/tools_call';
Expand Down Expand Up @@ -256,51 +260,69 @@ export function listDraftScenarios(): string[] {
export { listMetadataScenarios };

// All valid spec versions, used by the CLI to validate --spec-version input.
// 'extension' is intentionally excluded — extension scenarios are off-timeline
// and selected via `--suite extensions`, not `--spec-version`.
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
'2025-03-26',
'2025-06-18',
'2025-11-25',
'draft',
'extension'
...DATED_SPEC_VERSIONS,
DRAFT_PROTOCOL_VERSION
];

export function resolveSpecVersion(value: string): SpecVersion {
if (value === 'draft') return DRAFT_PROTOCOL_VERSION;
if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
return value as SpecVersion;
}
console.error(`Unknown spec version: ${value}`);
console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
console.error(
`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')} (or 'draft' as an alias for ${DRAFT_PROTOCOL_VERSION})`
);
process.exit(1);
}

// The draft version selects everything in the latest dated release plus
// scenarios tagged draft-only, so SEP authors can run the full suite against an
// SDK tracking the in-progress spec without retagging core scenarios.
function matchesSpecVersion(
scenario: { specVersions: ScenarioSpecTag[] },
version: SpecVersion
): boolean {
if (version === DRAFT_PROTOCOL_VERSION) {
return (
scenario.specVersions.includes(DRAFT_PROTOCOL_VERSION) ||
scenario.specVersions.includes(LATEST_SPEC_VERSION)
);
}
return scenario.specVersions.includes(version);
}

export function listScenariosForSpec(version: SpecVersion): string[] {
return scenariosList
.filter((s) => s.specVersions.includes(version))
.filter((s) => matchesSpecVersion(s, version))
.map((s) => s.name);
}

export function listClientScenariosForSpec(version: SpecVersion): string[] {
return allClientScenariosList
.filter((s) => s.specVersions.includes(version))
.filter((s) => matchesSpecVersion(s, version))
.map((s) => s.name);
}

export function listClientScenariosForAuthorizationServerForSpec(
version: SpecVersion
): string[] {
return allClientScenariosListForAuthorizationServer
.filter((s) => s.specVersions.includes(version))
.filter((s) => matchesSpecVersion(s, version))
.map((s) => s.name);
}

export function getScenarioSpecVersions(
name: string
): SpecVersion[] | undefined {
): ScenarioSpecTag[] | undefined {
return (
scenarios.get(name)?.specVersions ??
clientScenarios.get(name)?.specVersions ??
clientScenariosForAuthorizationServer.get(name)?.specVersions
);
}

export type { SpecVersion };
export type { SpecVersion, ScenarioSpecTag };
9 changes: 7 additions & 2 deletions src/scenarios/server/resources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@
* Resources test scenarios for MCP servers
*/

import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
import {
ClientScenario,
ConformanceCheck,
SpecVersion,
DRAFT_PROTOCOL_VERSION
} from '../../types';
import { connectToServer } from './client-helper';
import {
TextResourceContents,
Expand Down Expand Up @@ -438,7 +443,7 @@ Example request:

export class ResourcesNotFoundErrorScenario implements ClientScenario {
name = 'sep-2164-resource-not-found';
specVersions: SpecVersion[] = ['draft'];
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
description = `Test error handling for non-existent resources (SEP-2164).

**Server Implementation Requirements:**
Expand Down
Loading
Loading