Skip to content
47 changes: 46 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,27 @@ npx @modelcontextprotocol/conformance server --url <url> [--scenario <scenario>]
**Options:**

- `--url` - URL of the server to test
- `--scenario <scenario>` - Test scenario to run (e.g., "server-initialize". Runs all available scenarios by default
- `--scenario <scenario>` - Test scenario to run (e.g., "server-initialize"). Runs all available scenarios by default
- `--suite <suite>` - Suite to run: "active" (default), "all", "pending", or "auth"
- `--auth` - Include OAuth conformance tests when running active suite
- `--verbose` - Show verbose output (JSON format)
Comment thread
tobinsouth marked this conversation as resolved.
Outdated

### Server OAuth Testing
Comment thread
tobinsouth marked this conversation as resolved.
Outdated

To test OAuth implementation on your server:
Comment thread
tobinsouth marked this conversation as resolved.
Outdated

```bash
# Run only OAuth conformance tests
npx @modelcontextprotocol/conformance server --url http://localhost:3000/mcp --suite auth

# Run active tests plus OAuth tests
npx @modelcontextprotocol/conformance server --url http://localhost:3000/mcp --auth

# Run a specific OAuth scenario
npx @modelcontextprotocol/conformance server --url http://localhost:3000/mcp --scenario server/auth-prm-discovery
```

OAuth scenarios validate server OAuth infrastructure without requiring tokens.
Comment thread
tobinsouth marked this conversation as resolved.
Outdated

## Test Results

Expand Down Expand Up @@ -114,6 +134,31 @@ Run `npx @modelcontextprotocol/conformance list --server` to see all available s
- **resources-\*** - Resource management scenarios
- **prompts-\*** - Prompt management scenarios

### Server OAuth Scenarios

Run `npx @modelcontextprotocol/conformance list --auth` to see all OAuth scenarios.

**Discovery & Metadata:**

- `server/auth-prm-discovery` - Protected Resource Metadata (RFC 9728)
- `server/auth-as-metadata-discovery` - Authorization Server metadata (RFC 8414)
- `server/auth-discovery-mechanism` - Discovery endpoint availability

**OAuth Features:**

- `server/auth-as-pkce-support` - PKCE S256 support (RFC 7636)
- `server/auth-as-cimd-supported` - Client ID Metadata Document support
- `server/auth-as-token-auth-methods` - Token endpoint auth methods
- `server/auth-as-grant-types` - Grant types including client_credentials

**HTTP Responses:**

- `server/auth-401-unauthorized` - 401 response validation
- `server/auth-www-authenticate-header` - WWW-Authenticate header (RFC 6750)
- `server/auth-prm-resource-validation` - Resource URI validation

These scenarios test OAuth compliance without requiring tokens.

Comment thread
tobinsouth marked this conversation as resolved.
Outdated
## Architecture

See `src/runner/DESIGN.md` for detailed architecture documentation.
Expand Down
37 changes: 30 additions & 7 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ import {
listActiveClientScenarios,
listPendingClientScenarios,
listAuthScenarios,
listMetadataScenarios
listMetadataScenarios,
listServerAuthScenarios
} from './scenarios';
import { ConformanceCheck } from './types';
import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
Expand Down Expand Up @@ -201,9 +202,10 @@ program
)
.option(
'--suite <suite>',
'Suite to run: "active" (default, excludes pending), "all", or "pending"',
'Suite to run: "active" (default), "all", "pending", or "auth"',
'active'
)
.option('--auth', 'Include OAuth conformance tests (server/auth-* scenarios)')
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options) => {
try {
Expand All @@ -228,22 +230,31 @@ program
} else {
// Run scenarios based on suite
const suite = options.suite?.toLowerCase() || 'active';
const includeAuth = options.auth ?? false;
let scenarios: string[];

if (suite === 'all') {
scenarios = listClientScenarios();
} else if (suite === 'active') {
scenarios = listActiveClientScenarios();
// Add auth scenarios if --auth flag is set
if (includeAuth) {
scenarios = [...scenarios, ...listServerAuthScenarios()];
}
} else if (suite === 'pending') {
scenarios = listPendingClientScenarios();
} else if (suite === 'auth') {
// Run only auth scenarios
scenarios = listServerAuthScenarios();
} else {
console.error(`Unknown suite: ${suite}`);
console.error('Available suites: active, all, pending');
console.error('Available suites: active, all, pending, auth');
process.exit(1);
}

const authNote = includeAuth && suite !== 'auth' ? ' (with auth)' : '';
console.log(
`Running ${suite} suite (${scenarios.length} scenarios) against ${validated.url}\n`
`Running ${suite}${authNote} suite (${scenarios.length} scenarios) against ${validated.url}\n`
);

const allResults: { scenario: string; checks: ConformanceCheck[] }[] =
Expand Down Expand Up @@ -300,15 +311,27 @@ program
.description('List available test scenarios')
.option('--client', 'List client scenarios')
.option('--server', 'List server scenarios')
.option('--auth', 'List server OAuth auth scenarios')
.action((options) => {
if (options.server || (!options.client && !options.server)) {
const showAll = !options.client && !options.server && !options.auth;

if (options.server || showAll) {
console.log('Server scenarios (test against a server):');
const serverScenarios = listClientScenarios();
serverScenarios.forEach((s) => console.log(` - ${s}`));
}

if (options.client || (!options.client && !options.server)) {
if (options.server || (!options.client && !options.server)) {
if (options.auth || showAll) {
if (options.server || showAll) {
console.log('');
}
console.log('Server OAuth scenarios (use --auth or --suite auth):');
const authScenarios = listServerAuthScenarios();
authScenarios.forEach((s) => console.log(` - ${s}`));
}

if (options.client || showAll) {
if (options.server || options.auth || showAll) {
console.log('');
}
console.log('Client scenarios (test against a client):');
Expand Down
17 changes: 15 additions & 2 deletions src/scenarios/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@ import {
import { authScenariosList } from './client/auth/index';
import { listMetadataScenarios } from './client/auth/discovery-metadata';

// Server auth scenarios (OAuth conformance testing)
import {
serverAuthScenarios,
listServerAuthScenarios,
getServerAuthScenario
} from './server/auth/index';

// Pending client scenarios (not yet fully tested/implemented)
const pendingClientScenariosList: ClientScenario[] = [
// Elicitation scenarios (SEP-1330)
Expand Down Expand Up @@ -133,9 +140,12 @@ const activeClientScenariosList: ClientScenario[] =
)
);

// Client scenarios map - built from list
// Client scenarios map - built from list (includes server auth scenarios)
Comment thread
tobinsouth marked this conversation as resolved.
Outdated
export const clientScenarios = new Map<string, ClientScenario>(
allClientScenariosList.map((scenario) => [scenario.name, scenario])
[...allClientScenariosList, ...serverAuthScenarios].map((scenario) => [
scenario.name,
scenario
])
);

// Scenario scenarios
Expand Down Expand Up @@ -185,3 +195,6 @@ export function listAuthScenarios(): string[] {
}

export { listMetadataScenarios };

// Server auth scenario exports
export { listServerAuthScenarios, getServerAuthScenario, serverAuthScenarios };
193 changes: 193 additions & 0 deletions src/scenarios/server/auth/helpers/as-metadata.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
/**
* Authorization Server Metadata helpers.
*
* Provides utilities for fetching and validating AS metadata
* per RFC 8414 and OIDC Discovery.
*/

import { authFetch, buildPrmUrl, AuthTestResult } from './auth-fetch';

/**
* Result of fetching AS metadata.
*/
export interface AsMetadataResult {
/** Whether metadata was successfully fetched */
success: boolean;
/** The AS metadata document if successful */
metadata?: Record<string, unknown>;
/** The URL that was used to fetch metadata */
url?: string;
/** Whether OIDC discovery was used (vs RFC 8414) */
isOidc?: boolean;
/** Error message if fetch failed */
error?: string;
/** The AS URL from PRM */
asUrl?: string;
/** Raw response for debugging */
response?: AuthTestResult;
}

/**
* Result of fetching PRM.
*/
export interface PrmResult {
/** Whether PRM was successfully fetched */
success: boolean;
/** The PRM document if successful */
prm?: Record<string, unknown>;
/** The URL that was used to fetch PRM */
url?: string;
/** Error message if fetch failed */
error?: string;
/** Raw response for debugging */
response?: AuthTestResult;
}

/**
* Build AS metadata discovery URL.
*/
export function buildAsMetadataUrl(asUrl: string, useOidc: boolean): string {
const parsed = new URL(asUrl);
const base = `${parsed.protocol}//${parsed.host}`;

if (useOidc) {
return `${base}/.well-known/openid-configuration`;
}
return `${base}/.well-known/oauth-authorization-server`;
}

/**
* Fetch Protected Resource Metadata from a server.
*/
export async function fetchPrm(serverUrl: string): Promise<PrmResult> {
const pathBasedUrl = buildPrmUrl(serverUrl, true);
const rootUrl = buildPrmUrl(serverUrl, false);

// Try path-based first
try {
const response = await authFetch(pathBasedUrl);
if (
response.status === 200 &&
typeof response.body === 'object' &&
response.body !== null
) {
return {
success: true,
prm: response.body as Record<string, unknown>,
url: pathBasedUrl,
response
};
}
} catch {
// Will try root
}

// Try root
if (pathBasedUrl !== rootUrl) {
try {
const response = await authFetch(rootUrl);
if (
response.status === 200 &&
typeof response.body === 'object' &&
response.body !== null
) {
return {
success: true,
prm: response.body as Record<string, unknown>,
url: rootUrl,
response
};
}
} catch {
// Both failed
}
}

return {
success: false,
error: `No valid PRM found at ${pathBasedUrl} or ${rootUrl}`
};
}

/**
* Fetch Authorization Server metadata from the AS referenced in PRM.
*
* @param serverUrl - The MCP server URL
* @returns AS metadata result
*/
export async function fetchAsMetadata(
serverUrl: string
): Promise<AsMetadataResult> {
// First fetch PRM
const prmResult = await fetchPrm(serverUrl);

if (!prmResult.success || !prmResult.prm) {
return {
success: false,
error: prmResult.error || 'Failed to fetch PRM'
};
}

const authServers = prmResult.prm.authorization_servers as
| string[]
| undefined;

if (!Array.isArray(authServers) || authServers.length === 0) {
return {
success: false,
error: 'PRM missing authorization_servers array'
};
}

const asUrl = authServers[0];

// Try RFC 8414 first
const rfc8414Url = buildAsMetadataUrl(asUrl, false);
try {
const response = await authFetch(rfc8414Url);
if (
response.status === 200 &&
typeof response.body === 'object' &&
response.body !== null
) {
return {
success: true,
metadata: response.body as Record<string, unknown>,
url: rfc8414Url,
isOidc: false,
asUrl,
response
};
}
} catch {
// Will try OIDC
}

// Try OIDC Discovery
const oidcUrl = buildAsMetadataUrl(asUrl, true);
try {
const response = await authFetch(oidcUrl);
if (
response.status === 200 &&
typeof response.body === 'object' &&
response.body !== null
) {
return {
success: true,
metadata: response.body as Record<string, unknown>,
url: oidcUrl,
isOidc: true,
asUrl,
response
};
}
} catch {
// Both failed
}

return {
success: false,
error: `No AS metadata found at ${rfc8414Url} or ${oidcUrl}`,
asUrl
};
}
Loading
Loading