diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e15106e --- /dev/null +++ b/.gitignore @@ -0,0 +1,216 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml diff --git a/README.md b/README.md index 78f343a..9ca488a 100644 --- a/README.md +++ b/README.md @@ -1 +1,36 @@ -# quickstarts \ No newline at end of file +# MCPChecker Quickstarts + +Welcome to MCPChecker quickstarts! These hands-on examples help you get started with testing your MCP servers using MCPChecker. + +## What is MCPChecker? + +MCPChecker is a testing framework for MCP (Model Context Protocol) servers. It helps you verify that: +- Your tools are **discoverable** by AI agents +- Tool descriptions are **clear and actionable** +- Agents can **correctly use** your tools +- Your server handles **edge cases** properly + +Think of it as integration testing for AI tool use. + +## Available Quickstarts + +### 1. [Getting Started](./getting-started/) +**Difficulty:** Beginner +**Time:** 5 minutes +**What you'll learn:** +- Set up a basic MCP server +- Write your first MCPChecker test +- Run tests with Claude Code agent +- Understand test results + +**Perfect for:** First-time users who want to understand the basics of MCPChecker. + +## Getting Help + +- 📖 [Full Documentation](https://github.com/mcpchecker/mcpchecker) +- 💬 [Discussions](https://github.com/mcpchecker/mcpchecker/discussions) +- 🐛 [Report Issues](https://github.com/mcpchecker/mcpchecker/issues) + +## Contributing + +Have an idea for a quickstart? We'd love to hear from you! Check out our [contributing guidelines](https://github.com/mcpchecker/mcpchecker/CONTRIBUTING.md) to get started. diff --git a/getting-started/README.md b/getting-started/README.md new file mode 100644 index 0000000..dcc0b98 --- /dev/null +++ b/getting-started/README.md @@ -0,0 +1,218 @@ +# Getting Started with MCPChecker + +> **Get started with MCPChecker in 5 minutes** + +A minimal, batteries-included example showing how to test an HTTP MCP server with MCPChecker. + +**What you get:** +- ✅ Working HTTP MCP server (FastMCP official quickstart example) +- ✅ MCPChecker test for the `add` tool +- ✅ One command to run everything +- ✅ Perfect starting point to test your own MCP servers + +## Why MCPChecker? + +You've built an MCP server with tools. It works. But: +- **Is your tool description clear enough for an LLM to discover it?** +- **Can an AI agent actually use your tool correctly?** +- **Does your tool handle edge cases properly?** + +MCPChecker helps you test these questions automatically by: +1. Running real AI agents (like Claude Code) against your tools +2. Verifying agents can discover and use your tools correctly +3. Testing edge cases and error handling +4. Ensuring tool descriptions are clear and actionable + +Think of it as integration testing for AI tool use. + +## What's Included + +- **HTTP MCP Server** (`server/server.py`) - Simple FastMCP server with: + - `add` tool - Adds two numbers (taken from the official Python-SDK [Quickstart](https://github.com/modelcontextprotocol/python-sdk?tab=readme-ov-file#quickstart)) +- **MCPChecker Tests** (`evals/`) - Test task for the `add` tool + +The server uses **streamable HTTP transport** with `mcp.run(transport="streamable-http")`. + +## Quick Start + +### 0. Configure the Judge LLM + +MCPChecker uses an LLM to verify test results. Set these environment variables before running tests, for instance with OpenAI: + +```bash +export JUDGE_BASE_URL="https://api.openai.com/v1" +export JUDGE_API_KEY="sk-your-key-here" +export JUDGE_MODEL_NAME="gpt-4o-mini" +``` + +The judge LLM evaluates whether the agent completed tasks correctly by analyzing the agent's output. + +### 1. Install Prerequisites + +**Install Claude Code** (AI agent): + +Claude Code is used as the AI agent that runs the tests. + +```bash +# macOS/Linux +curl -fsSL https://claude.ai/install.sh | bash +``` + +See the [official installation guide](https://github.com/anthropics/claude-code?tab=readme-ov-file#get-started) for Windows and other installation methods. + +**Install uv** (Python package manager): +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +**Install mcpchecker** - Download from [releases](https://github.com/mcpchecker/mcpchecker/releases): + +```bash +# Linux (amd64) +curl -L -o mcpchecker https://github.com/mcpchecker/mcpchecker/releases/latest/download/mcpchecker-linux-amd64 +chmod +x mcpchecker +sudo mv mcpchecker /usr/local/bin/ + +# macOS (arm64 - Apple Silicon) +curl -L -o mcpchecker https://github.com/mcpchecker/mcpchecker/releases/latest/download/mcpchecker-darwin-arm64 +chmod +x mcpchecker +sudo mv mcpchecker /usr/local/bin/ +``` + +### 2. Start the MCP Server + +In one terminal, start the HTTP server: + +```bash +cd server +PORT=8000 ./server.py +``` + +The server will start on `http://localhost:8000/mcp` using streamable HTTP transport. + +**Note**: The `PORT` environment variable tells FastMCP which port to use. + +### 3. Run the Tests + +**Option A: Manual** (two terminals) + +In another terminal, run mcpchecker: + +```bash +cd evals +mcpchecker eval eval.yaml +``` + +You should see: +``` + Task: add-test + Path: /../getting-started/evals/tasks/add.yaml + Difficulty: easy + Task Status: PASSED + Assertions: PASSED (3/3) +``` + +## Understanding the Test Files + +This quickstart includes a complete evaluation setup. Let's look at what gets tested and how it's defined: + +### The Main Eval Configuration (`evals/eval.yaml`) + +```yaml +kind: Eval +metadata: + name: "demo-server-test" + +config: + # Use Claude Code as the AI agent + agent: + type: "builtin.claude-code" + + # MCP server configuration + mcpConfigFile: mcp-config.yaml + + # LLM judge configuration + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + + # Test tasks + taskSets: + - path: tasks/add.yaml + assertions: + toolsUsed: + - server: demo-server + tool: add + minToolCalls: 1 + maxToolCalls: 5 +``` + +**What this does:** +- Configures **Claude Code** as the agent that will attempt the tasks +- Points to **mcp-config.yaml** to connect to your MCP server +- Defines the **judge LLM** settings (using your environment variables) +- Loads tasks from **tasks/add.yaml** and asserts the `add` tool must be used + +### The Task Definition (`evals/tasks/add.yaml`) + +```yaml +kind: Task +apiVersion: mcpchecker/v1alpha2 +metadata: + name: "add-test" + difficulty: easy + +spec: + verify: + - llmJudge: + contains: "8" + + prompt: + inline: | + I need to know what 5 + 3 equals. Can you help me figure this out? +``` + +**What this tests:** +- **Natural language prompt**: No mention of tools - agent must discover the `add` tool +- **Verification**: Judge LLM checks that the result contains "8" +- **Tool discovery**: Can the agent find and use the right tool from a simple question? + +This tests whether the agent can: +1. **Discover** the `add` tool from its description +2. **Understand** when to use it based on the natural language prompt +3. **Call it correctly** with the right parameters (a=5, b=3) + +### The MCP Server Config (`evals/mcp-config.yaml`) + +```yaml +mcpServers: + demo-server: + type: http + url: http://localhost:8000/mcp + enableAllTools: true +``` + +**What this does:** +- Defines a server named **demo-server** +- Connects via **HTTP** to `http://localhost:8000/mcp` +- **Enables all tools** exposed by the server + +## Expected Output + +When tests pass, you'll see: + +``` +✅ add-test: PASSED + Tool calls: + - demo-server.add(a=5, b=3) → 8 + Verifications: + - Result contains "8" ✓ +``` + +The output also generates a JSON file (`demo-server-test-out.json`) with detailed results including: +- Complete agent conversation transcript +- All tool calls made +- Verification results +- Timing information diff --git a/getting-started/evals/eval.yaml b/getting-started/evals/eval.yaml new file mode 100644 index 0000000..a616360 --- /dev/null +++ b/getting-started/evals/eval.yaml @@ -0,0 +1,28 @@ +kind: Eval +metadata: + name: "demo-server-test" + +config: + # Use Claude Code as the AI agent + agent: + type: "builtin.claude-code" + + # MCP server configuration + mcpConfigFile: mcp-config.yaml + + # LLM judge configuration + llmJudge: + env: + baseUrlKey: JUDGE_BASE_URL + apiKeyKey: JUDGE_API_KEY + modelNameKey: JUDGE_MODEL_NAME + + # Test tasks + taskSets: + - path: tasks/add.yaml + assertions: + toolsUsed: + - server: demo-server + tool: add + minToolCalls: 1 + maxToolCalls: 5 diff --git a/getting-started/evals/gevals-demo-server-test-out.json b/getting-started/evals/gevals-demo-server-test-out.json new file mode 100644 index 0000000..cde4cb6 --- /dev/null +++ b/getting-started/evals/gevals-demo-server-test-out.json @@ -0,0 +1,128 @@ +[ + { + "taskName": "add-test", + "taskPath": "/home/matzew/go/src/github.com/genmcp/gevals-quickstart/gevals/tasks/add.yaml", + "taskPassed": true, + "taskOutput": "The answer is **8**. \n\n5 + 3 = 8\n", + "taskJudgeReason": "The MODEL_RESPONSE contains the correct answer '8', which matches the core information in the REFERENCE_ANSWER.", + "difficulty": "easy", + "assertionResults": { + "toolsUsed": { + "passed": true + }, + "minToolCalls": { + "passed": true + }, + "maxToolCalls": { + "passed": true + } + }, + "allAssertionsPassed": true, + "callHistory": { + "ToolCalls": [ + { + "serverName": "demo-server", + "timestamp": "2026-01-26T13:26:07.81718234+01:00", + "success": true, + "name": "add", + "result": { + "content": [ + { + "type": "text", + "text": "8" + } + ], + "structuredContent": { + "result": 8 + } + }, + "request": { + "Session": {}, + "Params": { + "_meta": { + "claudecode/toolUseId": "toolu_vrtx_01URpcMkdNU2y8628nLTpwwv" + }, + "name": "add", + "arguments": { + "a": 5, + "b": 3 + } + }, + "Extra": { + "TokenInfo": null, + "Header": { + "Accept": [ + "application/json, text/event-stream" + ], + "Accept-Encoding": [ + "gzip, deflate" + ], + "Accept-Language": [ + "*" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "167" + ], + "Content-Type": [ + "application/json" + ], + "Mcp-Protocol-Version": [ + "2025-11-25" + ], + "Mcp-Session-Id": [ + "3IAXKB32PFJWOPHX2CRBZUUOET" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "User-Agent": [ + "claude-code/2.1.19" + ] + } + } + } + } + ], + "ResourceReads": null, + "PromptGets": null + }, + "setupOutput": { + "Steps": [], + "Success": true, + "Error": "" + }, + "agentOutput": { + "Steps": [ + { + "type": "agent", + "success": true, + "message": "The answer is **8**. \n\n5 + 3 = 8\n", + "outputs": { + "output": "The answer is **8**. \n\n5 + 3 = 8\n" + } + } + ], + "Success": true, + "Error": "" + }, + "verifyOutput": { + "Steps": [ + { + "type": "llmJudge", + "success": true, + "message": "The MODEL_RESPONSE contains the correct answer '8', which matches the core information in the REFERENCE_ANSWER." + } + ], + "Success": true, + "Error": "" + }, + "cleanupOutput": { + "Steps": [], + "Success": true, + "Error": "" + } + } +] diff --git a/getting-started/evals/mcp-config.yaml b/getting-started/evals/mcp-config.yaml new file mode 100644 index 0000000..9b567de --- /dev/null +++ b/getting-started/evals/mcp-config.yaml @@ -0,0 +1,5 @@ +mcpServers: + demo-server: + type: http + url: http://localhost:8000/mcp + enableAllTools: true diff --git a/getting-started/evals/mcpchecker-demo-server-test-out.json b/getting-started/evals/mcpchecker-demo-server-test-out.json new file mode 100644 index 0000000..18bee85 --- /dev/null +++ b/getting-started/evals/mcpchecker-demo-server-test-out.json @@ -0,0 +1,128 @@ +[ + { + "taskName": "add-test", + "taskPath": "/home/matzew/go/src/github.com/mcpchecker/quickstarts/getting-started/evals/tasks/add.yaml", + "taskPassed": true, + "taskOutput": "5 + 3 equals **8**.\n", + "taskJudgeReason": "The MODEL_RESPONSE contains the core information from the REFERENCE_ANSWER. Both indicate that 5 + 3 equals 8, fulfilling the CONTAINS criterion with no information missing or contradicting.", + "difficulty": "easy", + "assertionResults": { + "toolsUsed": { + "passed": true + }, + "minToolCalls": { + "passed": true + }, + "maxToolCalls": { + "passed": true + } + }, + "allAssertionsPassed": true, + "callHistory": { + "ToolCalls": [ + { + "serverName": "demo-server", + "timestamp": "2026-01-26T17:04:37.263307387+01:00", + "success": true, + "name": "add", + "result": { + "content": [ + { + "type": "text", + "text": "8" + } + ], + "structuredContent": { + "result": 8 + } + }, + "request": { + "Session": {}, + "Params": { + "_meta": { + "claudecode/toolUseId": "toolu_vrtx_01JTCBhXqk6MLB2bjtDNZfsM" + }, + "name": "add", + "arguments": { + "a": 5, + "b": 3 + } + }, + "Extra": { + "TokenInfo": null, + "Header": { + "Accept": [ + "application/json, text/event-stream" + ], + "Accept-Encoding": [ + "gzip, deflate" + ], + "Accept-Language": [ + "*" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "167" + ], + "Content-Type": [ + "application/json" + ], + "Mcp-Protocol-Version": [ + "2025-11-25" + ], + "Mcp-Session-Id": [ + "RI7MWODIF2R4WUFHCGP2DZBS7X" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "User-Agent": [ + "claude-code/2.1.19" + ] + } + } + } + } + ], + "ResourceReads": null, + "PromptGets": null + }, + "setupOutput": { + "Steps": [], + "Success": true, + "Error": "" + }, + "agentOutput": { + "Steps": [ + { + "type": "agent", + "success": true, + "message": "5 + 3 equals **8**.\n", + "outputs": { + "output": "5 + 3 equals **8**.\n" + } + } + ], + "Success": true, + "Error": "" + }, + "verifyOutput": { + "Steps": [ + { + "type": "llmJudge", + "success": true, + "message": "The MODEL_RESPONSE contains the core information from the REFERENCE_ANSWER. Both indicate that 5 + 3 equals 8, fulfilling the CONTAINS criterion with no information missing or contradicting." + } + ], + "Success": true, + "Error": "" + }, + "cleanupOutput": { + "Steps": [], + "Success": true, + "Error": "" + } + } +] diff --git a/getting-started/evals/tasks/add.yaml b/getting-started/evals/tasks/add.yaml new file mode 100644 index 0000000..da76112 --- /dev/null +++ b/getting-started/evals/tasks/add.yaml @@ -0,0 +1,14 @@ +kind: Task +apiVersion: mcpchecker/v1alpha2 +metadata: + name: "add-test" + difficulty: easy + +spec: + verify: + - llmJudge: + contains: "8" + + prompt: + inline: | + I need to know what 5 + 3 equals. Can you help me figure this out? diff --git a/getting-started/server/server.py b/getting-started/server/server.py new file mode 100755 index 0000000..5758dda --- /dev/null +++ b/getting-started/server/server.py @@ -0,0 +1,25 @@ +#!/usr/bin/env -S uv run --quiet --script +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "mcp>=1.1.0", +# ] +# /// +"""FastMCP example with simple add tool.""" + +from mcp.server.fastmcp import FastMCP + +# Create an MCP server +mcp = FastMCP("Demo") + + +# Add an addition tool +@mcp.tool() +def add(a: int, b: int) -> int: + """Add two numbers""" + return a + b + + +# Run with streamable HTTP transport +if __name__ == "__main__": + mcp.run(transport="streamable-http")