Add transport protocol support to MCP server configurations

cli/src/mcp_tef_cli/client.py

@@ -11,6 +11,7 @@
 from mcp_tef_cli.models import (
     DifferentiationRecommendationResponse,
     HealthResponse,
+    MCPServerConfig,
     ModelSettingsCreate,
     OverlapMatrixResponse,
     PaginatedTestCaseResponse,
@@ -174,7 +175,7 @@ async def create_test_case(
         self,
         name: str,
         query: str,
-        available_mcp_servers: list[str],
+        available_mcp_servers: list[MCPServerConfig],
         expected_mcp_server_url: str | None = None,
         expected_tool_name: str | None = None,
         expected_parameters: dict | None = None,
@@ -184,7 +185,7 @@ async def create_test_case(
         Args:
             name: Descriptive name for the test case
             query: User query to evaluate
-            available_mcp_servers: List of MCP server URLs available for selection
+            available_mcp_servers: List of MCPServerConfig objects
             expected_mcp_server_url: Expected MCP server URL (null for negative tests)
             expected_tool_name: Expected tool name (null for negative tests)
             expected_parameters: Expected parameters as dict

cli/src/mcp_tef_cli/commands/test_case.py

@@ -17,7 +17,12 @@
     EXIT_INVALID_ARGUMENTS,
     EXIT_SUCCESS,
 )
-from mcp_tef_cli.models import PaginatedTestCaseResponse, TestCaseCreate, TestCaseResponse
+from mcp_tef_cli.models import (
+    MCPServerConfig,
+    PaginatedTestCaseResponse,
+    TestCaseCreate,
+    TestCaseResponse,
+)
 from mcp_tef_cli.output import print_error, print_success
 from mcp_tef_cli.utils import handle_api_errors, resolve_tef_url
 
@@ -168,7 +173,7 @@ def format_test_case_table(tc: TestCaseResponse, title: str = "Test Case") -> No
         console.print(f"Expected Params:  {json.dumps(tc.expected_parameters)}")
     console.print("Available Servers:")
     for server in tc.available_mcp_servers:
-        console.print(f"  - {server}")
+        console.print(f"  - {server.url} ({server.transport})")
     console.print(f"Created:          {tc.created_at}")
     console.print(f"Updated:          {tc.updated_at}")
 
@@ -275,6 +280,52 @@ def parse_set_option(values: tuple[str, ...]) -> dict[str, str]:
     return result
 
 
+def parse_server_spec(server_spec: str) -> MCPServerConfig:
+    """Parse a server specification into MCPServerConfig.
+
+    Supports two formats:
+    - URL only: "http://localhost:3000/sse" (uses default transport)
+    - URL with transport: "http://localhost:3000/sse:sse"
+
+    Args:
+        server_spec: Server specification string
+
+    Returns:
+        MCPServerConfig object
+
+    Raises:
+        click.BadParameter: If format is invalid or transport is not recognized
+    """
+    if ":" in server_spec and not server_spec.startswith(("http://", "https://")):
+        raise click.BadParameter(
+            f"Invalid server format: '{server_spec}'. "
+            "Expected 'URL' or 'URL:transport' (e.g., 'http://localhost:3000:sse')"
+        )
+
+    # Check if transport is specified (format: url:transport)
+    # Need to be careful not to split on the ':' in 'http://'
+    parts = server_spec.rsplit(":", 1)
+
+    if len(parts) == 2:
+        url, transport = parts
+        # Check if this is actually a transport spec or just part of the URL
+        # Valid transports are 'sse' or 'streamable-http'
+        if transport in ("sse", "streamable-http"):
+            # This is a transport specification
+            if not url.startswith(("http://", "https://")):
+                raise click.BadParameter(
+                    f"Invalid URL in server spec: '{url}'. Must start with http:// or https://"
+                )
+            return MCPServerConfig(url=url, transport=transport)
+
+    # No valid transport found, treat entire string as URL with default transport
+    if not server_spec.startswith(("http://", "https://")):
+        raise click.BadParameter(
+            f"Invalid server URL: '{server_spec}'. Must start with http:// or https://"
+        )
+    return MCPServerConfig(url=server_spec)
+
+
 @test_case.command(name="create")
 @click.option("--name", default=None, help="Descriptive name for the test case")
 @click.option("--query", default=None, help="User query to evaluate")
@@ -296,7 +347,12 @@ def parse_set_option(values: tuple[str, ...]) -> dict[str, str]:
 @click.option(
     "--servers",
     default=None,
-    help="Comma-separated MCP server URLs available for selection",
+    help=(
+        "Comma-separated MCP server specifications. "
+        "Format: 'URL' or 'URL:transport'. "
+        "Transport defaults to 'streamable-http' if not specified. "
+        "Examples: 'http://localhost:3000/sse:sse' or 'http://localhost:3001'"
+    ),
 )
 @click.option(
     "--from-file",
@@ -367,7 +423,10 @@ def create(
         "name": "Test case name",              // required
         "query": "User query to evaluate",     // required
         "available_mcp_servers": [             // required, non-empty
-          "${MCP_SERVER_URL}"                  // supports variable substitution
+          {
+            "url": "${MCP_SERVER_URL}",        // required, supports variable substitution
+            "transport": "streamable-http"     // optional, defaults to "streamable-http"
+          }
         ],
         "expected_mcp_server_url": "...",      // optional (null for negative tests)
         "expected_tool_name": "tool_name",     // optional (must pair with server)
@@ -383,20 +442,27 @@ def create(
     Examples:
 
       \b
-      # Create test case with expected tool
+      # Create test case with expected tool (SSE transport)
       mtef test-case create \\
         --name "Weather test" \\
         --query "What is the weather in San Francisco?" \\
         --expected-server "http://localhost:3000/sse" \\
         --expected-tool "get_weather" \\
-        --servers "http://localhost:3000/sse"
+        --servers "http://localhost:3000/sse:sse"
+
+      \b
+      # Create test case with multiple servers (mixed transports)
+      mtef test-case create \\
+        --name "Multi-server test" \\
+        --query "Get my calendar events" \\
+        --servers "http://localhost:3000:sse,http://localhost:3001"
 
       \b
       # Create negative test case (no tool should be selected)
       mtef test-case create \\
         --name "No tool needed" \\
         --query "What is 2 + 2?" \\
-        --servers "http://localhost:3000/sse"
+        --servers "http://localhost:3000/sse:sse"
 
       \b
       # Create from JSON file (single or multiple test cases)
@@ -447,8 +513,9 @@ def create(
             print_error("--servers is required (or use --from-file)")
             raise SystemExit(EXIT_INVALID_ARGUMENTS)
 
-        # Parse servers from comma-separated string
-        server_urls = [url.strip() for url in servers.split(",") if url.strip()]
+        # Parse servers from comma-separated string and convert to MCPServerConfig
+        server_specs = [spec.strip() for spec in servers.split(",") if spec.strip()]
+        server_configs = [parse_server_spec(spec) for spec in server_specs]
 
         # Parse expected parameters JSON
         try:
@@ -463,7 +530,7 @@ def create(
                 TestCaseCreate(
                     name=name,
                     query=query,
-                    available_mcp_servers=server_urls,
+                    available_mcp_servers=server_configs,
                     expected_mcp_server_url=expected_server,
                     expected_tool_name=expected_tool,
                     expected_parameters=expected_parameters,

cli/src/mcp_tef_cli/models.py

@@ -8,7 +8,7 @@
 from datetime import datetime
 from typing import Any
 
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, field_validator, model_validator
 
 __all__ = [
     "HealthResponse",
@@ -18,6 +18,7 @@
     "ToolQualityResult",
     "ToolQualityResponse",
     # Test case models
+    "MCPServerConfig",
     "TestCaseCreate",
     "TestCaseResponse",
     "PaginatedTestCaseResponse",
@@ -102,6 +103,22 @@ class ToolQualityResponse(BaseModel):
 # =============================================================================
 
 
+class MCPServerConfig(BaseModel):
+    """MCP server configuration with transport type."""
+
+    url: str = Field(
+        ...,
+        min_length=1,
+        pattern=r"^https?://",
+        description="Server URL (must be http or https)",
+    )
+    transport: str = Field(
+        default="streamable-http",
+        pattern=r"^(sse|streamable-http)$",
+        description="Transport type: 'sse' or 'streamable-http'",
+    )
+
+
 class ToolDefinition(BaseModel):
     """Definition of a tool available from an MCP server."""
 
@@ -123,10 +140,30 @@ class TestCaseCreate(BaseModel):
     expected_parameters: dict | None = Field(
         default=None, description="Expected parameters as JSON object"
     )
-    available_mcp_servers: list[str] = Field(
-        ..., description="MCP server URLs available for selection", min_length=1
+    available_mcp_servers: list[MCPServerConfig] = Field(
+        ..., description="MCP server configurations available for selection", min_length=1
     )
 
+    @field_validator("available_mcp_servers", mode="before")
+    @classmethod
+    def normalize_available_mcp_servers(cls, v: Any) -> list[Any]:
+        """Convert string URLs to MCPServerConfig objects for convenience."""
+        if not isinstance(v, list):
+            return v
+
+        normalized = []
+        for item in v:
+            if isinstance(item, str):
+                # Convert string URL to MCPServerConfig dict
+                normalized.append({"url": item, "transport": "streamable-http"})
+            elif isinstance(item, dict):
+                # Already a dict, pass through (will be validated as MCPServerConfig)
+                normalized.append(item)
+            else:
+                # Already a MCPServerConfig object or other type
+                normalized.append(item)
+        return normalized
+
     @model_validator(mode="after")
     def validate_expected_tool_fields(self) -> "TestCaseCreate":
         """Validate cross-field constraints for expected tool configuration."""
@@ -138,14 +175,13 @@ def validate_expected_tool_fields(self) -> "TestCaseCreate":
             )
 
         # expected_server must be in available_mcp_servers
-        if (
-            self.expected_mcp_server_url
-            and self.expected_mcp_server_url not in self.available_mcp_servers
-        ):
-            raise ValueError(
-                f"expected_mcp_server_url '{self.expected_mcp_server_url}' "
-                "must be in available_mcp_servers"
-            )
+        if self.expected_mcp_server_url:
+            available_urls = [server.url for server in self.available_mcp_servers]
+            if self.expected_mcp_server_url not in available_urls:
+                raise ValueError(
+                    f"expected_mcp_server_url '{self.expected_mcp_server_url}' "
+                    "must be in available_mcp_servers"
+                )
 
         # expected_parameters requires expected_tool_name
         if self.expected_parameters and not self.expected_tool_name:
@@ -163,13 +199,35 @@ class TestCaseResponse(BaseModel):
     expected_mcp_server_url: str | None = Field(default=None, description="Expected MCP server URL")
     expected_tool_name: str | None = Field(default=None, description="Expected tool name")
     expected_parameters: dict | None = Field(default=None, description="Expected parameters")
-    available_mcp_servers: list[str] = Field(..., description="Available MCP servers")
+    available_mcp_servers: list[MCPServerConfig] = Field(
+        ..., description="Available MCP server configurations"
+    )
     available_tools: dict[str, list[ToolDefinition]] | None = Field(
         default=None, description="Available tools by server URL"
     )
     created_at: datetime = Field(..., description="Creation timestamp")
     updated_at: datetime = Field(..., description="Last update timestamp")
 
+    @field_validator("available_mcp_servers", mode="before")
+    @classmethod
+    def normalize_available_mcp_servers(cls, v: Any) -> list[Any]:
+        """Convert string URLs to MCPServerConfig objects for convenience."""
+        if not isinstance(v, list):
+            return v
+
+        normalized = []
+        for item in v:
+            if isinstance(item, str):
+                # Convert string URL to MCPServerConfig dict
+                normalized.append({"url": item, "transport": "streamable-http"})
+            elif isinstance(item, dict):
+                # Already a dict, pass through (will be validated as MCPServerConfig)
+                normalized.append(item)
+            else:
+                # Already a MCPServerConfig object or other type
+                normalized.append(item)
+        return normalized
+
 
 class PaginatedTestCaseResponse(BaseModel):
     """Paginated test case response."""

cli/tests/unit/test_test_case_parsing.py

@@ -378,7 +378,8 @@ def test_load_with_env_var_substitution(self, tmp_path):
         )
 
         assert len(test_cases) == 1
-        assert test_cases[0].available_mcp_servers == ["http://localhost:3000/sse"]
+        assert len(test_cases[0].available_mcp_servers) == 1
+        assert test_cases[0].available_mcp_servers[0].url == "http://localhost:3000/sse"
         assert test_cases[0].expected_mcp_server_url == "http://localhost:3000/sse"
 
     def test_load_multiple_with_env_vars(self, tmp_path):
@@ -404,8 +405,10 @@ def test_load_multiple_with_env_vars(self, tmp_path):
         )
 
         assert len(test_cases) == 2
-        assert test_cases[0].available_mcp_servers == ["http://server:8000"]
-        assert test_cases[1].available_mcp_servers == ["http://server:8000"]
+        assert len(test_cases[0].available_mcp_servers) == 1
+        assert test_cases[0].available_mcp_servers[0].url == "http://server:8000"
+        assert len(test_cases[1].available_mcp_servers) == 1
+        assert test_cases[1].available_mcp_servers[0].url == "http://server:8000"
 
     def test_load_with_unresolved_var_fails_validation(self, tmp_path):
         """Unresolved variable that results in invalid URL fails validation."""
@@ -420,8 +423,9 @@ def test_load_with_unresolved_var_fails_validation(self, tmp_path):
 
         # The unresolved ${UNDEFINED} is kept as-is, which is a valid string
         # but may not be a valid URL depending on use case
-        test_cases = load_test_cases_from_file(str(file_path), env_vars={})
-        assert test_cases[0].available_mcp_servers == ["${UNDEFINED}"]
+        # This should fail validation because ${UNDEFINED} doesn't match the URL pattern
+        with pytest.raises(BadParameter):
+            load_test_cases_from_file(str(file_path), env_vars={})
 
     def test_load_from_os_env(self, tmp_path, monkeypatch):
         """Variables are resolved from OS environment."""
@@ -437,4 +441,5 @@ def test_load_from_os_env(self, tmp_path, monkeypatch):
 
         test_cases = load_test_cases_from_file(str(file_path))
 
-        assert test_cases[0].available_mcp_servers == ["http://from-os:9000"]
+        assert len(test_cases[0].available_mcp_servers) == 1
+        assert test_cases[0].available_mcp_servers[0].url == "http://from-os:9000"