{
  "name": "Telemio observability MCP investigation benchmark protocol",
  "version": "0.1.0-draft",
  "status": "planned-not-executed",
  "published_at": "2026-11-06",
  "results_published": false,
  "result_policy": "No performance rankings or quantitative vendor comparisons until all scenarios, runs, labels, budgets and logs are available.",
  "scenario_count_planned": 25,
  "scenario_families": [
    {
      "name": "request-failures",
      "cases_planned": 5,
      "examples": [
        "HTTP 5xx regression",
        "upstream dependency failure",
        "incorrect route configuration"
      ]
    },
    {
      "name": "latency-regressions",
      "cases_planned": 5,
      "examples": [
        "slow database query",
        "service timeout",
        "resource saturation"
      ]
    },
    {
      "name": "background-workloads",
      "cases_planned": 5,
      "examples": [
        "queue retries",
        "dropped jobs",
        "consumer lag"
      ]
    },
    {
      "name": "release-related-regressions",
      "cases_planned": 5,
      "examples": [
        "deployment-specific error increase",
        "release-specific latency regression",
        "changed feature behavior"
      ]
    },
    {
      "name": "ai-agent-runtime-issues",
      "cases_planned": 5,
      "examples": [
        "tool failure",
        "excessive retries",
        "model call latency spike"
      ]
    }
  ],
  "fairness_controls": [
    "Same model and exact model version per paired comparison",
    "Same injected faults and time-window access",
    "Comparable permissions and signals across setups",
    "Fixed time, token, tool-call, and cost budgets",
    "Identical high-level incident request with adapter-specific tool connection only",
    "Blinded evaluation against recorded ground truth",
    "Timeouts and failed attempts included",
    "Publish all attempted runs and costs"
  ],
  "metrics": [
    {
      "id": "root_cause_accuracy",
      "unit": "fraction",
      "definition": "Correct fault label plus supporting telemetry evidence divided by all started attempts"
    },
    {
      "id": "time_to_diagnosis_p50",
      "unit": "seconds",
      "definition": "Median end-to-end wall time including failed and timeout attempts with explicit censoring"
    },
    {
      "id": "time_to_diagnosis_p95",
      "unit": "seconds",
      "definition": "p95 wall time including timeouts with explicit censoring"
    },
    {
      "id": "tool_calls_per_attempt",
      "unit": "calls",
      "definition": "Tool requests per started attempt; report median and distribution"
    },
    {
      "id": "tokens_per_attempt",
      "unit": "tokens",
      "definition": "Input and output tokens per started attempt; report model-specific accounting"
    },
    {
      "id": "completion_rate",
      "unit": "fraction",
      "definition": "Completed investigations without transport errors or timeouts divided by started attempts"
    },
    {
      "id": "cross_signal_evidence",
      "unit": "fraction",
      "definition": "Attempts correctly correlating required traces, logs, metrics, and/or runtimes per scenario label"
    },
    {
      "id": "cost_per_correct_diagnosis",
      "unit": "USD",
      "definition": "Total model and service cost for all attempts divided by correctly diagnosed attempts"
    }
  ],
  "publication_gate": [
    "Scenarios and expected diagnoses reproducible",
    "Harness configurations versioned",
    "Raw runs and tool call traces retained",
    "Independent scorer and review recorded",
    "Competitor products and features checked on test date"
  ],
  "reference_urls": [
    "https://telemio.sh/agent/mcp.md",
    "https://telemio.sh/.well-known/mcp/server-card.json"
  ]
}