{
  "schemaVersion": 1,
  "identifier": "https://driven.ai/drivenbench#dataset",
  "name": "DrivenBench",
  "description": "DrivenBench compares 11 AI models for investment agents on real-world workflows and tasks across capability, benchmark cost upper bound, and directional latency, with separate regression checks.",
  "url": "https://driven.ai/drivenbench",
  "version": "1.0",
  "datePublished": "2026-08-25",
  "dateModified": "2026-08-25",
  "publisher": {
    "name": "Driven",
    "url": "https://driven.ai"
  },
  "citation": "https://driven.ai/whats-new/blog/most-expensive-ai-model-isnt-always-the-best",
  "machineReadableUrl": "https://driven.ai/drivenbench/data.json",
  "aiSummaryUrl": "https://driven.ai/drivenbench/llms.txt",
  "benchmarkDomain": "AI investment agents",
  "scoringVersion": "v4",
  "reportDate": "2026-08-15",
  "questionBank": "v4.10 + l25 v4.11",
  "judgeModel": "anthropic:claude-opus-4-8",
  "methodology": {
    "modelCount": 11,
    "capabilityEvaluationRows": 55,
    "capabilityRunsPerRow": 3,
    "capabilityRunsPerModel": 165,
    "regressionGuardrailRows": 19,
    "regressionRunsPerRow": 1,
    "totalRuns": 2024,
    "rankingMetric": "Capability score only",
    "costMetric": "Total benchmark cost upper bound in USD",
    "latencyMetric": "Directional median and p90 completion time; models ran in different waves and provider load varied"
  },
  "models": [
    {
      "id": "anthropic:claude-sonnet-5",
      "evaluatedModelName": "Claude Sonnet 5",
      "displayName": "Claude Sonnet 5",
      "version": null,
      "provider": "Anthropic",
      "rank": 1,
      "capabilityScorePercent": 93.9,
      "capabilityPasses": 155,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 57.69,
      "latencyMedianSeconds": 30,
      "latencyP90Seconds": 70,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": true
    },
    {
      "id": "fireworks:accounts/fireworks/models/kimi-k3",
      "evaluatedModelName": "Kimi K3",
      "displayName": "Kimi K3",
      "version": null,
      "provider": "Moonshot AI",
      "rank": 1,
      "capabilityScorePercent": 93.9,
      "capabilityPasses": 155,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 59.42,
      "latencyMedianSeconds": 59,
      "latencyP90Seconds": 169,
      "regressionGuardrailPasses": 18,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": false
    },
    {
      "id": "anthropic:claude-opus-5",
      "evaluatedModelName": "Claude Opus 5",
      "displayName": "Claude Opus 5",
      "version": null,
      "provider": "Anthropic",
      "rank": 3,
      "capabilityScorePercent": 90.9,
      "capabilityPasses": 150,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 160.37,
      "latencyMedianSeconds": 40,
      "latencyP90Seconds": 90,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": false
    },
    {
      "id": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
      "evaluatedModelName": "V4 Pro 0813",
      "displayName": "DeepSeek V4 Pro",
      "version": "0813",
      "provider": "DeepSeek",
      "rank": 4,
      "capabilityScorePercent": 89.7,
      "capabilityPasses": 148,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 30.64,
      "latencyMedianSeconds": 24,
      "latencyP90Seconds": 61,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": true
    },
    {
      "id": "xai:grok-4.6",
      "evaluatedModelName": "Grok 4.6",
      "displayName": "Grok 4.6",
      "version": null,
      "provider": "xAI",
      "rank": 5,
      "capabilityScorePercent": 89.1,
      "capabilityPasses": 147,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 73.65,
      "latencyMedianSeconds": 75,
      "latencyP90Seconds": 158,
      "regressionGuardrailPasses": 18,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": false
    },
    {
      "id": "fireworks:accounts/fireworks/models/glm-5p2",
      "evaluatedModelName": "GLM 5.2",
      "displayName": "GLM 5.2",
      "version": null,
      "provider": "Zhipu AI",
      "rank": 6,
      "capabilityScorePercent": 87.3,
      "capabilityPasses": 144,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 25.64,
      "latencyMedianSeconds": 23,
      "latencyP90Seconds": 62,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": true
    },
    {
      "id": "openai:gpt-5.6-sol",
      "evaluatedModelName": "GPT-5.6 Sol",
      "displayName": "GPT-5.6 Sol",
      "version": null,
      "provider": "OpenAI",
      "rank": 7,
      "capabilityScorePercent": 85.5,
      "capabilityPasses": 141,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 92.72,
      "latencyMedianSeconds": 27,
      "latencyP90Seconds": 61,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": false
    },
    {
      "id": "vertex:gemini-3.7-flash",
      "evaluatedModelName": "Gemini 3.7 Flash",
      "displayName": "Gemini 3.7 Flash",
      "version": null,
      "provider": "Google",
      "rank": 8,
      "capabilityScorePercent": 81.8,
      "capabilityPasses": 135,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 13.74,
      "latencyMedianSeconds": 23,
      "latencyP90Seconds": 44,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": true
    },
    {
      "id": "openai:gpt-5.6-luna",
      "evaluatedModelName": "GPT-5.6 Luna",
      "displayName": "GPT-5.6 Luna",
      "version": null,
      "provider": "OpenAI",
      "rank": 9,
      "capabilityScorePercent": 79.4,
      "capabilityPasses": 131,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 3.63,
      "latencyMedianSeconds": 19,
      "latencyP90Seconds": 39,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": true
    },
    {
      "id": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
      "evaluatedModelName": "V4 Flash 0731",
      "displayName": "DeepSeek V4 Flash",
      "version": "0731",
      "provider": "DeepSeek",
      "rank": 10,
      "capabilityScorePercent": 77.6,
      "capabilityPasses": 128,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 2.97,
      "latencyMedianSeconds": 20,
      "latencyP90Seconds": 46,
      "regressionGuardrailPasses": 19,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": true
    },
    {
      "id": "openai:gpt-5.6-terra",
      "evaluatedModelName": "GPT-5.6 Terra",
      "displayName": "GPT-5.6 Terra",
      "version": null,
      "provider": "OpenAI",
      "rank": 11,
      "capabilityScorePercent": 76.4,
      "capabilityPasses": 126,
      "capabilityRuns": 165,
      "totalBenchmarkCostUpperBoundUsd": 33.54,
      "latencyMedianSeconds": 16,
      "latencyP90Seconds": 32,
      "regressionGuardrailPasses": 18,
      "regressionGuardrailRuns": 19,
      "onObservedParetoFrontier": false
    }
  ],
  "dimensions": [
    {
      "id": "s01",
      "name": "Numerical Reasoning",
      "description": "Reconciles supplied figures across holdings, CPI contributions, fees, mNAV, units, and portfolio weights.",
      "capabilityEvaluationRows": 2,
      "regressionGuardrailRows": 6,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 5,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 5,
          "capabilityRuns": 6,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 6,
          "capabilityRuns": 6,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 6,
          "regressionGuardrailRuns": 6
        }
      ]
    },
    {
      "id": "s02",
      "name": "Bash Execution",
      "description": "Produces real files, spreadsheets, and archives; recovers after command failures; and applies exact edits.",
      "capabilityEvaluationRows": 6,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 16,
          "capabilityRuns": 18,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 16,
          "capabilityRuns": 18,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 16,
          "capabilityRuns": 18,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s03",
      "name": "Financial Data Tools",
      "description": "Routes quotes, financial statements, transcripts, and market charts to the right tools without inventing data.",
      "capabilityEvaluationRows": 6,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 12,
          "capabilityRuns": 18,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 14,
          "capabilityRuns": 18,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 13,
          "capabilityRuns": 18,
          "capabilityScorePercent": 72.22222222222221,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 13,
          "capabilityRuns": 18,
          "capabilityScorePercent": 72.22222222222221,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 16,
          "capabilityRuns": 18,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s04",
      "name": "External Research",
      "description": "Searches proactively, changes route after failures, and verifies targeted claims against authoritative sources.",
      "capabilityEvaluationRows": 4,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 6,
          "capabilityRuns": 12,
          "capabilityScorePercent": 50,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s05",
      "name": "Programmatic Analysis",
      "description": "Writes financial calculation code, preserves behavior through revisions, respects sandbox boundaries, and computes risk metrics.",
      "capabilityEvaluationRows": 4,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 11,
          "capabilityRuns": 12,
          "capabilityScorePercent": 91.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 11,
          "capabilityRuns": 12,
          "capabilityScorePercent": 91.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s06",
      "name": "Data Integrity",
      "description": "Delivers honestly when sources are missing or partial, continuing useful work without fabricating evidence.",
      "capabilityEvaluationRows": 5,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 13,
          "capabilityRuns": 15,
          "capabilityScorePercent": 86.66666666666667,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 15,
          "capabilityRuns": 15,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 15,
          "capabilityRuns": 15,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 13,
          "capabilityRuns": 15,
          "capabilityScorePercent": 86.66666666666667,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 15,
          "capabilityRuns": 15,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 14,
          "capabilityRuns": 15,
          "capabilityScorePercent": 93.33333333333333,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 11,
          "capabilityRuns": 15,
          "capabilityScorePercent": 73.33333333333333,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 13,
          "capabilityRuns": 15,
          "capabilityScorePercent": 86.66666666666667,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 11,
          "capabilityRuns": 15,
          "capabilityScorePercent": 73.33333333333333,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 10,
          "capabilityRuns": 15,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 11,
          "capabilityRuns": 15,
          "capabilityScorePercent": 73.33333333333333,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s08",
      "name": "Delegation & Integration",
      "description": "Constrains subagents, attributes failures, verifies claims, audits coverage, and synthesizes without distortion.",
      "capabilityEvaluationRows": 6,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 16,
          "capabilityRuns": 18,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 17,
          "capabilityRuns": 18,
          "capabilityScorePercent": 94.44444444444444,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 18,
          "capabilityRuns": 18,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s09",
      "name": "Citations & Evidence",
      "description": "Preserves qualifiers, audits citations, and binds conclusions to the evidence that supports them.",
      "capabilityEvaluationRows": 1,
      "regressionGuardrailRows": 2,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 3,
          "capabilityRuns": 3,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        }
      ]
    },
    {
      "id": "s10",
      "name": "Files & Large Documents",
      "description": "Finds relevant material in long documents, filters bulk output, and reads oversized tables in complete chunks.",
      "capabilityEvaluationRows": 3,
      "regressionGuardrailRows": 1,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 8,
          "capabilityRuns": 9,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 7,
          "capabilityRuns": 9,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 7,
          "capabilityRuns": 9,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 7,
          "capabilityRuns": 9,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 7,
          "capabilityRuns": 9,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 7,
          "capabilityRuns": 9,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 4,
          "capabilityRuns": 9,
          "capabilityScorePercent": 44.44444444444444,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 4,
          "capabilityRuns": 9,
          "capabilityScorePercent": 44.44444444444444,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        }
      ]
    },
    {
      "id": "s11",
      "name": "Coverage Reporting",
      "description": "Reports empty directories, partial file coverage, and item counts truthfully. This section is regression-only.",
      "capabilityEvaluationRows": 0,
      "regressionGuardrailRows": 3,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 0,
          "capabilityRuns": 0,
          "capabilityScorePercent": null,
          "regressionGuardrailPasses": 3,
          "regressionGuardrailRuns": 3
        }
      ]
    },
    {
      "id": "s12",
      "name": "Scheduled Tasks",
      "description": "Handles trigger semantics, clean outputs, pause-versus-delete behavior, and the quality of scheduled reports.",
      "capabilityEvaluationRows": 3,
      "regressionGuardrailRows": 2,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 6,
          "capabilityRuns": 9,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 2,
          "capabilityRuns": 9,
          "capabilityScorePercent": 22.22222222222222,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 8,
          "capabilityRuns": 9,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 8,
          "capabilityRuns": 9,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 8,
          "capabilityRuns": 9,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 6,
          "capabilityRuns": 9,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 5,
          "capabilityRuns": 9,
          "capabilityScorePercent": 55.55555555555556,
          "regressionGuardrailPasses": 2,
          "regressionGuardrailRuns": 2
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 6,
          "capabilityRuns": 9,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 2
        }
      ]
    },
    {
      "id": "s13",
      "name": "Trading & Accounts",
      "description": "Binds fills to the right orders, follows market rules, and clarifies ambiguous trade instructions before acting.",
      "capabilityEvaluationRows": 4,
      "regressionGuardrailRows": 1,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 11,
          "capabilityRuns": 12,
          "capabilityScorePercent": 91.66666666666666,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 5,
          "capabilityRuns": 12,
          "capabilityScorePercent": 41.66666666666667,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 3,
          "capabilityRuns": 12,
          "capabilityScorePercent": 25,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 2,
          "capabilityRuns": 12,
          "capabilityScorePercent": 16.666666666666664,
          "regressionGuardrailPasses": 1,
          "regressionGuardrailRuns": 1
        }
      ]
    },
    {
      "id": "s14",
      "name": "Memory & Sessions",
      "description": "Retains rules, strategy context, and persistent preferences after compaction or truncation.",
      "capabilityEvaluationRows": 3,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 9,
          "capabilityRuns": 9,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 8,
          "capabilityRuns": 9,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 6,
          "capabilityRuns": 9,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 7,
          "capabilityRuns": 9,
          "capabilityScorePercent": 77.77777777777779,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 6,
          "capabilityRuns": 9,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 8,
          "capabilityRuns": 9,
          "capabilityScorePercent": 88.88888888888889,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 6,
          "capabilityRuns": 9,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s15",
      "name": "Skills & Routing",
      "description": "Triggers the right skill, degrades honestly when a source is unavailable, locates skills, and persists real skill files.",
      "capabilityEvaluationRows": 4,
      "regressionGuardrailRows": 0,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 6,
          "capabilityRuns": 12,
          "capabilityScorePercent": 50,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 8,
          "capabilityRuns": 12,
          "capabilityScorePercent": 66.66666666666666,
          "regressionGuardrailPasses": 0,
          "regressionGuardrailRuns": 0
        }
      ]
    },
    {
      "id": "s00",
      "name": "Supplemental Coverage",
      "description": "Covers time-sensitive news, time windows, schedule confirmation, future-information isolation, reporting, and subagent failure attribution.",
      "capabilityEvaluationRows": 4,
      "regressionGuardrailRows": 4,
      "modelResults": [
        {
          "modelId": "anthropic:claude-sonnet-5",
          "modelName": "Claude Sonnet 5",
          "capabilityPasses": 11,
          "capabilityRuns": 12,
          "capabilityScorePercent": 91.66666666666666,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
          "modelName": "Kimi K3",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "anthropic:claude-opus-5",
          "modelName": "Claude Opus 5",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Pro-0813",
          "modelName": "DeepSeek V4 Pro",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "xai:grok-4.6",
          "modelName": "Grok 4.6",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "fireworks:accounts/fireworks/models/glm-5p2",
          "modelName": "GLM 5.2",
          "capabilityPasses": 9,
          "capabilityRuns": 12,
          "capabilityScorePercent": 75,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "openai:gpt-5.6-sol",
          "modelName": "GPT-5.6 Sol",
          "capabilityPasses": 12,
          "capabilityRuns": 12,
          "capabilityScorePercent": 100,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "vertex:gemini-3.7-flash",
          "modelName": "Gemini 3.7 Flash",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "openai:gpt-5.6-luna",
          "modelName": "GPT-5.6 Luna",
          "capabilityPasses": 11,
          "capabilityRuns": 12,
          "capabilityScorePercent": 91.66666666666666,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731",
          "modelName": "DeepSeek V4 Flash",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        },
        {
          "modelId": "openai:gpt-5.6-terra",
          "modelName": "GPT-5.6 Terra",
          "capabilityPasses": 10,
          "capabilityRuns": 12,
          "capabilityScorePercent": 83.33333333333334,
          "regressionGuardrailPasses": 4,
          "regressionGuardrailRuns": 4
        }
      ]
    }
  ],
  "disclosedRegressionEvents": [
    {
      "modelId": "xai:grok-4.6",
      "rowId": "l08",
      "status": "Intermittent · disclosed and waived",
      "description": "The first run lost the final citation; 1 of 3 diagnostic reruns reproduced the issue.",
      "modelName": "Grok 4.6"
    },
    {
      "modelId": "fireworks:accounts/fireworks/models/kimi-k3",
      "rowId": "l12",
      "status": "Intermittent · disclosed and waived",
      "description": "The first run missed the weight baseline. Diagnostic reruns did not reproduce it.",
      "modelName": "Kimi K3"
    },
    {
      "modelId": "openai:gpt-5.6-terra",
      "rowId": "l25",
      "status": "Stable failure · disclosed and waived",
      "description": "Pausing a task cleared its name; the data-destructive behavior reproduced in three diagnostic runs.",
      "modelName": "GPT-5.6 Terra"
    }
  ],
  "disclaimer": "Results describe this evaluation and may vary with model versions, run timing, and provider conditions. DrivenBench is for comparative research and does not constitute investment advice."
}