{
  "experimentId": "script-writing-openrouter-2026-08-22",
  "title": "Which OpenRouter model writes the best one-minute screenplay scene?",
  "date": "2026-08-22",
  "status": "reference-extension-complete",
  "amendments": [
    {
      "id": "A1-reasoning-exhaustion",
      "triggeredAtSpendUsd": 0.04543504,
      "reason": "Two DeepSeek calls and one Nemotron call consumed the full 1600-token allowance as hidden reasoning and returned no screenplay. No screenplay text or comparative score was inspected before the run was stopped.",
      "change": "Disable optional reasoning for DeepSeek V4 Pro 0813 and Nemotron 3 Ultra. Preserve the original empty attempts and their cost as superseded harness failures, then regenerate those cells once.",
      "scope": ["deepseek-v4-pro-0813", "nemotron-3-ultra"]
    },
    {
      "id": "A2-judge-schema-compatibility",
      "triggeredAtSpendUsd": 0.2947,
      "failedJudgeSpendUsd": 0.02665832,
      "reason": "GLM 5.3 ignored the required absolute-score field contract on all seven attempted JSON-object responses. Grok and Mistral strict-schema scores succeeded. The blind map was not decoded.",
      "change": "Replace GLM 5.3 with MiniMax M3, which supports strict JSON Schema. Preserve GLM failures and cost as superseded evaluator attempts.",
      "scope": ["glm-5.3", "minimax-m3"]
    },
    {
      "id": "A3-minimax-provider-structured-output",
      "triggeredAtSpendUsd": 1.9595,
      "failedJudgeSpendUsd": 0.0056604,
      "reason": "Five of nine MiniMax extension cells routed through Parasail returned HTTP 200 with billed completion tokens but an empty structured response on both identical attempts. Four MiniMax cells routed through CoreWeave or Morph succeeded, as did all contemporaneous Grok and Mistral cells. No screenplay identity or quality was inspected.",
      "change": "Exclude the Parasail provider for MiniMax scoring while preserving the five failed cells and their cost, then regenerate only those missing evaluator cells. The judge model, schema, prompt, blind codes, and successful scores remain unchanged.",
      "scope": ["minimax-m3", "Parasail"]
    }
  ],
  "objective": "Compare eight OpenRouter models on original, filmable one-minute screenplay scenes across neutral and reference-directed prompts.",
  "randomizationSeed": 20260822,
  "commonSystemPrompt": "Write one original, self-contained screenplay scene that plays in approximately 60 seconds. Use one location and exactly two speaking characters. Use standard screenplay elements: a slugline, concise present-tense action, character cues, and dialogue. Target 150-220 total words. The required discovery or reversal must happen on screen, and the scene must end on a consequential dramatic or comic turn. Return only the screenplay; no title, synopsis, analysis, or notes.",
  "referenceGuard": "Use only the high-level storytelling traits in this reference card. Write original characters, dialogue, situations, and phrasing. Do not reproduce recognizable characters, lines, distinctive phrases, scenes, or plot details from the named work or creator.",
  "request": {
    "maxTokens": 1600,
    "temperature": null,
    "seed": null,
    "tools": false,
    "provider": {
      "allowFallbacks": true,
      "requireParameters": true,
      "dataCollection": "deny"
    },
    "usageInclude": true
  },
  "phases": {
    "pilot": {
      "conditionIds": [
        "dinner_control",
        "dinner_pulp_fiction"
      ],
      "repeats": [1, 2],
      "generationRequests": 32,
      "absoluteJudgeCalls": 96,
      "pairwiseMatchups": 112,
      "pairwiseJudgeCalls": 336,
      "estimatedCostUsd": 2.3758,
      "amendmentSpendUsd": 0.04377596,
      "estimatedCostWithContingencyUsd": 3.0246,
      "budgetCapUsd": 3.03
    },
    "referenceExtension": {
      "description": "Add three repeats for the remaining Inception, Step Brothers, and Eyes Wide Shut reference conditions. Preserve every pilot output and score.",
      "additive": true,
      "conditionIds": [
        "dinner_inception",
        "dinner_step_brothers",
        "dinner_eyes_wide_shut"
      ],
      "repeats": [1, 2, 3],
      "generationRequests": 72,
      "absoluteJudgeCalls": 216,
      "pairwiseMatchups": 252,
      "pairwiseJudgeCalls": 756,
      "estimatedCostUsd": 5.3456,
      "estimatedCostWithContingencyUsd": 6.6821,
      "startingRecordedSpendUsd": 1.479595,
      "actualAdditionalSpendUsd": 3.411062,
      "actualCumulativeSpendUsd": 4.890657,
      "budgetCapUsd": 8.17
    },
    "extension": {
      "description": "Remaining work after the reference extension: add repeat 3 to the control and Pulp Fiction conditions, plus repeats 1-3 for the museum and siblings conditions. Preserve every successful output and score.",
      "generationRequests": 64,
      "absoluteJudgeCalls": 192,
      "pairwiseMatchups": 224,
      "pairwiseJudgeCalls": 672,
      "estimatedAdditionalCostUsd": 4.7517,
      "additionalBudgetCapUsd": 5.94,
      "projectedCumulativeCapUsd": 10.84
    },
    "full": {
      "conditionIds": [
        "dinner_control",
        "museum_suspense",
        "siblings_reversal",
        "dinner_pulp_fiction",
        "dinner_inception",
        "dinner_step_brothers",
        "dinner_eyes_wide_shut"
      ],
      "repeats": [1, 2, 3],
      "generationRequests": 168,
      "absoluteJudgeCalls": 504,
      "pairwiseMatchups": 588,
      "pairwiseJudgeCalls": 1764,
      "estimatedCostUsd": 12.4731,
      "estimatedCostWithContingencyUsd": 15.6462,
      "budgetCapUsd": 15.65
    }
  },
  "conditions": [
    {
      "id": "dinner_control",
      "label": "Dinner betrayal - no reference",
      "premise": "Two adults, a man and a woman, are midway through dinner. During the scene, he discovers that she lied when she said her father had cancer.",
      "direction": null,
      "isReferenceCondition": false
    },
    {
      "id": "museum_suspense",
      "label": "Locked-museum visual suspense",
      "premise": "A night security guard finds an apparently injured stranger inside a locked museum. During their conversation, each reveals knowledge the other should not possess.",
      "direction": null,
      "isReferenceCondition": false
    },
    {
      "id": "siblings_reversal",
      "label": "Childhood-home emotional reversal",
      "premise": "Two adult siblings are packing their childhood home. One discovers that the sibling insisting they keep it has already sold it.",
      "direction": null,
      "isReferenceCondition": false
    },
    {
      "id": "dinner_pulp_fiction",
      "label": "Dinner betrayal - Pulp Fiction / Quentin Tarantino",
      "premise": "Two adults, a man and a woman, are midway through dinner. During the scene, he discovers that she lied when she said her father had cancer.",
      "direction": "Treat Pulp Fiction and Quentin Tarantino as the tonal reference: mundane conversation carrying menace, sharply specific dialogue, shifting leverage, and a sudden moral or dramatic reversal.",
      "isReferenceCondition": true
    },
    {
      "id": "dinner_inception",
      "label": "Dinner betrayal - Inception / Christopher Nolan",
      "premise": "Two adults, a man and a woman, are midway through dinner. During the scene, he discovers that she lied when she said her father had cancer.",
      "direction": "Treat Inception and Christopher Nolan as the structural reference: layered memory or perception, precise cause and effect, emotional stakes, and a reveal that recontextualizes the conversation without leaving the one-location scene.",
      "isReferenceCondition": true
    },
    {
      "id": "dinner_step_brothers",
      "label": "Dinner betrayal - Step Brothers / Will Ferrell",
      "premise": "Two adults, a man and a woman, are midway through dinner. During the scene, he discovers that she lied when she said her father had cancer.",
      "direction": "Treat Step Brothers and Will Ferrell ensemble comedy as the tonal reference: committed absurdity, childish adult logic, escalating social discomfort, and sincere commitment to a ridiculous position. The comedy must target the deception and behavior, not cancer or illness.",
      "isReferenceCondition": true
    },
    {
      "id": "dinner_eyes_wide_shut",
      "label": "Dinner betrayal - Eyes Wide Shut / Stanley Kubrick",
      "premise": "Two adults, a man and a woman, are midway through dinner. During the scene, he discovers that she lied when she said her father had cancer.",
      "direction": "Treat Eyes Wide Shut and Stanley Kubrick as the tonal reference: clinical restraint, silence, ritualized behavior, emotional distance, formal precision, and mounting psychological unease.",
      "isReferenceCondition": true
    }
  ],
  "contestants": [
    {
      "id": "claude-opus-5",
      "label": "Claude Opus 5",
      "route": "anthropic/claude-opus-5",
      "pricingPerMillionUsd": {"input": 5, "output": 25},
      "reasoning": {"effort": "low", "exclude": true}
    },
    {
      "id": "gpt-5.6-sol",
      "label": "GPT-5.6 Sol",
      "route": "openai/gpt-5.6-sol",
      "pricingPerMillionUsd": {"input": 2, "output": 10},
      "reasoning": {"effort": "low", "exclude": true}
    },
    {
      "id": "deepseek-v4-pro-0813",
      "label": "DeepSeek V4 Pro 0813",
      "route": "deepseek/deepseek-v4-pro-0813",
      "pricingPerMillionUsd": {"input": 1.122, "output": 3.366},
      "reasoning": {"enabled": false, "exclude": true}
    },
    {
      "id": "kimi-k3",
      "label": "Kimi K3",
      "route": "moonshotai/kimi-k3",
      "pricingPerMillionUsd": {"input": 3, "output": 15},
      "reasoning": {"effort": "low", "exclude": true}
    },
    {
      "id": "nemotron-3-ultra",
      "label": "Nemotron 3 Ultra",
      "route": "nvidia/nemotron-3-ultra-550b-a55b",
      "pricingPerMillionUsd": {"input": 0.6, "output": 3.6},
      "reasoning": {"enabled": false, "exclude": true}
    },
    {
      "id": "inkling",
      "label": "Inkling",
      "route": "thinkingmachines/inkling",
      "pricingPerMillionUsd": {"input": 1, "output": 4.05},
      "reasoning": {"effort": "low", "exclude": true}
    },
    {
      "id": "gemini-3.7-flash",
      "label": "Gemini 3.7 Flash",
      "route": "google/gemini-3.7-flash",
      "pricingPerMillionUsd": {"input": 0.375, "output": 1.875},
      "reasoning": {"effort": "low", "exclude": true}
    },
    {
      "id": "qwen3.8-max",
      "label": "Qwen3.8 Max",
      "route": "qwen/qwen3.8-max",
      "pricingPerMillionUsd": {"input": 2, "output": 6},
      "reasoning": {"effort": "low", "exclude": true}
    }
  ],
  "judges": [
    {
      "id": "grok-4.6",
      "label": "Grok 4.6",
      "route": "x-ai/grok-4.6",
      "pricingPerMillionUsd": {"input": 2, "output": 6},
      "structuredOutputMode": "json_schema",
      "reasoning": {"effort": "low", "exclude": true}
    },
    {
      "id": "minimax-m3",
      "label": "MiniMax M3",
      "route": "minimax/minimax-m3",
      "pricingPerMillionUsd": {"input": 0.3, "output": 1.2},
      "structuredOutputMode": "json_schema",
      "providerIgnore": ["Parasail"],
      "reasoning": {"enabled": false, "exclude": true}
    },
    {
      "id": "mistral-medium-3.5",
      "label": "Mistral Medium 3.5",
      "route": "mistralai/mistral-medium-3-5",
      "pricingPerMillionUsd": {"input": 1.5, "output": 7.5},
      "structuredOutputMode": "json_schema",
      "reasoning": {"effort": "none", "exclude": true}
    }
  ],
  "costAssumptions": {
    "generation": {"inputTokensPerCall": 500, "outputTokensPerCall": 1000},
    "absoluteJudging": {"inputTokensPerCall": 1200, "outputTokensPerCall": 350},
    "pairwiseJudging": {"inputTokensPerCall": 2200, "outputTokensPerCall": 500},
    "contingencyRate": 0.25,
    "promptCachingDiscountIncluded": false
  },
  "analysis": {
    "activePhase": "reference-extension",
    "activeConditionIds": [
      "dinner_control",
      "dinner_pulp_fiction",
      "dinner_inception",
      "dinner_step_brothers",
      "dinner_eyes_wide_shut"
    ],
    "expectedGenerationRequests": 104,
    "expectedAbsoluteJudgeCalls": 312,
    "expectedPairwiseMatchups": 364,
    "expectedPairwiseJudgeCalls": 1092,
    "budgetCapUsd": 8.17,
    "primaryOutcome": "Bradley-Terry pairwise ranking",
    "tieValue": 0.5,
    "bootstrapCluster": "condition-repeat block",
    "confidenceLevel": 0.95,
    "acceptableScriptMinimumMeanRubric": 75,
    "publicationAssets": [
      "overall-ranking-forest-plot",
      "condition-win-rate-heatmap",
      "results-table"
    ]
  }
}