{ "evaluation_id": "EVAL-20260403-103635", "question_id": "ANALYSIS-024", "question_text": "A quantitative trading firm backtests a strategy: 15% annual return, Sharpe ratio 2.1, max drawdown 8%. They want to go live. (1) What could go wrong between backtest and live trading? List at least 5 risks. (2) The backtest used 5 years of data and tested 200 parameter combinations. Calculate the probability this outperformance is due to overfitting. (3) Design a live testing protocol that minimizes capital at risk while validating the strategy.", "category": "analysis", "timestamp": "2026-04-03T10:36:35.549Z", "display_date": "Apr 03, 2026", "winner": { "name": "GPT-5.4", "provider": "openrouter", "score": 9.29 }, "avg_score": 7.101111, "matrix_size": 74, "models_used": [ { "id": "deepseek_v4", "name": "DeepSeek V4", "provider": "openrouter" }, { "id": "claude_opus_46", "name": "Claude Opus 4.6", "provider": "openrouter" }, { "id": "gpt_5_4", "name": "GPT-5.4", "provider": "openrouter" }, { "id": "mimo_v2_flash", "name": "MiMo-V2-Flash", "provider": "Xiaomi" }, { "id": "claude_sonnet_46", "name": "Claude Sonnet 4.6", "provider": "openrouter" }, { "id": "grok_420", "name": "Grok 4.20", "provider": "openrouter" }, { "id": "gemini_31_pro", "name": "Gemini 3.1 Pro", "provider": "openrouter" }, { "id": "gpt_oss_120b", "name": "GPT-OSS-120B", "provider": "OpenAI" }, { "id": "gemini_3_flash", "name": "Gemini 3 Flash Preview", "provider": "Google" } ], "rankings": { "gpt_5_4": { "display_name": "GPT-5.4", "provider": "openrouter", "average_score": 9.29, "score_count": 9, "min_score": 8.8, "max_score": 9.8, "rank": 1 }, "grok_420": { "display_name": "Grok 4.20", "provider": "openrouter", "average_score": 8.52, "score_count": 9, "min_score": 7, "max_score": 9.55, "rank": 2 }, "claude_opus_46": { "display_name": "Claude Opus 4.6", "provider": "openrouter", "average_score": 7.67, "score_count": 9, "min_score": 4.95, "max_score": 9.2, "rank": 3 }, "mimo_v2_flash": { "display_name": "MiMo-V2-Flash", "provider": "Xiaomi", "average_score": 7.55, "score_count": 8, "min_score": 5.1, "max_score": 9, "rank": 4 }, "gpt_oss_120b": { "display_name": "GPT-OSS-120B", "provider": "OpenAI", "average_score": 7.51, "score_count": 8, "min_score": 4.4, "max_score": 9.2, "rank": 5 }, "deepseek_v4": { "display_name": "DeepSeek V4", "provider": "openrouter", "average_score": 7.49, "score_count": 9, "min_score": 6.05, "max_score": 9, "rank": 6 }, "claude_sonnet_46": { "display_name": "Claude Sonnet 4.6", "provider": "openrouter", "average_score": 7.09, "score_count": 8, "min_score": 3.8, "max_score": 9.2, "rank": 7 }, "gemini_3_flash": { "display_name": "Gemini 3 Flash Preview", "provider": "Google", "average_score": 6.14, "score_count": 5, "min_score": 0.5, "max_score": 8.2, "rank": 8 }, "gemini_31_pro": { "display_name": "Gemini 3.1 Pro", "provider": "openrouter", "average_score": 2.65, "score_count": 9, "min_score": 0.85, "max_score": 5.45, "rank": 9 } }, "judgments": [ { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7.75, "brief_justification": "Accurate risk list and practical live testing steps; probability estimate is vague and not rigorously derived, limiting correctness and depth." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 7.25, "brief_justification": "" }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.8, "brief_justification": "Excellent response that accurately identifies risks, correctly handles the ambiguous probability question with statistical context, and provides a highly practical, industry-standard staging protocol." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.95, "brief_justification": "" }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.65, "brief_justification": "The response provides excellent analysis of risks and overfitting probability using the Deflated Sharpe Ratio framework, but it is cut off at the end, leaving the live testing protocol incomplete." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.05, "brief_justification": "The response provides excellent and accurate detail for the first question but completely fails to address the second and third questions, and cuts off abruptly at the end." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7.3, "brief_justification": "" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 0.85, "brief_justification": "The response is a tiny, incomplete fragment that appears to be a self-correction note rather than an actual answer. It addresses none of the three parts of the question in any meaningful way. There is no list of risks, no overfitting calculation, and no live testing protocol. The snippet that exists is cut off mid-sentence and provides essentially no useful information." }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.35, "brief_justification": "Exceptionally thorough and well-structured response. Part 1 goes well beyond the required 5 risks with 12 well-explained items. Part 2 correctly notes the calculation cannot be done exactly, provides the standard multiple-testing approximation (1-0.95^200), and importantly caveats that this doesn't directly mean the strategy is overfit—just that naive significance is meaningless. References to advanced methods (PBO, Deflated Sharpe, White's Reality Check) show real depth. Part 3 is outstanding—a" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7, "brief_justification": "The response covers the key risks well in part (1) with 5 clearly articulated risks. Part (2) attempts a quantitative approach using the Bailey et al. framework but the calculation has issues: the formula used appears to conflate annualized and per-observation Sharpe ratios. The correct expected max SR under the null from 200 trials over 5 years would be much higher in annualized terms (sqrt(2*ln(200)) ≈ 3.26 annualized), which would actually suggest the observed SR of 2.1 could plausibly arise " }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.8, "brief_justification": "The response covers the five risks well with clear explanations. However, the overfitting calculation in Part 2 has significant issues: the variance formula for the Sharpe ratio is not correctly applied (mixing annualized SR variance with T in years rather than number of observations), the t-distribution application is questionable, and the simplified DSR formula presented is not the actual DSR formula from Bailey & López de Prado. The final conclusion of >90% overfitting probability is directio" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 7.05, "brief_justification": "The response provides a strong and detailed Part 1 with well-illustrated risks (more than 5, with quantitative examples), but the response is clearly incomplete — it cuts off mid-table in Part 1 and never addresses Part 2 (overfitting probability calculation) or Part 3 (live testing protocol design). Part 2 required a specific calculation involving multiple comparisons / Bonferroni-type correction or similar statistical reasoning about 200 parameter combinations over 5 years, which is entirely m" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.75, "brief_justification": "The response is comprehensive, well-structured, and highly practical. The 8+ risks in part (1) are all legitimate and well-explained. Part (2) correctly identifies the multiple-testing problem and references appropriate frameworks (DSR, Bailey/López de Prado), though the 85-95% estimate is more of an informed heuristic than a rigorous calculation—the question asked to 'calculate' the probability, and the response doesn't show explicit math (e.g., E[max Sharpe] under null). Still, the qualitative" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 7.2, "brief_justification": "The response covers Part 1 excellently with 10 well-explained risks. Part 2 has a significant conceptual issue: the POF calculation (1-(1-α)^N) answers 'what is the probability at least one of 200 null-hypothesis strategies passes a 5% test?' which is not the same as 'what is the probability THIS specific strategy's outperformance is due to overfitting.' The z-score of 41.7 actually suggests the signal is extremely strong even after multiple testing correction, which contradicts the 99.996% over" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 0.5, "brief_justification": "The response is completely empty - no content was provided to evaluate. The model failed to address any of the three parts of the question." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 1.2, "brief_justification": "The response is incomplete and does not answer the three requested parts. It mentions only one nuance about correlated trials, provides no calculation, no list of risks, and no live testing protocol. Clarity is limited because the thought is truncated." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 4.95, "brief_justification": "Lists several valid live-trading risks clearly, but the key quantitative part is flawed: it claims a precise overfitting probability from insufficient information, uses questionable Sharpe inference, and the answer is incomplete/truncated without a live testing protocol." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.05, "brief_justification": "Well-structured and lists relevant live-trading risks and a sensible phased rollout protocol, but the key calculation is not actually computed. It gives an unsupported 20-40% estimate for overfitting, mixes frameworks loosely, and uses questionable formulas/assumptions, so correctness is significantly weakened." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 5.1, "brief_justification": "The risks section is solid and clearly written, but the key quantitative part is flawed: the response invents an overfitting probability without enough information, misapplies Sharpe/DSR-style calculations, and presents unjustified numeric conclusions (>90%, 0.99998). The live-testing protocol is reasonable but incomplete/truncated, limiting completeness and practical usefulness." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 3.8, "brief_justification": "Identifies several real backtest-to-live risks with some helpful examples, but the answer is incomplete and cut off. It fails to answer part (2) with any probability calculation, does not design the requested live testing protocol in part (3), and includes some questionable or inconsistent quantitative examples." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7, "brief_justification": "Clear, well-structured, and practically useful, with strong coverage of live-trading risks and a sensible phased validation plan. However, the key numerical claim in part (2) is not actually calculated from the given information and overstates certainty; the response presents an unsupported 85–95% overfitting probability without a valid derivation, which materially hurts correctness." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 4.4, "brief_justification": "The response lists relevant live-trading risks clearly, but it fails to correctly answer parts (2) and (3). It incorrectly claims a calculable 99.996% overfitting probability from 200 tests using a simplistic multiple-testing formula, which is not justified from the given information and ignores dependence, true null assumptions, and proper methods like out-of-sample validation/deflated Sharpe. The answer is also truncated before finishing the Bonferroni section and does not provide the requeste" }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 5.45, "brief_justification": "Correct on correlation point but incomplete response missing key parts of the question." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 9.2, "brief_justification": "Highly accurate and comprehensive with excellent depth and practical insights." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.2, "brief_justification": "Comprehensive, accurate, and practical with excellent depth and clarity." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 9, "brief_justification": "Comprehensive, detailed, and well-structured response covering all aspects thoroughly." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 9.2, "brief_justification": "Extremely thorough and detailed analysis covering all critical risks with clear examples and calculations." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9.2, "brief_justification": "Comprehensive, accurate, and actionable with excellent depth." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 9.2, "brief_justification": "Detailed, accurate, and well-structured response covering all aspects with insightful analysis." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8, "brief_justification": "Comprehensive and accurate response with good practical advice." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 5.25, "brief_justification": "Response is incomplete, only addresses part of question 2 with a nuanced point but fails to list risks, calculate probability, or design protocol. Partial correctness on overfitting nuance but lacks full answer." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.4, "brief_justification": "Strong technical depth on overfitting metrics and execution risks, though the probability calculation lacks a definitive conclusion." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9, "brief_justification": "Comprehensive and accurate. Correctly identifies the impossibility of exact overfitting probability without assumptions, provides the standard approximation, and offers a detailed, actionable live testing protocol." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 8.8, "brief_justification": "Accurate risk identification, sound PBO estimation, and practical testing protocol." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9, "brief_justification": "Excellent, comprehensive response. Correctly identifies multiple critical risks, applies appropriate quantitative methodology to estimate overfitting probability (citing relevant literature), and provides a detailed, actionable, and industry-standard live testing protocol. Minor deduction in correctness for not explicitly mentioning the need for out-of-sample data in the probability calculation, but the overall estimate is sound." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.3, "brief_justification": "Comprehensive and well-structured, but the overfitting probability calculation is oversimplified and lacks nuance for live strategy validation." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.2, "brief_justification": "Strong, practical answers to all parts. Part 2 uses a reasonable heuristic (false discovery rate) but lacks a precise calculation. Part 3 protocol is solid but could include more on risk management." }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 1.45, "brief_justification": "The response is severely incomplete - it appears to be a fragment of a self-correction note mid-sentence, missing all three required parts of the question. No risks are listed, no probability calculation is provided, and no live testing protocol is designed. The partial content about correlated parameters shows some domain knowledge but is essentially useless as a standalone response." }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8, "brief_justification": "The response is technically strong with excellent depth on overfitting analysis, DSR framework, and risk taxonomy. Part 1 covers 7 well-explained risks with practical detail. Part 2 shows sophisticated quantitative work with multiple methods. However, the response is incomplete - Part 3 (live testing protocol) is entirely missing, which is a significant gap in completeness. The math in Part 2 is mostly correct though the synthesis table is cut off. The DSR calculation shows genuine expertise but" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.35, "brief_justification": "Exceptional response covering all three parts thoroughly. The risks section goes well beyond the minimum 5 with nuanced, practitioner-level insights. The overfitting calculation is handled with intellectual honesty—correctly noting the limitations of the simple approximation while still providing the expected quantitative answer and pointing to more rigorous methods (Deflated Sharpe, PBO, CSCV). The live testing protocol is detailed, staged, and actionable with concrete metrics and kill-switch c" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7.35, "brief_justification": "The response covers the key risks well and provides a practical live testing protocol. The overfitting probability calculation attempts to use the Bailey et al. PBO framework but the math is somewhat hand-wavy and the final estimate of 20-40% lacks rigorous derivation. The formula application is partially correct but the conclusion jumps without showing proper statistical work. The risks section is solid with 5+ relevant items. The live testing protocol is actionable and well-structured. Minor f" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 8.2, "brief_justification": "Strong response covering all three parts with good detail. The five risks are well-articulated and practically relevant. The overfitting probability calculation attempts the DSR framework correctly in spirit, though the mathematical execution has some imprecision (the SE formula used is a simplified approximation, and the final >90% conclusion is reasonable but not rigorously derived). The live testing protocol is practical and well-structured. Minor issues: the response appears truncated at Pha" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.8, "brief_justification": "Comprehensive and well-structured response covering all three parts thoroughly. The risk enumeration is excellent with 8 distinct, well-explained risks. The overfitting probability calculation references appropriate frameworks (Bailey et al., Deflated Sharpe Ratio) and gives reasonable estimates, though the exact probability range (85-95%) is somewhat heuristic rather than precisely calculated. The live testing protocol is practical and actionable with clear phases and kill criteria. Minor weakn" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8, "brief_justification": "The response is well-structured and covers risks comprehensively with 10 well-explained items. The statistical treatment of overfitting probability is mathematically sound and uses appropriate frameworks (Lo 2002 SE, Bonferroni correction, POF formula). The response appears to be cut off before completing section 2.3 and entirely missing section 3 (live testing protocol), which significantly hurts completeness. The content that is present is accurate, clear, and practically useful for a quant tr" }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 3.25, "brief_justification": "The response is incomplete and cuts off mid-sentence. It fails to address any of the three parts of the question, focusing instead on a tangential statistical nuance about parameter correlation without providing the required list of risks, probability calculation, or live testing protocol." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 7.55, "brief_justification": "Response covers all parts comprehensively with good structure and formulas. However, correctness is reduced due to critical errors in Part 2: incorrect standard error for annual Sharpe (should be ~0.45/sqrt(5) or properly annualized), flawed probability calculation ignoring the max of normals properly, and inconsistent DSR application that leads to contradictory conclusions (expected max 1.17 vs 2.7). The probability estimate is misleadingly low." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9, "brief_justification": "Response is factually accurate, logically sound, and covers all parts thoroughly with insightful details on risks, multiple-testing math, and a practical phased protocol. Minor caveat on the probability framing but overall excellent." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.75, "brief_justification": "The response covers the main points with reasonable structure and some formulas, but contains inaccuracies in the overfitting probability calculation (misapplied formula, incorrect expected max SR derivation) and provides only approximate ranges without rigorous justification. Risks list and live protocol are solid but not exhaustive." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 8.6, "brief_justification": "Provides extensive, well-structured risks with formulas and examples (exceeds 5 requested). Addresses overfitting via discussion of multiple biases though lacks exact probability calculation for part 2. Live protocol is implied in risk mitigation but not explicitly designed as a step-by-step protocol for part 3. High clarity with tables and categories." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.4, "brief_justification": "The response is factually accurate on risks and overfitting calculation, provides thorough coverage with a table and formulas, is clearly structured, offers insightful statistical depth, and gives practical live testing advice (though part 3 is cut off). Minor issues like incomplete section 3 and approximate formulas do not significantly detract." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 6, "brief_justification": "The response covers the main points for each part but contains inaccuracies in the overfitting probability calculation (misapplies the formula and gives an implausibly low p-value without proper multiple-testing adjustment), lists only 4 risks instead of 5+, and provides a reasonable but somewhat generic live testing protocol. Overall moderately useful with room for more precision and depth." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 1.85, "brief_justification": "Response only mentions a nuance about trial independence without providing the requested risk list, overfitting probability calculation, or live testing protocol, making it largely incomplete and of limited practical value." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 5.6, "brief_justification": "Provides detailed risk list and quantitative analysis, but probability answer is ambiguous and partially incorrect, and omits the live testing protocol, reducing completeness and usefulness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.8, "brief_justification": "Accurate risks and realistic protocol; thorough coverage; clear structure; good analytical depth; highly actionable." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.8, "brief_justification": "The response correctly identifies key risks and offers a reasonable, though simplified, overfitting analysis and a solid live‑testing framework, but the statistical calculations are approximate and the live protocol is cut off, reducing completeness and overall usefulness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.2, "brief_justification": "The response gives a thorough, well-structured risk analysis with good depth, but it fails to answer parts (2) and (3) of the question, reducing completeness, overall correctness, and practical usefulness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.1, "brief_justification": "Provides thorough risks and a solid live-testing protocol, clear and detailed. Probability estimate is vague and not a precise calculation, lowering correctness slightly." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 2.05, "brief_justification": "The response is a fragment of a 'thought process' or 'self-correction' rather than an actual answer to the prompt. It fails to address any of the three specific questions asked." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 9.2, "brief_justification": "The response is exceptionally high quality. It provides a sophisticated taxonomy of risks, a rigorous mathematical treatment of the multiple testing problem using order statistics and the Deflated Sharpe Ratio framework, and a practical live testing protocol. It correctly identifies that while a naive calculation suggests low probability of overfitting, a more realistic model (DSR) suggests the result is likely noise." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.8, "brief_justification": "The response is exceptionally thorough. It correctly identifies that the probability of overfitting cannot be calculated exactly but provides the standard statistical approximation (1-alpha^N) used in quantitative finance interviews. The risks listed are comprehensive, and the live testing protocol is highly practical and industry-standard." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 9, "brief_justification": "The response is highly accurate and structured. It correctly identifies key risks, provides a sophisticated approach to the overfitting calculation using the Bailey et al. framework, and offers a practical, multi-stage live testing protocol." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 9, "brief_justification": "The response is technically sophisticated, correctly identifying the Deflated Sharpe Ratio (DSR) as the appropriate framework for the overfitting calculation. It provides a highly professional risk assessment and a logical, phased protocol for live testing. The response was cut off at the very end, but the content provided is excellent." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 9, "brief_justification": "The response provides an exceptionally detailed and professional analysis of backtest-to-live risks (Part 1). However, it is incomplete as it cuts off before answering Part 2 (overfitting calculation) and Part 3 (live testing protocol)." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9.55, "brief_justification": "The response is excellent. It correctly identifies the primary risks, provides a statistically sound estimate for the probability of overfitting based on the Deflated Sharpe Ratio concept, and outlines a professional-grade phased deployment protocol used in institutional quantitative trading." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 9, "brief_justification": "The response is excellent. It provides a comprehensive list of risks, a mathematically sound (though simplified) approach to the probability of overfitting using the Bonferroni logic and Lo's SE formula, and a very practical live testing protocol. It correctly identifies the multiple testing problem inherent in the prompt's scenario." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 2.5, "brief_justification": "Response only addresses a minor nuance about one sub-question (correlation of parameter trials) but fails to answer the three main parts of the question: listing risks, calculating overfitting probability, and designing a live testing protocol." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.85, "brief_justification": "Comprehensive response covering all three parts with accurate risk taxonomy, rigorous overfitting probability calculations using multiple methods (order statistics, exact probability, Deflated Sharpe Ratio), and a sensible phased live testing protocol. Well-structured with clear mathematical notation and actionable recommendations." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.35, "brief_justification": "Comprehensive response covering all three parts. Lists 12 distinct risks with explanations. Correctly notes overfitting probability cannot be calculated exactly but provides useful multiple-testing approximation with proper caveats. Live testing protocol is thorough with specific phases, percentages, risk controls, and metrics. Slight扣分 for potentially misleading interpretation of the multiple-testing calculation despite clarification." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7.8, "brief_justification": "Well-structured response covering all three parts. Risks are accurate and explained. The PBO calculation discusses the right framework but provides a vague range rather than a precise figure. Live testing protocol is practical and actionable with specific parameters." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.85, "brief_justification": "Comprehensive coverage of backtest-to-live risks with accurate identification of 5 key issues. Statistical overfitting analysis uses proper Deflated Sharpe Ratio methodology though conclusion of >90% seems aggressive. Live testing protocol is well-designed but incomplete - ends abruptly mid-sentence at Phase 2. Clear structure and good depth throughout." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.8, "brief_justification": "Technically accurate and deeply detailed risk analysis for Part 1, but completely omits Parts 2 (overfitting probability calculation) and 3 (live testing protocol), significantly reducing practical value." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9, "brief_justification": "Comprehensive answer covering all three parts. Correctly identifies 8 key risks with proper explanations. Uses appropriate statistical frameworks (FWER, Deflated Sharpe Ratio) for overfitting probability, arriving at 85-95% which is reasonable given 200 trials on 5 years. Well-designed phased live testing protocol with specific kill criteria and capital limits. Professional structure with clear headings and actionable recommendations." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 5.55, "brief_justification": "Excellent risk identification with 10 valid risks and correct Sharpe SE calculation. However, the response is severely incomplete: it cuts off mid-calculation for the Bonferroni critical Sharpe and entirely omits the requested live testing protocol in part 3. The overfitting probability calculation method is sound but not finished, leaving the analysis incomplete. Major deduction for failing to address a core question component." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8, "brief_justification": "Comprehensive response covering all three parts with accurate risks, correct overfitting probability calculation (~2.5%), and a practical phased live testing protocol. Well-structured and actionable." } ], "meta": { "source": "The Multivac (app.themultivac.com)", "methodology": "10x10 blind peer matrix evaluation", "criteria": "correctness, completeness, clarity, depth, usefulness", "self_judgments": "excluded from rankings", "license": "Open data — cite as: The Multivac (2026)" } }