{ "evaluation_id": "EVAL-20260402-151829", "question_id": "CODE-030", "question_text": "Design a GraphQL schema for a social media platform with users, posts, comments, likes, and follows. Address: N+1 query problem with DataLoader pattern, cursor-based pagination, proper input validation, rate limiting per field, and subscription for real-time updates. Include resolver implementations for the trickiest queries.", "category": "code", "timestamp": "2026-04-02T15:18:29.690Z", "display_date": "Apr 02, 2026", "winner": { "name": "Gemini 3 Flash Preview", "provider": "Google", "score": 8.24 }, "avg_score": 7.148, "matrix_size": 88, "models_used": [ { "id": "claude_sonnet_46", "name": "Claude Sonnet 4.6", "provider": "openrouter" }, { "id": "grok_420", "name": "Grok 4.20", "provider": "openrouter" }, { "id": "gpt_oss_120b", "name": "GPT-OSS-120B", "provider": "OpenAI" }, { "id": "gemini_3_flash", "name": "Gemini 3 Flash Preview", "provider": "Google" }, { "id": "minimax_m25", "name": "MiniMax M2.5", "provider": "openrouter" }, { "id": "mimo_v2_flash", "name": "MiMo-V2-Flash", "provider": "Xiaomi" }, { "id": "claude_opus_46", "name": "Claude Opus 4.6", "provider": "openrouter" }, { "id": "gemini_31_pro", "name": "Gemini 3.1 Pro", "provider": "openrouter" }, { "id": "deepseek_v4", "name": "DeepSeek V4", "provider": "openrouter" }, { "id": "gpt_5_4", "name": "GPT-5.4", "provider": "openrouter" } ], "rankings": { "gemini_3_flash": { "display_name": "Gemini 3 Flash Preview", "provider": "Google", "average_score": 8.24, "score_count": 9, "min_score": 7.2, "max_score": 9.35, "rank": 1 }, "gpt_5_4": { "display_name": "GPT-5.4", "provider": "openrouter", "average_score": 8.02, "score_count": 9, "min_score": 6.15, "max_score": 9.6, "rank": 2 }, "gpt_oss_120b": { "display_name": "GPT-OSS-120B", "provider": "OpenAI", "average_score": 7.96, "score_count": 9, "min_score": 4.65, "max_score": 9.45, "rank": 3 }, "deepseek_v4": { "display_name": "DeepSeek V4", "provider": "openrouter", "average_score": 7.78, "score_count": 8, "min_score": 6.4, "max_score": 9.55, "rank": 4 }, "grok_420": { "display_name": "Grok 4.20", "provider": "openrouter", "average_score": 7.62, "score_count": 9, "min_score": 5.4, "max_score": 8.8, "rank": 5 }, "mimo_v2_flash": { "display_name": "MiMo-V2-Flash", "provider": "Xiaomi", "average_score": 7.26, "score_count": 9, "min_score": 5.05, "max_score": 8.8, "rank": 6 }, "claude_opus_46": { "display_name": "Claude Opus 4.6", "provider": "openrouter", "average_score": 7.17, "score_count": 9, "min_score": 4.65, "max_score": 9, "rank": 7 }, "claude_sonnet_46": { "display_name": "Claude Sonnet 4.6", "provider": "openrouter", "average_score": 6.47, "score_count": 9, "min_score": 2.65, "max_score": 8.6, "rank": 8 }, "minimax_m25": { "display_name": "MiniMax M2.5", "provider": "openrouter", "average_score": 6.13, "score_count": 8, "min_score": 1.85, "max_score": 8.6, "rank": 9 }, "gemini_31_pro": { "display_name": "Gemini 3.1 Pro", "provider": "openrouter", "average_score": 4.84, "score_count": 9, "min_score": 1.85, "max_score": 7.8, "rank": 10 } }, "judgments": [ { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 8.6, "brief_justification": "Comprehensive schema with detailed directives and types, addressing key concerns like N+1 and pagination." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.8, "brief_justification": "Comprehensive schema with detailed implementation addressing all requirements, though clarity could be slightly improved with more comments." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 9.45, "brief_justification": "Comprehensive and well-structured with clear implementation details." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 9, "brief_justification": "Comprehensive and detailed response covering all requested aspects with clear implementations." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 8.6, "brief_justification": "Comprehensive and well-structured with good attention to key issues like N+1 and pagination." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 8.8, "brief_justification": "Comprehensive and well-structured with practical implementation details." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 4.65, "brief_justification": "The response starts with a solid schema structure and includes relevant concepts like directives, connections, and contextual fields, but it is clearly incomplete/truncated before finishing queries, mutations, subscriptions, validation details, DataLoader handling, resolver implementations, and rate-limiting mechanics. What is present is mostly coherent, but it does not fully answer the prompt." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 1.85, "brief_justification": "The response is severely incomplete and cuts off mid-schema, so it fails to address most required areas such as full schema design, DataLoader handling, input validation, rate limiting details, subscriptions, and resolver implementations. What is present is mostly accurate and clearly written, but far from sufficient." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 2.65, "brief_justification": "The response starts with a strong-looking schema, but it is cut off mid-input definition and fails to address major requested items: DataLoader/N+1 strategy explanation, cursor pagination implementation details, input validation approach beyond directives, rate limiting per field implementation, subscriptions, and resolver implementations for tricky queries. Clear formatting early on, but incomplete and not practically usable as delivered." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 5.4, "brief_justification": "Good schema structure and touches the requested topics, but the response is incomplete/truncated, resolver code is unfinished, subscriptions and per-field rate limiting are not fully implemented, and some details are underspecified or potentially problematic (e.g., cursor stability, validation directive semantics, DataLoader assumptions)." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.75, "brief_justification": "Well-structured schema and resolver examples with solid coverage of DataLoader, pagination, and subscriptions, but several requested areas are only partially addressed. Rate limiting is declared but not implemented, input validation is mostly asserted rather than shown, some resolver code is inconsistent or incomplete (e.g. undefined likesByPostLoader, DataLoader instantiated globally instead of per-request, naive in-memory pagination/filtering), and cursor handling is not robust/opaque enough f" }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 4.65, "brief_justification": "The schema is generally sensible and mentions the requested concerns, but the response is truncated before the crucial resolver implementations and setup details. It does not fully deliver the asked-for tricky resolvers, DataLoader usage, pagination logic, validation, rate limiting integration, or subscription implementation, so it is only partially complete and moderately useful." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 7.2, "brief_justification": "Solid high-level schema and resolver design with good coverage of DataLoader, pagination, subscriptions, and rate limiting. However, it is incomplete for the requested domain and implementation details: comments/likes/follows types and tricky resolvers are underdeveloped, input validation uses an undeclared constraint directive, the rate-limit directive implementation is not accurate for GraphQL directive args, createPost insert likely returns IDs not rows in many SQL builders, and pagination ba" }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 1.85, "brief_justification": "The response is incomplete and cuts off mid-schema, so it fails to address key requested areas like resolver implementations, DataLoader handling, validation details, field-level rate limiting strategy, and subscription design. What is present is generally sensible but too partial to be practically useful." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 5.05, "brief_justification": "Well-structured and includes a solid schema plus DataLoader examples, but the response is incomplete and cut off, omits key requested pieces like full tricky resolvers, proper subscriptions/rate-limiting implementation, and claims cursor pagination while mainly using offset-based pagination with acknowledged limitations." }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 6.65, "brief_justification": "The response provides a well-structured GraphQL SDL schema with Relay-style pagination, a reasonable database model, cursor design, and input validation strategy. However, it appears truncated - the Zod validation example is cut off mid-line, and critically, the promised resolver implementations for 'the trickiest queries' are missing entirely. The DataLoader pattern is mentioned but never actually implemented with code. Rate limiting is shown as a directive in the schema but the implementation " }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 3.1, "brief_justification": "The response is cut off very early, only showing a partial SDL schema. It lacks DataLoader pattern implementation, resolver implementations, N+1 query problem solutions, input validation, rate limiting implementation, subscription definitions, and cursor-based pagination resolver logic. While the partial schema shown is reasonable and follows Relay conventions, the response fails to address the majority of the question's requirements. The truncation makes it largely unusable as a comprehensive a" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.3, "brief_justification": "The schema definition is well-structured with good use of directives, cursor-based pagination following Relay spec, custom scalars, and complexity analysis directives. However, the response is clearly truncated - it cuts off mid-input type definition and never delivers on several key requirements: no DataLoader pattern implementation, no resolver implementations (especially for 'trickiest queries' as requested), no rate limiting implementation details, no subscription definitions for real-time u" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7, "brief_justification": "The schema is well-structured with proper cursor-based pagination following Relay conventions, good DataLoader implementations addressing N+1 problems, and reasonable rate limiting via directives. However, the response is clearly truncated mid-code (the isLikedByMe resolver cuts off), missing subscription resolver implementations, the rate limiting directive implementation details, and the input validation directive implementation. The feed resolver is a solid example of a tricky query. The Data" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7.2, "brief_justification": "The schema is well-structured and covers the core entities. Cursor-based pagination follows the Relay connection spec pattern. However, there are notable issues: (1) The DataLoader pattern for posts/comments loads ALL records then paginates in-memory, which defeats the purpose of cursor-based pagination at scale. (2) Rate limiting is declared as a directive but never actually implemented - no middleware or directive resolver is shown. (3) Input validation is mentioned but barely implemented beyo" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 7.65, "brief_justification": "The response provides a well-structured and comprehensive GraphQL schema covering all requested entities (users, posts, comments, likes, follows) with proper Relay-style cursor-based pagination, rate limiting directives, subscription types, and input validation approach. The schema design is solid and follows best practices. However, the response appears truncated - the server-side boilerplate code cuts off mid-sentence, meaning the promised 'resolver implementations for the trickiest queries' a" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 7.8, "brief_justification": "The response covers all five requested topics (N+1/DataLoader, cursor-based pagination, input validation, rate limiting, subscriptions) with working code examples. The schema is well-structured following Relay conventions, and the cursor-based pagination resolver is correctly implemented with the fetch-N+1 trick. Minor issues: the @constraint directive is used in the schema but never explained or implemented; the rate limiting directive implementation is simplified and doesn't correctly extract " }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 5.1, "brief_justification": "The response starts well with a reasonable GraphQL schema design covering users, posts, comments, likes, follows, cursor-based pagination (Relay style), rate limiting directives, and subscriptions. However, the response is clearly truncated - it cuts off mid-schema definition and never delivers the promised resolver implementations for tricky queries, the DataLoader pattern implementation for N+1 problems, input validation code, rate limiting implementation details, or subscription resolver code" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.2, "brief_justification": "The response provides a solid GraphQL schema with proper Relay-style cursor-based pagination, DataLoader pattern for N+1 resolution, and subscription types. However, several areas are incomplete or have issues: (1) The response is cut off mid-sentence in the User.isFollowing resolver, leaving key resolvers unfinished. (2) Rate limiting per field is mentioned in the architecture overview but no actual implementation is provided - no complexity analysis code, no directive, no middleware. (3) Input" }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 6.15, "brief_justification": "The response provides a solid schema, DB design, and cursor pagination strategy, but cuts off abruptly and completely misses the requested DataLoader and resolver implementations." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 5.5, "brief_justification": "The schema design is excellent and follows best practices, but the response is severely cut off. It completely misses the requested DataLoader implementations, subscriptions, and resolvers." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.35, "brief_justification": "The schema design is advanced and well-structured, utilizing Relay connections and custom directives. However, the response cuts off abruptly and completely misses the requested resolver implementations, subscriptions, and DataLoader logic." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.05, "brief_justification": "The response provides a well-structured GraphQL schema, effective DataLoader patterns, and clear pagination logic, but it is abruptly cut off at the end, impacting completeness." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.4, "brief_justification": "Schema is well-structured, but the DataLoader implementation uses in-memory pagination which is a severe anti-pattern for large datasets. Rate limiting and input validation lack actual implementation." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 6.6, "brief_justification": "The schema design is excellent and covers most requirements, but the response cuts off abruptly before providing the requested resolver implementations and DataLoader logic." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 9, "brief_justification": "The response effectively addresses all prompt requirements with solid code examples and production considerations. However, it misses a few type definitions (e.g., Comment, LikePayload) in the schema, slightly impacting completeness." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 4.4, "brief_justification": "The response cuts off abruptly in the middle of the schema definition, completely missing the requested DataLoader implementations, resolver logic, and input validation details." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.1, "brief_justification": "" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.6, "brief_justification": "Excellent comprehensive response covering all required aspects: GraphQL SDL with directives, Relay-style pagination, DataLoader pattern, cursor design, input validation with Zod, and database schema with proper indexes. The response is well-structured and practically actionable. Minor deductions for the truncated Zod schema at the end and incomplete resolver implementations for the 'trickiest queries' as specifically requested. The N+1 DataLoader pattern is mentioned but not fully implemented in" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8, "brief_justification": "The schema is well-structured with proper use of Relay-style pagination, custom directives for rate limiting and auth, DataLoader-backed computed fields, and union types. The schema covers all required entities (users, posts, comments, likes, follows) with viewer-contextual fields. However, the response appears truncated (Query type cuts off mid-definition), and the actual DataLoader resolver implementations and subscription definitions promised in the title are missing. The N+1 problem is addre" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 5.65, "brief_justification": "The response starts well with a solid schema design using Relay Connection Specification, custom scalars, and rate limiting directives. However, the response is clearly truncated mid-sentence, cutting off at the Comment type definition. Critical requirements like DataLoader pattern implementation, resolver code for N+1 problem, subscription implementations, input validation, and the 'trickiest query' resolver implementations are entirely missing. What exists is correct and well-structured, but t" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.2, "brief_justification": "The response provides a well-structured GraphQL schema with proper cursor-based pagination using the Relay connection spec, DataLoader implementations for N+1 prevention, custom directives for rate limiting and validation, and subscription types. The TypeScript resolver implementations are practical and show real patterns. Minor issues: the response is cut off mid-sentence in the resolver section, the isLikedByMeLoader has a flawed assumption that all keys share the same userId, and the directiv" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 8, "brief_justification": "The response provides a comprehensive GraphQL schema with good coverage of all requested features. The schema design is well-structured with proper connection types, cursor-based pagination, and subscription support. DataLoader implementations are shown but have a notable flaw: loading all records and doing in-memory pagination defeats the purpose of efficient cursor-based pagination for large datasets. Rate limiting directive is declared but never actually implemented with resolver logic. The c" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.8, "brief_justification": "Comprehensive, well-structured response covering all requested features: DataLoader pattern, cursor-based pagination (Relay style), input validation with custom scalars and Joi, rate limiting via directives, and subscriptions. The schema SDL is thorough and production-ready. TypeScript boilerplate demonstrates practical implementation. Minor deductions for the truncated server.ts code and slightly surface-level treatment of some resolver implementations for the 'trickiest queries' as specificall" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.35, "brief_justification": "Well-structured response covering all requested topics: DataLoader for N+1, cursor-based pagination, rate limiting via directives, and subscriptions with Redis PubSub. Schema design follows Relay spec conventions. Code examples are practical and implementable. Minor gaps include: input validation using @constraint directive isn't natively supported without a plugin (not mentioned), the CommentConnection type is referenced but not defined, and the directive implementation for rateLimitDirective u" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 6.45, "brief_justification": "The schema design is solid with good use of Relay-style cursor pagination, custom directives for rate limiting, and proper input types. The SDL is well-structured and clear. However, the response is incomplete - it cuts off mid-schema (postLiked subscription is unfinished), and critically missing are: the DataLoader implementations for solving N+1 queries, actual resolver code for the 'trickiest queries', rate limiting implementation details, and input validation logic. The question explicitly a" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.8, "brief_justification": "The response demonstrates strong technical depth with well-structured schema design, proper DataLoader implementation, cursor-based pagination, and subscription setup. The architecture choices (Prisma, Redis, Apollo) are sound and well-justified. However, the response is cut off mid-implementation (User.isFollowing resolver is incomplete), rate limiting is mentioned in the architecture but not implemented in code, and the cursor-based pagination uses offset-based cursors rather than true keyset " }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9, "brief_justification": "The response comprehensively covers all requested aspects with accurate schema design, practical patterns for DataLoader/N+1, cursor pagination, validation, rate limiting, and subscriptions. Resolver examples are partially provided but the overall coverage is deep and actionable." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.65, "brief_justification": "The schema comprehensively addresses all requested features with production-grade patterns for pagination, rate limiting, auth, and DataLoader hints. Resolver implementations for the trickiest queries (feed) are missing, reducing completeness slightly." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 5.85, "brief_justification": "The response starts a solid schema with good practices like Relay pagination, custom directives for rate limiting, and mentions of DataLoader, but is incomplete (cuts off mid-type definition), lacks full resolver implementations for tricky queries, doesn't show DataLoader or subscription code, and has no input validation examples." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 7.9, "brief_justification": "Schema is comprehensive and addresses most requested features (DataLoader mentions, cursor pagination, validation directives, rate limiting, subscriptions implied via complexity). However, the response is truncated mid-input type, lacks any actual resolver implementations (especially for trickiest queries), has no explicit DataLoader or subscription examples, and misses rate limiting per field usage in the provided snippet." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7.8, "brief_justification": "The response covers all requested features comprehensively with good examples. Some DataLoader implementations are flawed (e.g., loading all posts then filtering in memory, incorrect loader usage for counts), pagination logic has edge cases, and rate limiting directive is declared but not integrated into resolvers. Still provides solid, practical foundation." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.65, "brief_justification": "The response provides a comprehensive, well-structured GraphQL schema covering all requested features with practical implementation details. It effectively addresses N+1 via DataLoaders, cursor pagination, validation, rate limiting, and subscriptions. Resolver examples for tricky parts are partially shown but the overall design is production-oriented and insightful." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.2, "brief_justification": "The response provides a solid, practical GraphQL schema design covering all requested features with good code examples. Minor issues include incomplete input validation beyond a basic directive, partial subscription implementation, and some simplifications in pagination/cursor logic, but overall it's accurate and insightful." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 7, "brief_justification": "Schema covers core entities, pagination, inputs, and directives for rate limiting. Addresses N+1 via mention of DataLoader but lacks actual implementation. Resolver examples missing entirely despite request for 'trickiest queries'. Subscription incomplete. Good structure and clarity overall." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.8, "brief_justification": "The response provides a solid, practical GraphQL schema design with good coverage of DataLoader for N+1, cursor pagination, and subscriptions. However, it is incomplete (resolvers cut off mid-statement), uses offset-based pagination instead of true keyset cursor pagination, and has some inaccuracies in the feed query implementation." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9, "brief_justification": "Comprehensive and detailed response covering all requested aspects with clear examples and practical implementations." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.8, "brief_justification": "Comprehensive schema with detailed implementations addressing all requested features." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 7.8, "brief_justification": "Detailed and technically sound, but schema is incomplete." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 7.45, "brief_justification": "The response presents a solid schema with pagination, rate limiting, and validation, and explains cursor encoding. However, it lacks concrete resolver code for the trickiest queries and detailed DataLoader usage, reducing completeness and depth." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 6.1, "brief_justification": "The schema is well-structured and correctly uses GraphQL features like pagination, directives, and DataLoader hints, but it is incomplete (truncated, missing subscriptions and resolver implementations) and thus only partially useful." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 5, "brief_justification": "The response starts well with clear SDL and mentions key concepts, but it is truncated and omits many required parts such as full type definitions, DataLoader setup, resolver code, pagination details, input validation, rate limiting implementation, and subscriptions, limiting its correctness, completeness, depth, and practical usefulness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 4.9, "brief_justification": "The schema is fairly well-structured and uses appropriate directives, but it is incomplete (truncated input types, missing subscriptions and resolver code) and thus lacks full coverage and practical utility." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7, "brief_justification": "The response provides a solid schema, directives, pagination, and DataLoader examples, but the resolver section is incomplete and omits several tricky queries, reducing overall completeness and correctness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 7.25, "brief_justification": "Accurate schema and implementations for key parts, clear presentation, good depth on performance patterns, but missing several required types (comments, likes, follows) and related resolvers, reducing overall completeness and practical utility." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.6, "brief_justification": "The answer provides a solid schema and DataLoader example, but cuts off mid‑implementation, lacks detailed rate‑limiting logic and full resolver coverage, reducing completeness and depth despite clear presentation." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 9.6, "brief_justification": "The response is exceptionally high quality. It provides a complete SDL, detailed database indexing strategies, a robust cursor implementation, and clear architectural patterns for N+1 and rate limiting. It only cuts off slightly at the very end of the Zod example, but the core requirements were thoroughly addressed." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 9, "brief_justification": "The response provides a high-quality, production-ready GraphQL schema that addresses all requirements including Relay-style pagination, custom directives for rate limiting and auth, and DataLoader-ready fields. It was cut off at the end (missing the actual resolver code implementation), which slightly reduces the completeness score, but the schema design itself is exemplary." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 7.1, "brief_justification": "The response is cut off mid-sentence, failing to provide the full schema, the DataLoader implementation, or the resolver logic requested. While the provided portion is high quality and follows best practices like Relay Connection specs, it is incomplete." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 8.25, "brief_justification": "The schema design is exceptionally high quality, utilizing advanced GraphQL patterns like custom directives for validation/rate-limiting and proper Relay-style connections. However, the response is incomplete as it cuts off mid-input-definition and fails to provide the requested resolver implementations for DataLoader and pagination." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.8, "brief_justification": "The response provides a high-quality GraphQL schema and implementation details. It correctly addresses the N+1 problem using DataLoaders, implements cursor-based pagination logic, and uses custom directives for rate limiting and validation. The only minor drawback is that the code snippet for the 'isLikedByMe' loader in the resolver section was cut off at the very end, though the logic was already established in the DataLoader section." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 9.55, "brief_justification": "The response provides a comprehensive and technically sound GraphQL schema and resolver implementation. It correctly addresses the N+1 problem using DataLoaders, implements cursor-based pagination, and includes real-time subscriptions. The use of a custom directive for rate limiting and the logic for complex feed ordering are particularly strong." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.8, "brief_justification": "The response provides a high-quality, professional GraphQL schema that addresses all requirements. It correctly implements Relay-style cursor pagination, uses directives for rate limiting, and outlines the DataLoader pattern to solve N+1 issues. While the code snippet cuts off at the end, the schema and architectural explanation are excellent." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 8.25, "brief_justification": "The schema follows industry standards (Relay spec) and addresses the technical requirements like rate limiting and pagination. However, the response is truncated at the end, missing the full subscription implementation and the requested resolver logic for DataLoader." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 8.8, "brief_justification": "The response provides a high-quality GraphQL schema and resolver implementation. It correctly addresses the N+1 problem using DataLoader, implements Relay-style cursor pagination, and includes real-time subscription logic. It loses a small amount of completeness because the resolver code cuts off at the very end and the rate-limiting section is described conceptually rather than shown in code." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 6.9, "brief_justification": "Well-structured schema covering all required aspects but response is incomplete - cuts off mid-sentence at input validation section and missing the promised resolver implementations for tricky queries." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 5.85, "brief_justification": "Schema is well-structured with correct Relay pagination, DataLoader mentions, rate limiting directives, and auth patterns. However, response is truncated/incomplete - missing subscriptions entirely (required), mutations section is not included, and actual resolver implementations for the trickiest queries (like feed) are not provided. Custom scalars and union types are properly defined, but critical requirements from the question remain unaddressed." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 4, "brief_justification": "Response is severely incomplete - schema cuts off mid-definition, missing DataLoader implementation, resolver code, subscriptions for real-time updates, and input validation. Only provides partial SDL without the promised comprehensive solution." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.3, "brief_justification": "Well-structured GraphQL schema with proper types, enums, directives, and cursor-based pagination. However, response is abruptly cut off mid-schema (incomplete input definition) and critically lacks the requested resolver implementations, DataLoader pattern code, and subscription setup for real-time updates. The N+1 solution is only mentioned conceptually but not implemented." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7.35, "brief_justification": "Comprehensive schema covering all entities with cursor-based pagination and DataLoader implementation. However, response is truncated (resolver code incomplete), and @validate/@rateLimit directives are defined but their implementation is not shown. Subscription types defined but no implementation provided." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 7.55, "brief_justification": "Well-structured schema with proper DataLoader implementation for N+1, cursor-based pagination, and subscriptions. However, rate limiting is only defined as a directive without implementation, and the cursor-based pagination logic has issues - using post IDs as cursors doesn't properly handle datetime-based ordering. Input validation is minimal." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.25, "brief_justification": "Well-structured GraphQL schema with all requested features (DataLoader, cursor pagination, validation, rate limiting, subscriptions). Schema is syntactically correct and follows best practices. Partial resolver implementation provided - boilerplate shown but not all tricky query implementations fully detailed." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8, "brief_justification": "Well-structured response covering schema design, DataLoader for N+1, cursor pagination, rate limiting, and subscriptions. Code is mostly correct and actionable. Minor gaps: incomplete subscription types, missing CommentConnection, and some resolver details not fully implemented." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.2, "brief_justification": "Well-structured schema with solid DataLoader implementation for N+1 problem and cursor-based pagination. However, rate limiting is only conceptually addressed without implementation, the resolver code is cut off/incomplete, and input validation details are minimal." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.8, "brief_justification": "The response provides a comprehensive and technically accurate design. It covers all requested aspects including schema, pagination, validation, and rate limiting. The inclusion of database schema and cursor logic adds significant depth. The only minor deduction is for the incomplete code block at the end, but the overall content is excellent." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8, "brief_justification": "Strong schema design with proper pagination and directives. Missing full resolver code for the trickiest queries and incomplete schema (subscriptions/mutations missing)." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 3.2, "brief_justification": "The response is incomplete, cutting off mid-definition of the Comment type. It fails to provide the requested resolver implementations, DataLoader setup, or subscription definitions, making it largely useless for the specific technical requirements asked." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 7, "brief_justification": "Schema is well-structured with directives and pagination, but incomplete (truncated input types) and missing resolver implementations for trickiest queries." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8, "brief_justification": "Schema is well-structured and addresses key requirements, but some resolver implementations are incomplete and validation directives are not fully integrated." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 9, "brief_justification": "Excellent comprehensive response covering all requirements with detailed implementations. Minor deduction for rate limiting implementation being only a directive definition without actual resolver enforcement logic." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.8, "brief_justification": "Excellent schema design covering all requirements. Resolvers are conceptual but accurate. Missing explicit resolver code for the trickiest parts (e.g., Feed pagination logic), but the structure is production-ready." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 9.35, "brief_justification": "The response provides a comprehensive and technically accurate design. It correctly implements the DataLoader pattern for N+1 issues, cursor-based pagination, and real-time subscriptions. The inclusion of rate limiting via directives and specific resolver logic for the 'tricky parts' demonstrates high depth and practical usefulness. The schema is well-structured and follows GraphQL best practices." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "minimax_m25", "respondent_name": "MiniMax M2.5", "weighted_score": 7.35, "brief_justification": "Strong schema design and pagination concepts, but missing explicit DataLoader implementation and the response is truncated." } ], "meta": { "source": "The Multivac (app.themultivac.com)", "methodology": "10x10 blind peer matrix evaluation", "criteria": "correctness, completeness, clarity, depth, usefulness", "self_judgments": "excluded from rankings", "license": "Open data — cite as: The Multivac (2026)" } }