{ "evaluation_id": "EVAL-20260402-135609", "question_id": "CODE-019", "question_text": "Implement a Bloom filter from scratch (no libraries) with the following: configurable false positive rate, optimal hash function count calculation, serialization/deserialization, a counting variant that supports deletion, and memory usage statistics. Include mathematical proof of your false positive rate formula.", "category": "code", "timestamp": "2026-04-02T13:56:09.641Z", "display_date": "Apr 02, 2026", "winner": { "name": "Grok 4.20", "provider": "openrouter", "score": 8.68 }, "avg_score": 6.8133335, "matrix_size": 80, "models_used": [ { "id": "gemini_3_flash", "name": "Gemini 3 Flash Preview", "provider": "Google" }, { "id": "grok_420", "name": "Grok 4.20", "provider": "openrouter" }, { "id": "deepseek_v4", "name": "DeepSeek V4", "provider": "openrouter" }, { "id": "claude_opus_46", "name": "Claude Opus 4.6", "provider": "openrouter" }, { "id": "gemini_31_pro", "name": "Gemini 3.1 Pro", "provider": "openrouter" }, { "id": "claude_sonnet_46", "name": "Claude Sonnet 4.6", "provider": "openrouter" }, { "id": "gpt_oss_120b", "name": "GPT-OSS-120B", "provider": "OpenAI" }, { "id": "mimo_v2_flash", "name": "MiMo-V2-Flash", "provider": "Xiaomi" }, { "id": "gpt_5_4", "name": "GPT-5.4", "provider": "openrouter" } ], "rankings": { "grok_420": { "display_name": "Grok 4.20", "provider": "openrouter", "average_score": 8.68, "score_count": 9, "min_score": 7.2, "max_score": 9.8, "rank": 1 }, "gemini_3_flash": { "display_name": "Gemini 3 Flash Preview", "provider": "Google", "average_score": 8.03, "score_count": 9, "min_score": 6.75, "max_score": 8.8, "rank": 2 }, "gpt_5_4": { "display_name": "GPT-5.4", "provider": "openrouter", "average_score": 7.56, "score_count": 9, "min_score": 5.8, "max_score": 8.85, "rank": 3 }, "gpt_oss_120b": { "display_name": "GPT-OSS-120B", "provider": "OpenAI", "average_score": 7.42, "score_count": 9, "min_score": 4.05, "max_score": 9.45, "rank": 4 }, "claude_opus_46": { "display_name": "Claude Opus 4.6", "provider": "openrouter", "average_score": 7.31, "score_count": 9, "min_score": 3.9, "max_score": 9.6, "rank": 5 }, "mimo_v2_flash": { "display_name": "MiMo-V2-Flash", "provider": "Xiaomi", "average_score": 7.16, "score_count": 9, "min_score": 4.2, "max_score": 9.45, "rank": 6 }, "claude_sonnet_46": { "display_name": "Claude Sonnet 4.6", "provider": "openrouter", "average_score": 6.98, "score_count": 9, "min_score": 4.6, "max_score": 8.6, "rank": 7 }, "deepseek_v4": { "display_name": "DeepSeek V4", "provider": "openrouter", "average_score": 5.56, "score_count": 9, "min_score": 4.15, "max_score": 6.6, "rank": 8 }, "gemini_31_pro": { "display_name": "Gemini 3.1 Pro", "provider": "openrouter", "average_score": 2.63, "score_count": 8, "min_score": 0.65, "max_score": 5.85, "rank": 9 } }, "judgments": [ { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.6, "brief_justification": "Accurate mathematical proof and implementation; covers all requested features with clear structure and practical code." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9, "brief_justification": "The answer delivers a functional, well-documented Python implementation covering all required features, includes a clear mathematical proof, and is practically useful, with minor minor nuances in hashing and memory calculations." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.3, "brief_justification": "Implementation covers most requested features but uses external libraries despite 'no libraries' requirement and omits the mathematical proof, reducing correctness and completeness; code is clear and fairly useful." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.8, "brief_justification": "Accurate implementation with correct formulas, covers all requested features, clearly presented, offers solid analysis and practical code, minor minor omission in deserialization metadata." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 3.9, "brief_justification": "The mathematical derivation is mostly standard but not fully rigorous, and the optimal-k proof is hand-wavy. More importantly, the implementation is truncated before the actual BloomFilter class methods and thus does not deliver the requested functionality such as serialization/deserialization, counting variant deletion support, or memory stats in usable form." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 0.65, "brief_justification": "The response is truncated almost immediately, provides no implementation, no proof, and does not satisfy any of the requested features beyond a brief introductory sentence." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 4.6, "brief_justification": "The response is clearly written and starts with the standard Bloom filter FPR derivation, but it is incomplete: the implementation is truncated and does not actually provide the BloomFilter/counting variant classes, serialization/deserialization logic, configurable FPR handling end-to-end, or memory usage statistics. The mathematical proof is only a sketch and omits a rigorous derivation of the optimal k step." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7.35, "brief_justification": "Covers all requested features with working code and a reasonable derivation, but the math is only approximate rather than a full proof, the optimal-k statement/formula is inconsistently presented, serialization uses lossy float packing, and counting-filter deletion/memory implementation has practical inaccuracies." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 4.15, "brief_justification": "Readable structure and includes most requested feature headings, but violates the no-libraries requirement, leaves the mathematical proof incomplete, and has questionable design choices such as capacity enforcement and inaccurate memory accounting." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 4.05, "brief_justification": "The response explains Bloom filter theory reasonably well, but the implementation is incomplete and cuts off mid-code, so required features like serialization, counting deletion support, and memory stats are not actually delivered. Math discussion is decent but not a full proof, and the partial code limits practical usefulness." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 6.75, "brief_justification": "Clear and mostly accurate, with the standard Bloom filter formulas and a working sketch. However, it does not truly implement a separate standard and counting variant, uses inefficient Python lists despite claiming bit manipulation, deserialization fails to restore capacity/fp_rate consistently, and the false-positive derivation is more of a standard derivation than a full proof." }, { "judge_key": "gpt_5_4", "judge_name": "GPT-5.4", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 4.2, "brief_justification": "The response gives a mostly correct false-positive derivation and reasonable formulas for m and k, but the implementation is incomplete/truncated, so serialization for the counting variant and likely deserialization/stats are unfinished. It also uses floor via int() instead of a more appropriate rounding/ceiling for optimal parameter sizing, and memory statistics are only approximate/logical rather than actual usage. Clear structure and some depth are present, but the incomplete code significant" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 6.55, "brief_justification": "The mathematical proof/derivation of the false positive rate formula is thorough and correct. The standard Bloom filter implementation shown is well-structured with good design choices (double hashing, proper optimal parameter calculations). However, the response is critically incomplete - the code is cut off mid-serialization method, and the counting Bloom filter variant with deletion support is entirely missing. The deserialization method is also missing. Since the question explicitly required" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 1.2, "brief_justification": "The response is almost entirely incomplete. It begins with a promising outline mentioning mathematical proof and code, but cuts off after barely starting the mathematical notation (just defining n). No actual implementation, no proof, no code, no serialization/deserialization, no counting variant, no memory statistics - essentially nothing of substance was delivered." }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 7.5, "brief_justification": "The mathematical proof of the false positive rate is correct and well-presented, covering the derivation of optimal k and m. The implementation includes well-crafted hash functions (MurmurHash3, FNV-1a), double hashing with Kirsch-Mitzenmacher reference, and a clean BitArray class. However, the response is clearly incomplete - it cuts off mid-implementation without showing the actual BloomFilter class, the counting variant supporting deletion, serialization/deserialization, or memory usage stati" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 7.2, "brief_justification": "The implementation covers all requested features: configurable FP rate, optimal hash count calculation, serialization/deserialization, counting variant with deletion, memory statistics, and a mathematical proof. However, there are several issues: (1) The hashing approach computes two full SHA-256 hashes per hash function call, making it very slow - true double hashing should compute h1 and h2 once and combine them. (2) The serialization uses 'I' (unsigned 32-bit int) which limits capacity/m to ~" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 4.45, "brief_justification": "The implementation uses external libraries (bitarray, mmh3) despite the requirement to implement 'from scratch (no libraries)'. The mathematical proof of the false positive rate formula is incomplete - it cuts off after 'The false positive rate'. The code structure is reasonable but has issues: raising ValueError when capacity is exceeded is unusual for a Bloom filter (they degrade gracefully), the memory_usage calculation for CountingBloomFilter is unnecessarily complex and inaccurate, and the " }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 6.55, "brief_justification": "The response provides excellent theoretical coverage with a proper mathematical proof of the false positive rate formula, optimal k and m derivations, and well-structured design documentation. The code architecture is sound with good design choices (double hashing, bytearray storage, serialization format). However, the implementation is critically incomplete - the code is cut off mid-assignment in the BloomFilter constructor, and the CountingBloomFilter class, serialization/deserialization metho" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 7.4, "brief_justification": "The mathematical proof is correct and well-presented, covering the derivation of optimal k and m. The implementation covers all requested features: configurable FP rate, optimal hash count, serialization/deserialization, counting variant with deletion, and memory stats. However, there are notable issues: (1) using a Python list of ints for the bit array is extremely memory-inefficient (each Python int object is ~28 bytes, not 1 bit), making the memory stats misleading; (2) the deserialization cr" }, { "judge_key": "claude_opus_46", "judge_name": "Claude Opus 4.6", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.85, "brief_justification": "The mathematical proof of the false positive rate is well-presented and correct. The implementation covers the standard Bloom filter, counting variant with deletion, serialization/deserialization, and memory stats. However, the code is truncated - the CountingBloomFilter's serialize method is cut off, and its deserialize method is missing entirely. The serialization of the error rate as an integer (multiplied by 100000) is a fragile approach that loses precision. The standard Bloom filter uses a" }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 5.95, "brief_justification": "The mathematical proof and initial implementation are excellent, but the response cuts off abruptly, missing the deserialization logic and the entire counting variant with deletion." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 5.8, "brief_justification": "The response provides an excellent mathematical proof and starts a high-quality implementation, but it is severely truncated and missing the actual Bloom filter logic, counting variant, and other requested features." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.05, "brief_justification": "The response provides an excellent mathematical proof and robust foundational components, but cuts off abruptly, missing the actual Bloom Filter implementation, counting variant, serialization, and memory statistics." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9.4, "brief_justification": "The implementation is highly accurate, complete, and well-structured. It correctly implements all requested features, including a solid mathematical proof and efficient double hashing, though the counting variant's memory stats reflect theoretical rather than actual Python list memory." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 4.2, "brief_justification": "The response violates the 'no libraries' constraint by using third-party packages (bitarray, mmh3) and is abruptly cut off, completely missing the requested mathematical proof." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 5.7, "brief_justification": "The response provides accurate mathematical proofs and clear explanations, but the code cuts off abruptly mid-implementation, omitting the counting variant, serialization, and memory statistics, rendering the solution unusable." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.4, "brief_justification": "The response covers all requirements, including the math proof and counting variant. However, it claims to use bit manipulation but actually uses a standard Python list of integers for the bit array." }, { "judge_key": "gemini_31_pro", "judge_name": "Gemini 3.1 Pro", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.4, "brief_justification": "The response provides a solid mathematical proof and well-structured code covering most requirements, but it is severely truncated at the end, resulting in incomplete and syntactically invalid code." }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 7.85, "brief_justification": "The response is mathematically rigorous with a clear derivation of the false positive rate formula and optimal hash count. The implementation is well-structured with double hashing, memory stats, and good code quality. However, the response is cut off mid-serialization, missing the complete serialize/deserialize methods and the counting Bloom filter variant entirely. The mathematical proofs are solid but the derivation of the optimal k could be more rigorous (the differentiation step is glossed " }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.3, "brief_justification": "The mathematical proof is rigorous and well-structured, the BitArray and hasher implementations are correct and well-explained. However, the response appears truncated—the BloomFilter class itself is cut off mid-docstring, and the counting variant, serialization/deserialization, and memory statistics sections are missing entirely. The foundations shown are high quality, but the incomplete delivery significantly impacts completeness and usefulness scores." }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 1.2, "brief_justification": "The response is severely incomplete - it cuts off mid-sentence during the mathematical proof section, providing no actual implementation, no complete proof, no serialization, no memory statistics, and none of the requested features. Only the introduction and beginning of the proof were started before the response was truncated." }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.55, "brief_justification": "Strong implementation covering all requested features: configurable FPR, optimal hash count, serialization/deserialization, counting variant with deletion, and memory stats. The mathematical proof is clear and correct. Minor issues: the struct format uses 'f' (float) for p which loses precision (should use 'd' for double), the double hashing scheme recomputes SHA-256 twice per hash which is inefficient, and the 4-bit counter memory calculation has a bug (bytes_used variable is unused). The proof" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 5.8, "brief_justification": "The implementation uses external libraries (bitarray, mmh3) despite the 'no libraries' requirement, which is a significant violation. The mathematical proof is incomplete (cut off). The counting variant and serialization are reasonably implemented. The hash function approach using mmh3 with seeds is practical but violates constraints. Memory usage calculation has issues (bit_array_size // 8 doesn't account for ceiling division properly). The remove() method has a logical flaw - checking containm" }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7, "brief_justification": "Provides a solid proof and well-structured code for standard and counting Bloom filters, but the counting variant's serialization is incomplete and some implementation details are simplistic, reducing overall correctness and completeness." }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 7.05, "brief_justification": "The response demonstrates strong theoretical grounding with a clear mathematical proof of the false positive rate formula, well-explained optimal k and m derivations, and good implementation details. The double hashing scheme, serialization format, and counting variant are all well-described. However, the code is incomplete - it gets cut off mid-implementation, missing the actual BloomFilter methods (add, contains, stats, serialize/deserialize), the CountingBloomFilter class entirely, and the me" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 7.75, "brief_justification": "The implementation is well-structured and covers all requested features: configurable FP rate, optimal k/m calculation, serialization/deserialization, counting variant with deletion, and memory stats. The mathematical proof is correct and clearly presented. However, there are some issues: the memory stats are misleading (Python lists use ~28 bytes per int, not 1 bit or 4 bytes as claimed), the deserialization creates a dummy instance with hardcoded values before overwriting, and the actual memor" }, { "judge_key": "claude_sonnet_46", "judge_name": "Claude Sonnet 4.6", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 6.85, "brief_justification": "The mathematical proof is solid and well-explained. The implementation covers most requirements: standard Bloom filter, counting variant with deletion, serialization/deserialization, and stats. However, the code is truncated mid-serialization for CountingBloomFilter, leaving it incomplete. The bit array uses a Python list of ints rather than a proper bitarray, which is memory-inefficient. Memory stats are somewhat superficial (logical bits only, not actual Python object overhead). The double has" }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.1, "brief_justification": "The response provides a strong mathematical derivation and proof for the false positive rate, optimal k and m formulas, and a solid standard Bloom filter implementation with most requested features. However, the code is incomplete (truncated at the serialize method), the Counting Bloom Filter variant for deletions is entirely missing despite being explicitly required, and serialization/deserialization is only partially shown." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.65, "brief_justification": "Excellent mathematical proof and solid core implementation with good hash functions and bit array. Code is incomplete (class cuts off) and counting variant, serialization, optimal k calculation, and memory stats are mentioned in spec but not fully implemented in the provided snippet." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 3.25, "brief_justification": "The response is incomplete: it cuts off immediately after starting the math proof ('Let: * n =') and provides no actual code, no implementation of any requested features (standard Bloom filter, counting variant, serialization, optimal k calculation, memory stats), and no finished proof. While the introduction is clear, the content is essentially empty." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 7.9, "brief_justification": "Excellent mathematical proof and clean, well-structured code with good hash functions and bit array. However, the response is incomplete: the main BloomFilter and CountingBloomFilter classes are missing, as are serialization, memory stats, and full integration despite being listed in the requirements." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.05, "brief_justification": "Implementation uses external libraries (bitarray, mmh3, pickle) violating 'no libraries' rule; counting variant is incomplete (lacks proper _calculate_size override and current_false_positive_rate); proof is truncated. Core math formulas are correct but code has issues." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.65, "brief_justification": "The response provides a comprehensive, mathematically grounded implementation covering all requested features. Minor code truncation at the end slightly impacts completeness, but theory, formulas, and structure are excellent." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 7.8, "brief_justification": "The response provides a solid, well-structured implementation with correct math derivations and all requested features. Minor issues include using lists (not bits) for bit_array, a flawed deserialize method that ignores passed parameters, and a slightly optimistic memory estimate, but overall it is accurate and comprehensive." }, { "judge_key": "grok_420", "judge_name": "Grok 4.20", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.2, "brief_justification": "The response provides a solid mathematical derivation and a mostly complete Python implementation covering all requested features. However, the code is truncated mid-method in the CountingBloomFilter.serialize, contains minor inaccuracies in the get_stats memory calculation (uses bits/8 but stores as list of ints), and the deserialization logic has a small bug where it calls cls(cap, err) which recalculates m/k before overriding. The proof is mathematically sound." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.8, "brief_justification": "Comprehensive implementation with detailed mathematical proof and practical features." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.45, "brief_justification": "Detailed mathematical proof and implementation, but missing full implementation details." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 5.85, "brief_justification": "The response starts with a promise of a complete implementation and mathematical proof but cuts off abruptly, lacking the promised content. The partial content shown is clear but incomplete." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 8.6, "brief_justification": "Detailed implementation with mathematical proofs and practical features." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9.45, "brief_justification": "Comprehensive implementation with all requested features, clear documentation, and mathematical proof." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 8.8, "brief_justification": "Comprehensive and accurate with detailed implementation and theory." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8.8, "brief_justification": "Comprehensive and accurate with detailed mathematical proof and practical implementation." }, { "judge_key": "deepseek_v4", "judge_name": "DeepSeek V4", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 9.45, "brief_justification": "Comprehensive implementation with mathematical proof, serialization, and counting variant." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 5.8, "brief_justification": "Provides correct core Bloom filter logic, formulas, and stats, but truncates serialization code and lacks a counting/deletion variant, making it incomplete for the full request." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 5.95, "brief_justification": "Proof and core bit array code are correct and clear, but the implementation is incomplete, lacking full Bloom filter methods, counting variant, serialization, and memory stats, reducing overall completeness and usefulness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 3.4, "brief_justification": "The response starts a proof and mentions code but is incomplete, lacking full implementation, serialization, memory stats, and a complete mathematical derivation, limiting its correctness and usefulness." }, { "judge_key": "gpt_oss_120b", "judge_name": "GPT-OSS-120B", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 5.2, "brief_justification": "The response offers a clear proof and solid low-level components, but stops short of delivering a full Bloom filter implementation, counting variant, serialization, and memory stats, making it incomplete for the asked requirements." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.85, "brief_justification": "The response provides an excellent mathematical proof and a high-quality implementation of a standard Bloom Filter. However, the code for the 'Counting Bloom Filter' variant was cut off due to output length limits, and the serialization method is incomplete." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 9.6, "brief_justification": "The response provides a mathematically rigorous proof, a high-quality implementation of both standard and counting Bloom filters, and covers all requested features including serialization and memory stats. The use of enhanced double hashing (Kirsch-Mitzenmacher) shows deep technical knowledge." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 2.65, "brief_justification": "The response is incomplete. It cuts off immediately after defining the first variable in the mathematical proof and provides no code implementation, serialization, or memory statistics as requested." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 8.55, "brief_justification": "The mathematical proof is rigorous and the implementation of the hash functions and BitArray is high quality. However, the response is incomplete as it cuts off before implementing the actual BloomFilter and CountingBloomFilter classes, which were specifically requested." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 9.8, "brief_justification": "The response provides a high-quality, complete implementation of both a standard and a counting Bloom filter. It includes the required mathematical proof, serialization logic, and memory statistics. The code is well-structured and follows best practices for the requested constraints." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.35, "brief_justification": "The response failed to provide the requested mathematical proof (it ends abruptly). It also used external libraries (bitarray, mmh3) despite the 'no libraries' constraint. The code is logically sound but incomplete relative to the prompt." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 9.45, "brief_justification": "The response provides a high-quality implementation of both a standard and counting Bloom filter. It includes the mathematical proofs requested, uses an efficient double-hashing scheme to avoid multiple expensive hash calls, and implements serialization. The code is well-structured, though the snippet provided was cut off at the very end of the class definition." }, { "judge_key": "gemini_3_flash", "judge_name": "Gemini 3 Flash Preview", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 8.6, "brief_justification": "The response provides a high-quality implementation and a clear mathematical proof. It correctly derives the false positive rate and implements both standard and counting variants. The code is well-structured, though the serialization for the CountingBloomFilter is cut off at the very end." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 7.55, "brief_justification": "Mathematical proofs are accurate and well-presented. Standard Bloom filter implementation is correct with proper double hashing. However, response is truncated mid-serialization method and completely missing the required counting Bloom filter variant with deletion support." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 6.5, "brief_justification": "Thorough mathematical proof with correct derivations. Well-structured code with BitArray and hasher classes. However, response is truncated (ends mid-class definition) and critically missing the counting Bloom filter variant that supports deletion, which was explicitly requested. Serialization and memory stats are partially implemented." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 6.8, "brief_justification": "Mathematical proof is correct and well-presented. Hash functions and BitArray are properly implemented. However, response is truncated - main BloomFilter class, CountingBloomFilter, and full serialization/memory stats are missing. Cannot be fully used as intended." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.75, "brief_justification": "Comprehensive implementation covering all requirements with correct mathematical proofs. Well-structured code with proper serialization. Minor issue in CountingBloomFilter memory calculation (uses 0.5 bytes incorrectly)." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.6, "brief_justification": "Correct implementation of Bloom filter with all requested features, but response is incomplete - mathematical proof section is missing/truncated. Uses external libraries (bitarray, mmh3, pickle) which may violate 'from scratch' requirement. Code structure is clear but the critical mathematical proof is absent." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 7.55, "brief_justification": "Mathematical proofs are correct and well-presented, with clear explanations of optimal m and k formulas. However, the code is incomplete - it cuts off mid-assignment with 'self._bits =' and never finishes the implementation. While the architectural details and theory are thorough, the actual working code is not provided." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "gemini_3_flash", "respondent_name": "Gemini 3 Flash Preview", "weighted_score": 8, "brief_justification": "Comprehensive implementation covering all requirements with correct mathematical proof. The counting variant with deletion support, serialization, and stats are all present. Minor design inefficiency (redundant bit_array and counts) but core functionality is sound." }, { "judge_key": "minimax_m25", "judge_name": "MiniMax M2.5", "respondent_key": "mimo_v2_flash", "respondent_name": "MiMo-V2-Flash", "weighted_score": 7.85, "brief_justification": "Thorough implementation with excellent mathematical proof and clear structure. However, the response is incomplete - the CountingBloomFilter serialization method is cut off and lacks a deserialize method. Minor precision issue with float serialization for error_rate." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gpt_5_4", "respondent_name": "GPT-5.4", "weighted_score": 8.6, "brief_justification": "The response provides a mathematically correct derivation of the false positive rate and optimal parameters. The implementation is robust, using double hashing for efficiency and including all requested features (configurability, serialization, counting variant, stats). The only minor deduction is for the abrupt cut-off in the provided code snippet, though the logic up to that point is sound." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "claude_opus_46", "respondent_name": "Claude Opus 4.6", "weighted_score": 8.65, "brief_justification": "Mathematical proof is rigorous and accurate, implementation covers all requested features, code is clear and well-structured, but memory usage statistics are only partially implemented." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gemini_31_pro", "respondent_name": "Gemini 3.1 Pro", "weighted_score": 2.8, "brief_justification": "Response is incomplete and cuts off mid-sentence, failing to provide the required implementation, proof, or any of the requested features." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "claude_sonnet_46", "respondent_name": "Claude Sonnet 4.6", "weighted_score": 7.65, "brief_justification": "Strong mathematical foundation and hash implementation, but missing the counting variant and serialization/deserialization methods requested." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "grok_420", "respondent_name": "Grok 4.20", "weighted_score": 8.6, "brief_justification": "Accurate implementation with correct formulas, covers all requested features, clear structure, includes mathematical proof, and provides practical usage." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "deepseek_v4", "respondent_name": "DeepSeek V4", "weighted_score": 6.15, "brief_justification": "Implementation has issues: uses external libraries (mmh3, bitarray) contrary to 'no libraries', incomplete mathematical proof, and counting filter memory calculation is flawed. However, structure is clear and features are mostly covered." }, { "judge_key": "mimo_v2_flash", "judge_name": "MiMo-V2-Flash", "respondent_key": "gpt_oss_120b", "respondent_name": "GPT-OSS-120B", "weighted_score": 9, "brief_justification": "Comprehensive implementation with all requested features, clear mathematical proof, and practical code structure." } ], "meta": { "source": "The Multivac (app.themultivac.com)", "methodology": "10x10 blind peer matrix evaluation", "criteria": "correctness, completeness, clarity, depth, usefulness", "self_judgments": "excluded from rankings", "license": "Open data — cite as: The Multivac (2026)" } }