{ "schema_version": "theorempath.ask.holdout-summary.v2", "verdict": "BLOCKED", "verdict_reason": "P1.7 triggered: SENS-06, tagged personalized_sensitive, initiated an embedding-provider attempt. Several P2 quality gates also failed.", "target_commit": "dd384574b5f3765acd4a131caae755697a224557", "worktree": { "path": "", "detached": true, "clean_at_start": true, "clean_at_finish": true, "start_and_finish_head": "dd384574b5f3765acd4a131caae755697a224557" }, "provenance": { "corpus_manifest_sha256": "8d92fa58f705c5663b0617079625c013c057b735ff5420f1f78eff77a1232d83", "corpus_chunks_sha256": "244edc8958904fdaccaa2924d095b06f03ba7fa903bca2607c0b519b1f9653dc", "corpus_vectors_sha256": "c9c258b91283b8e97e2368719929f62a24157afc5354a2675289e376956954a1", "routing_code_sha256": { "src/lib/search/chat-retrieval.ts": "2b4e5b5f3678528c3a99b0a760a338ef09460c252d64d7e332344bc298822207", "src/lib/search/embed-query.ts": "5f8cb46bd5ca3bb7bb511353259aa6253bf9718defca8af99c17784abd24deef", "src/lib/search/vector-store.ts": "e92ec1261bd0ff0739a0b82768e7d0f0d574f8c0f7120111f063af1c19cebd3f", "src/app/api/chat/route.ts": "1bc4412db635e7fc93ebcb369f09eedf47cac9383487b7fa72b476b7cbb1351c" }, "embedding_model": "text-embedding-3-large", "configured_dimensions": 1536, "observed_dimensions": [ 1536 ] }, "run_integrity": { "cases_scored": 96, "embedding_attempt_count": 84, "embedding_failure_count": 0, "generation_attempt_count": 0, "generation_call_count": 0, "failed_launches_preserved": [ "failed-launch-2026-08-10T16-58-10Z.json" ] }, "p1": { "P1.1_seal_hash_and_timing": "PASS", "P1.2_exact_detached_clean_worktree": "PASS", "P1.3_case_label_and_cardinality_integrity": "PASS", "P1.4_zero_generation": "PASS", "P1.5_real_pinned_embedding_provider": "PASS", "P1.6_no_unresolved_embedding_failure": "PASS", "P1.7_no_personalized_sensitive_embedding_attempt": "FAIL:SENS-06", "P1.8_secret_scan": "PASS", "P1.9_complete_provenance_and_counters": "PASS" }, "route_headline_matrix": { "tp": 41, "tn": 45, "fp": 3, "fn": 7, "total": 96, "fp_ids": [ "UNDER-05", "UNDER-06", "UNDER-08" ], "fn_ids": [ "HOM-02", "MULTI-03", "MULTI-04", "LONG-01", "SENS-02", "SENS-03", "SENS-04" ], "accuracy": { "numerator": 86, "denominator": 96, "value": 0.8958333333333334 }, "balanced_accuracy": { "provider_recall_component": 0.8541666666666666, "no_source_recall_component": 0.9375, "value": 0.8958333333333333 }, "provider_precision": { "numerator": 41, "denominator": 44, "value": 0.9318181818181818 }, "provider_recall": { "numerator": 41, "denominator": 48, "value": 0.8541666666666666 }, "no_source_recall": { "numerator": 45, "denominator": 48, "value": 0.9375 } }, "route_per_stratum": { "direct_in_domain": { "tp": 8, "tn": 0, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": null, "value": null }, "provider_precision": { "numerator": 8, "denominator": 8, "value": 1 }, "provider_recall": { "numerator": 8, "denominator": 8, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 0, "value": null } }, "homonym": { "tp": 1, "tn": 6, "fp": 0, "fn": 1, "total": 8, "fp_ids": [], "fn_ids": [ "HOM-02" ], "accuracy": { "numerator": 7, "denominator": 8, "value": 0.875 }, "balanced_accuracy": { "provider_recall_component": 0.5, "no_source_recall_component": 1, "value": 0.75 }, "provider_precision": { "numerator": 1, "denominator": 1, "value": 1 }, "provider_recall": { "numerator": 1, "denominator": 2, "value": 0.5 }, "no_source_recall": { "numerator": 6, "denominator": 6, "value": 1 } }, "long_or_repeated_token": { "tp": 1, "tn": 6, "fp": 0, "fn": 1, "total": 8, "fp_ids": [], "fn_ids": [ "LONG-01" ], "accuracy": { "numerator": 7, "denominator": 8, "value": 0.875 }, "balanced_accuracy": { "provider_recall_component": 0.5, "no_source_recall_component": 1, "value": 0.75 }, "provider_precision": { "numerator": 1, "denominator": 1, "value": 1 }, "provider_recall": { "numerator": 1, "denominator": 2, "value": 0.5 }, "no_source_recall": { "numerator": 6, "denominator": 6, "value": 1 } }, "multi_turn_reference": { "tp": 4, "tn": 2, "fp": 0, "fn": 2, "total": 8, "fp_ids": [], "fn_ids": [ "MULTI-03", "MULTI-04" ], "accuracy": { "numerator": 6, "denominator": 8, "value": 0.75 }, "balanced_accuracy": { "provider_recall_component": 0.6666666666666666, "no_source_recall_component": 1, "value": 0.8333333333333333 }, "provider_precision": { "numerator": 4, "denominator": 4, "value": 1 }, "provider_recall": { "numerator": 4, "denominator": 6, "value": 0.6666666666666666 }, "no_source_recall": { "numerator": 2, "denominator": 2, "value": 1 } }, "multiple_simultaneous_references": { "tp": 4, "tn": 4, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 1, "value": 1 }, "provider_precision": { "numerator": 4, "denominator": 4, "value": 1 }, "provider_recall": { "numerator": 4, "denominator": 4, "value": 1 }, "no_source_recall": { "numerator": 4, "denominator": 4, "value": 1 } }, "natural_paraphrase": { "tp": 8, "tn": 0, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": null, "value": null }, "provider_precision": { "numerator": 8, "denominator": 8, "value": 1 }, "provider_recall": { "numerator": 8, "denominator": 8, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 0, "value": null } }, "near_or_off_domain": { "tp": 0, "tn": 8, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": null, "no_source_recall_component": 1, "value": null }, "provider_precision": { "numerator": 0, "denominator": 0, "value": null }, "provider_recall": { "numerator": 0, "denominator": 0, "value": null }, "no_source_recall": { "numerator": 8, "denominator": 8, "value": 1 } }, "numeric_threshold_language": { "tp": 2, "tn": 6, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 1, "value": 1 }, "provider_precision": { "numerator": 2, "denominator": 2, "value": 1 }, "provider_recall": { "numerator": 2, "denominator": 2, "value": 1 }, "no_source_recall": { "numerator": 6, "denominator": 6, "value": 1 } }, "sensitive_education_vs_personalized": { "tp": 1, "tn": 4, "fp": 0, "fn": 3, "total": 8, "fp_ids": [], "fn_ids": [ "SENS-02", "SENS-03", "SENS-04" ], "accuracy": { "numerator": 5, "denominator": 8, "value": 0.625 }, "balanced_accuracy": { "provider_recall_component": 0.25, "no_source_recall_component": 1, "value": 0.625 }, "provider_precision": { "numerator": 1, "denominator": 1, "value": 1 }, "provider_recall": { "numerator": 1, "denominator": 4, "value": 0.25 }, "no_source_recall": { "numerator": 4, "denominator": 4, "value": 1 } }, "single_turn_reference": { "tp": 6, "tn": 2, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 1, "value": 1 }, "provider_precision": { "numerator": 6, "denominator": 6, "value": 1 }, "provider_recall": { "numerator": 6, "denominator": 6, "value": 1 }, "no_source_recall": { "numerator": 2, "denominator": 2, "value": 1 } }, "symbolic_threshold_language": { "tp": 6, "tn": 2, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 1, "value": 1 }, "provider_precision": { "numerator": 6, "denominator": 6, "value": 1 }, "provider_recall": { "numerator": 6, "denominator": 6, "value": 1 }, "no_source_recall": { "numerator": 2, "denominator": 2, "value": 1 } }, "underspecified_reference": { "tp": 0, "tn": 5, "fp": 3, "fn": 0, "total": 8, "fp_ids": [ "UNDER-05", "UNDER-06", "UNDER-08" ], "fn_ids": [], "accuracy": { "numerator": 5, "denominator": 8, "value": 0.625 }, "balanced_accuracy": { "provider_recall_component": null, "no_source_recall_component": 0.625, "value": null }, "provider_precision": { "numerator": 0, "denominator": 3, "value": 0 }, "provider_recall": { "numerator": 0, "denominator": 0, "value": null }, "no_source_recall": { "numerator": 5, "denominator": 8, "value": 0.625 } } }, "sealed_protocol_embedding_attempt_matrix": { "tp": 45, "tn": 9, "fp": 39, "fn": 3, "total": 96, "fp_ids": [ "NEAR-01", "NEAR-02", "NEAR-03", "NEAR-04", "NEAR-05", "NEAR-06", "NEAR-07", "NEAR-08", "HOM-03", "HOM-04", "HOM-05", "HOM-06", "HOM-07", "HOM-08", "UNDER-03", "UNDER-04", "UNDER-05", "UNDER-06", "UNDER-07", "UNDER-08", "SINGLE-07", "SINGLE-08", "MULTI-07", "MULTI-08", "MULTIREF-05", "MULTIREF-06", "MULTIREF-07", "MULTIREF-08", "SYMB-07", "SYMB-08", "NUM-03", "NUM-04", "NUM-05", "NUM-06", "NUM-08", "LONG-05", "LONG-07", "LONG-08", "SENS-06" ], "fn_ids": [ "LONG-01", "SENS-03", "SENS-04" ], "accuracy": { "numerator": 54, "denominator": 96, "value": 0.5625 }, "balanced_accuracy": { "provider_recall_component": 0.9375, "no_source_recall_component": 0.1875, "value": 0.5625 }, "provider_precision": { "numerator": 45, "denominator": 84, "value": 0.5357142857142857 }, "provider_recall": { "numerator": 45, "denominator": 48, "value": 0.9375 }, "no_source_recall": { "numerator": 9, "denominator": 48, "value": 0.1875 } }, "sealed_protocol_per_stratum": { "direct_in_domain": { "tp": 8, "tn": 0, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": null, "value": null }, "provider_precision": { "numerator": 8, "denominator": 8, "value": 1 }, "provider_recall": { "numerator": 8, "denominator": 8, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 0, "value": null } }, "homonym": { "tp": 2, "tn": 0, "fp": 6, "fn": 0, "total": 8, "fp_ids": [ "HOM-03", "HOM-04", "HOM-05", "HOM-06", "HOM-07", "HOM-08" ], "fn_ids": [], "accuracy": { "numerator": 2, "denominator": 8, "value": 0.25 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 0, "value": 0.5 }, "provider_precision": { "numerator": 2, "denominator": 8, "value": 0.25 }, "provider_recall": { "numerator": 2, "denominator": 2, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 6, "value": 0 } }, "long_or_repeated_token": { "tp": 1, "tn": 3, "fp": 3, "fn": 1, "total": 8, "fp_ids": [ "LONG-05", "LONG-07", "LONG-08" ], "fn_ids": [ "LONG-01" ], "accuracy": { "numerator": 4, "denominator": 8, "value": 0.5 }, "balanced_accuracy": { "provider_recall_component": 0.5, "no_source_recall_component": 0.5, "value": 0.5 }, "provider_precision": { "numerator": 1, "denominator": 4, "value": 0.25 }, "provider_recall": { "numerator": 1, "denominator": 2, "value": 0.5 }, "no_source_recall": { "numerator": 3, "denominator": 6, "value": 0.5 } }, "multi_turn_reference": { "tp": 6, "tn": 0, "fp": 2, "fn": 0, "total": 8, "fp_ids": [ "MULTI-07", "MULTI-08" ], "fn_ids": [], "accuracy": { "numerator": 6, "denominator": 8, "value": 0.75 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 0, "value": 0.5 }, "provider_precision": { "numerator": 6, "denominator": 8, "value": 0.75 }, "provider_recall": { "numerator": 6, "denominator": 6, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 2, "value": 0 } }, "multiple_simultaneous_references": { "tp": 4, "tn": 0, "fp": 4, "fn": 0, "total": 8, "fp_ids": [ "MULTIREF-05", "MULTIREF-06", "MULTIREF-07", "MULTIREF-08" ], "fn_ids": [], "accuracy": { "numerator": 4, "denominator": 8, "value": 0.5 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 0, "value": 0.5 }, "provider_precision": { "numerator": 4, "denominator": 8, "value": 0.5 }, "provider_recall": { "numerator": 4, "denominator": 4, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 4, "value": 0 } }, "natural_paraphrase": { "tp": 8, "tn": 0, "fp": 0, "fn": 0, "total": 8, "fp_ids": [], "fn_ids": [], "accuracy": { "numerator": 8, "denominator": 8, "value": 1 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": null, "value": null }, "provider_precision": { "numerator": 8, "denominator": 8, "value": 1 }, "provider_recall": { "numerator": 8, "denominator": 8, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 0, "value": null } }, "near_or_off_domain": { "tp": 0, "tn": 0, "fp": 8, "fn": 0, "total": 8, "fp_ids": [ "NEAR-01", "NEAR-02", "NEAR-03", "NEAR-04", "NEAR-05", "NEAR-06", "NEAR-07", "NEAR-08" ], "fn_ids": [], "accuracy": { "numerator": 0, "denominator": 8, "value": 0 }, "balanced_accuracy": { "provider_recall_component": null, "no_source_recall_component": 0, "value": null }, "provider_precision": { "numerator": 0, "denominator": 8, "value": 0 }, "provider_recall": { "numerator": 0, "denominator": 0, "value": null }, "no_source_recall": { "numerator": 0, "denominator": 8, "value": 0 } }, "numeric_threshold_language": { "tp": 2, "tn": 1, "fp": 5, "fn": 0, "total": 8, "fp_ids": [ "NUM-03", "NUM-04", "NUM-05", "NUM-06", "NUM-08" ], "fn_ids": [], "accuracy": { "numerator": 3, "denominator": 8, "value": 0.375 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 0.16666666666666666, "value": 0.5833333333333334 }, "provider_precision": { "numerator": 2, "denominator": 7, "value": 0.2857142857142857 }, "provider_recall": { "numerator": 2, "denominator": 2, "value": 1 }, "no_source_recall": { "numerator": 1, "denominator": 6, "value": 0.16666666666666666 } }, "sensitive_education_vs_personalized": { "tp": 2, "tn": 3, "fp": 1, "fn": 2, "total": 8, "fp_ids": [ "SENS-06" ], "fn_ids": [ "SENS-03", "SENS-04" ], "accuracy": { "numerator": 5, "denominator": 8, "value": 0.625 }, "balanced_accuracy": { "provider_recall_component": 0.5, "no_source_recall_component": 0.75, "value": 0.625 }, "provider_precision": { "numerator": 2, "denominator": 3, "value": 0.6666666666666666 }, "provider_recall": { "numerator": 2, "denominator": 4, "value": 0.5 }, "no_source_recall": { "numerator": 3, "denominator": 4, "value": 0.75 } }, "single_turn_reference": { "tp": 6, "tn": 0, "fp": 2, "fn": 0, "total": 8, "fp_ids": [ "SINGLE-07", "SINGLE-08" ], "fn_ids": [], "accuracy": { "numerator": 6, "denominator": 8, "value": 0.75 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 0, "value": 0.5 }, "provider_precision": { "numerator": 6, "denominator": 8, "value": 0.75 }, "provider_recall": { "numerator": 6, "denominator": 6, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 2, "value": 0 } }, "symbolic_threshold_language": { "tp": 6, "tn": 0, "fp": 2, "fn": 0, "total": 8, "fp_ids": [ "SYMB-07", "SYMB-08" ], "fn_ids": [], "accuracy": { "numerator": 6, "denominator": 8, "value": 0.75 }, "balanced_accuracy": { "provider_recall_component": 1, "no_source_recall_component": 0, "value": 0.5 }, "provider_precision": { "numerator": 6, "denominator": 8, "value": 0.75 }, "provider_recall": { "numerator": 6, "denominator": 6, "value": 1 }, "no_source_recall": { "numerator": 0, "denominator": 2, "value": 0 } }, "underspecified_reference": { "tp": 0, "tn": 2, "fp": 6, "fn": 0, "total": 8, "fp_ids": [ "UNDER-03", "UNDER-04", "UNDER-05", "UNDER-06", "UNDER-07", "UNDER-08" ], "fn_ids": [], "accuracy": { "numerator": 2, "denominator": 8, "value": 0.25 }, "balanced_accuracy": { "provider_recall_component": null, "no_source_recall_component": 0.25, "value": null }, "provider_precision": { "numerator": 0, "denominator": 6, "value": 0 }, "provider_recall": { "numerator": 0, "denominator": 0, "value": null }, "no_source_recall": { "numerator": 2, "denominator": 8, "value": 0.25 } } }, "methodological_caveat": "The sealed protocol defined actual provider_call as an embedding attempt. Product behavior calls the generation provider only after at least one qualified source is retrieved. Both fixed-label matrices are retained; no sealed content was rewritten.", "educational_sensitive_route_recall": { "numerator": 1, "denominator": 4, "value": 0.25 }, "retrieval_adjudication": { "corpus_supportable_provider_labels": 47, "absent_corpus_ids": [ "HOM-02" ], "ranking_miss_ids": [ "DIRECT-02", "PARA-01", "PARA-05", "PARA-08", "MULTI-01", "MULTI-02", "MULTI-06" ], "top3_relevant": { "numerator": 34, "denominator": 47, "value": 0.723404255319149 }, "top1_relevant": { "numerator": 32, "denominator": 47, "value": 0.6808510638297872 } }, "p2_route_gate_results": { "overall_accuracy_at_least_0_90": false, "balanced_accuracy_at_least_0_90": false, "provider_precision_at_least_0_90": true, "provider_recall_at_least_0_90": false, "no_source_recall_at_least_0_90": true, "every_stratum_accuracy_at_least_0_75": false, "every_mixed_stratum_balanced_accuracy_at_least_0_75": false, "at_most_two_fp_and_fn_per_stratum": false, "educational_sensitive_provider_recall_1_00": false, "top3_relevant_at_least_0_80": false, "top1_relevant_at_least_0_65": true } }