{ "nbformat": 4, "nbformat_minor": 5, "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3.12.14" }, "execution_note": "Code cells executed with Python via an AST-based offline notebook runner; outputs are actual stdout and rich MIME return values. No Jupyter package dependency." }, "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Audit the evaluation before trusting the score\n", "\n", "This notebook reruns the attached offline analysis. Place all bundle files in one directory and run from that directory. The default path uses Python’s standard library only. Published measurements, metadata and illustrative numerical probes are identified in the article and provenance. No model or API calls occur.\n" ], "id": "cell-1" }, { "cell_type": "code", "metadata": {}, "source": [ "import sys, json, runpy\n", "from pathlib import Path\n", "if str(Path.cwd()) not in sys.path:\n", " sys.path.insert(0, str(Path.cwd()))\n", "analysis = runpy.run_path('analyze.py', run_name='offline_analysis')\n", "result = analysis['analyze']()\n", "analysis['draw'](result)\n", "print('All analysis assertions passed; figures and results regenerated.')\n", "print(json.dumps(result, indent=2))\n" ], "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "All analysis assertions passed; figures and results regenerated.\n", "{\n", " \"entries\": [\n", " {\n", " \"dataset\": \"intervention/aime.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"explicitly skipped\",\n", " \"deprecated\": false,\n", " \"skip_reason\": \"AIME problem text is not redistributed in OSS; rows expose `url` and `source` instead of `problem`.\",\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.answer_checking_autograder\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.hint_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/daft.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.answer_checking_autograder\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.hint_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/gpqa.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.answer_checking_autograder\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.hint_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/gpqa_encoded.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.answer_checking_autograder\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.encoded_hint_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/memory_advice.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"explicitly skipped\",\n", " \"deprecated\": true,\n", " \"skip_reason\": \"Memory retrieval evals depend on tool/runtime-specific memory APIs and are intentionally excluded from the mock OSS scaffold.\",\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.memory_advice_answer_grader\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.memory_advice_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/scruples_first_person.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.string_match_answer_grader\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.scruples_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/scruples_suggest_right.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.string_match_answer_grader\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.scruples_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/scruples_suggest_wrong.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.string_match_answer_grader\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.scruples_monitor\"\n", " },\n", " {\n", " \"dataset\": \"intervention/wmdp_sandbagging.jsonl\",\n", " \"archetype\": \"intervention\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.intervention_answer_graders.answer_checking_autograder\",\n", " \"z_rule\": \"shared_rules.intervention_monitors.sandbagging_monitor\"\n", " },\n", " {\n", " \"dataset\": \"process/process_aime_math.jsonl\",\n", " \"archetype\": \"process\",\n", " \"status\": \"explicitly skipped\",\n", " \"deprecated\": false,\n", " \"skip_reason\": \"AIME problem text is not redistributed in OSS; rows expose `url` and `source` instead of `problem`.\",\n", " \"y_rule\": \"shared_rules.process_labeling.answer_grader\",\n", " \"z_rule\": \"shared_rules.process_labeling.monitor\"\n", " },\n", " {\n", " \"dataset\": \"process/process_gpqa_physics.jsonl\",\n", " \"archetype\": \"process\",\n", " \"status\": \"metadata-eligible for mock scaffold\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"shared_rules.process_labeling.answer_grader\",\n", " \"z_rule\": \"shared_rules.process_labeling.monitor\"\n", " },\n", " {\n", " \"dataset\": \"outcome/flaky_tools_charxiv.jsonl\",\n", " \"archetype\": \"outcome\",\n", " \"status\": \"explicitly skipped\",\n", " \"deprecated\": false,\n", " \"skip_reason\": \"Flaky Tools evals depend on tool/runtime-specific tool availability and are intentionally excluded from the mock OSS scaffold.\",\n", " \"y_rule\": \"shared_rules.flaky_tools_outcome.answer_judge\",\n", " \"z_rule\": \"shared_rules.flaky_tools_outcome.monitor\"\n", " },\n", " {\n", " \"dataset\": \"outcome/livebench.md\",\n", " \"archetype\": \"outcome\",\n", " \"status\": \"description only\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"Outcome answer judging is dataset-specific (not included as JSONL in this repo).\",\n", " \"z_rule\": {\n", " \"valid_answers\": [\n", " \"A\",\n", " \"B\"\n", " ],\n", " \"z_positive_answers\": [\n", " \"A\"\n", " ]\n", " }\n", " },\n", " {\n", " \"dataset\": \"outcome/shadearena.md\",\n", " \"archetype\": \"outcome\",\n", " \"status\": \"description only\",\n", " \"deprecated\": false,\n", " \"skip_reason\": null,\n", " \"y_rule\": \"Side-task success label from task evaluation/judge (positive subset filtering applied per Appendix C).\",\n", " \"z_rule\": {\n", " \"valid_answers\": [\n", " \"A\",\n", " \"B\",\n", " \"C\",\n", " \"D\"\n", " ],\n", " \"z_positive_answers\": [\n", " \"C\"\n", " ],\n", " \"meaning_of_positive\": \"Harmful side-task materially pursued.\"\n", " }\n", " }\n", " ],\n", " \"entry_count\": 14,\n", " \"status_counts\": {\n", " \"explicitly skipped\": 4,\n", " \"metadata-eligible for mock scaffold\": 8,\n", " \"description only\": 2\n", " },\n", " \"archetype_counts\": {\n", " \"intervention\": 9,\n", " \"process\": 2,\n", " \"outcome\": 3\n", " },\n", " \"resolved_rule_reference_occurrences\": 24,\n", " \"unique_resolved_rule_references\": 12,\n", " \"verified_prompt_reference_occurrences\": 26,\n", " \"unique_verified_prompt_paths\": 15,\n", " \"mock_constants\": {\n", " \"MOCK_MODEL_P_CORRECT\": {\n", " \"intervention_control\": 0.35,\n", " \"intervention_treatment\": 0.75,\n", " \"process\": 0.65\n", " },\n", " \"MOCK_MONITOR_SCOPE_QUALITY\": {\n", " \"cot_only\": {\n", " \"tpr\": 0.82,\n", " \"tnr\": 0.78\n", " },\n", " \"answer_only\": {\n", " \"tpr\": 0.6,\n", " \"tnr\": 0.85\n", " },\n", " \"all_messages\": {\n", " \"tpr\": 0.88,\n", " \"tnr\": 0.88\n", " }\n", " }\n", " },\n", " \"scope_configuration_arithmetic\": [\n", " {\n", " \"scope\": \"cot_only\",\n", " \"mock_tpr\": 0.82,\n", " \"mock_tnr\": 0.78,\n", " \"configured_product\": 0.6396,\n", " \"configured_balanced_accuracy\": 0.8\n", " },\n", " {\n", " \"scope\": \"answer_only\",\n", " \"mock_tpr\": 0.6,\n", " \"mock_tnr\": 0.85,\n", " \"configured_product\": 0.51,\n", " \"configured_balanced_accuracy\": 0.725\n", " },\n", " {\n", " \"scope\": \"all_messages\",\n", " \"mock_tpr\": 0.88,\n", " \"mock_tnr\": 0.88,\n", " \"configured_product\": 0.7744,\n", " \"configured_balanced_accuracy\": 0.88\n", " }\n", " ],\n", " \"dataset_archive_inspected\": false,\n", " \"pair_integrity_verified\": false,\n", " \"model_performance_measured\": false\n", "}\n" ] } ], "id": "cell-2" }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Regenerated figures\n", "\n", "The following rich SVG outputs come from files regenerated by the preceding code, not screenshots of an external result.\n" ], "id": "cell-3" }, { "cell_type": "code", "metadata": {}, "source": [ "class LocalSVG:\n", " def __init__(self, filename):\n", " self.filename = filename\n", " def _repr_mimebundle_(self, include=None, exclude=None):\n", " return {'image/svg+xml': Path(self.filename).read_text(), 'text/plain': self.filename}\n" ], "execution_count": 2, "outputs": [], "id": "cell-4" }, { "cell_type": "code", "metadata": {}, "source": [ "LocalSVG('mock-configuration.svg')\n" ], "execution_count": 3, "outputs": [ { "output_type": "execute_result", "data": { "image/svg+xml": "The visibility ordering is written into the mockLiteral source-code probabilities. These are not measured monitor capabilities.The visibility ordering is written into the mockLiteral source-code probabilities. These are not measured monitor capabilities.CoT only0.820.78Answer only0.600.85All messages0.880.88Configured TPRConfigured TNR", "text/plain": "mock-configuration.svg" }, "metadata": {}, "execution_count": 3 } ], "id": "cell-5" }, { "cell_type": "code", "metadata": {}, "source": [ "LocalSVG('release-matrix.svg')\n", "\n", "import base64\n", "from pathlib import Path\n", "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('mock-configuration.svg').read_bytes()).decode())\n", "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('release-matrix.svg').read_bytes()).decode())\n" ], "execution_count": 4, "outputs": [ { "output_type": "execute_result", "data": { "image/svg+xml": "Fourteen registry entries do not mean fourteen runnable evaluationsRegistry gates only: archive rows, tool adapters and actual models are separate requirements.Fourteen registry entries do not mean fourteen runnable evaluationsRegistry gates only: archive rows, tool adapters and actual models are separate requirements.Intervention72Process11Outcome12Metadata-eligible for mock scaffoldExplicitly skippedDescription only", "text/plain": "release-matrix.svg" }, "metadata": {}, "execution_count": 4 } ], "id": "cell-6" } ] }