{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.14"
    },
    "execution": {
      "method": "Python subprocess, exact code cell source",
      "deterministic_replay_passed": true
    },
    "runtime": {
      "packages": [],
      "local_assets": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "id": "question",
      "metadata": {},
      "source": [
        "# Retrieval relevance is not constraint satisfaction\n",
        "A higher-ranked product can violate a hard requirement. Validation helps only if retrieval includes a feasible alternative.\n",
        "\nOffline retrieval-versus-feasibility counterexample with exact attributes; no WebShop run or purchase.\n"
      ]
    },
    {
      "cell_type": "code",
      "id": "experiment",
      "metadata": {},
      "execution_count": 1,
      "source": [
        "\"\"\"Retrieval relevance is not constraint satisfaction\n",
        "Deterministic, standard-library experiment. RESULTS contains the full data.\n",
        "\"\"\"\n",
        "colors = ['red', 'blue', 'green', 'black', 'white']\n",
        "sizes = ['XS', 'S', 'M', 'L', 'XL']\n",
        "catalog, queries = [], []\n",
        "for i in range(25):\n",
        "    query = dict(model=f'model{i}', color=colors[i % 5], size=sizes[i // 5], budget=20+i)\n",
        "    queries.append(query)\n",
        "    valid = dict(title=f'jacket model{i}', color=query['color'], size=query['size'], price=15+i)\n",
        "    decoy = dict(valid, title=f'light trail jacket model{i}')\n",
        "    if i % 3 == 0:\n",
        "        decoy['color'] = colors[(i+1) % 5]\n",
        "    elif i % 3 == 1:\n",
        "        decoy['size'] = sizes[(i // 5 + 1) % 5]\n",
        "    else:\n",
        "        decoy['price'] = query['budget'] + 1\n",
        "    catalog.extend([decoy, valid])\n",
        "\n",
        "def violations(item, query):\n",
        "    return [key for key in ('color', 'size') if item[key] != query[key]] + (\n",
        "        ['price'] if item['price'] > query['budget'] else [])\n",
        "\n",
        "hits = dict(keyword=0, checked=0)\n",
        "failures = dict(color=0, size=0, price=0)\n",
        "for query in queries:\n",
        "    words = {'light', 'trail', 'jacket', query['model']}\n",
        "    candidates = [p for p in catalog if query['model'] in p['title'].split()]\n",
        "    ranked = sorted(candidates, key=lambda p: -len(words & set(p['title'].split())))\n",
        "    baseline = ranked[0]\n",
        "    checked = next((p for p in ranked if not violations(p, query)), None)\n",
        "    hits['keyword'] += not violations(baseline, query)\n",
        "    hits['checked'] += checked is not None and not violations(checked, query)\n",
        "    for reason in violations(baseline, query):\n",
        "        failures[reason] += 1\n",
        "print('catalog_items', len(catalog), 'queries', len(queries))\n",
        "print('completed', hits)\n",
        "print('keyword_constraint_violations', failures)\n",
        "assert hits == dict(keyword=0, checked=25)\n",
        "\n",
        "\n",
        "rows=[]\n",
        "for q in queries:\n",
        "    words={'light','trail','jacket',q['model']}\n",
        "    ranked=sorted([p for p in catalog if q['model'] in p['title'].split()],key=lambda p:-len(words & set(p['title'].split())))\n",
        "    rows.append(dict(query=q,candidates=[dict(p,rank=i+1,score=len(words&set(p['title'].split())),violations=violations(p,q)) for i,p in enumerate(ranked)]))\n",
        "# A validator cannot recover an option omitted from the candidate set.\n",
        "cutoff=[dict(k=k,completed=sum(any(not violations(p,q) for p in sorted([p for p in catalog if q['model'] in p['title'].split()],key=lambda p:-len({'light','trail','jacket',q['model']}&set(p['title'].split())))[:k]) for q in queries)) for k in (1,2)]\n",
        "assert [r['completed'] for r in cutoff]==[0,25]\n",
        "RESULTS=dict(queries=rows,completed=hits,violation_counts=failures,retrieval_cutoff=cutoff)\n",
        "print('Feasible selections at retrieval cutoffs:',cutoff)\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "catalog_items 50 queries 25\n",
            "completed {'keyword': 0, 'checked': 25}\n",
            "keyword_constraint_violations {'color': 9, 'size': 8, 'price': 8}\n",
            "Feasible selections at retrieval cutoffs: [{'k': 1, 'completed': 0}, {'k': 2, 'completed': 25}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import base64\n",
        "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('figure.svg').read_bytes()).decode())\n"
      ]
    }
  ]
}