{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.14"
    },
    "execution": {
      "method": "Python subprocess, exact code cell source",
      "deterministic_replay_passed": true
    },
    "runtime": {
      "packages": [],
      "local_assets": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "id": "question",
      "metadata": {},
      "source": [
        "# What text-only observations erase in visual web tasks\n",
        "Four identical labels hide four different colors. A contact sheet exposes exactly what the observation representation keeps and loses.\n",
        "\nSynthetic RGB-feature teaching example; no VLM, screenshot parser, or VisualWebArena evaluation.\n"
      ]
    },
    {
      "cell_type": "code",
      "id": "experiment",
      "metadata": {},
      "execution_count": 1,
      "source": [
        "\"\"\"What text-only observations erase in visual web tasks\n",
        "Deterministic, standard-library experiment. RESULTS contains the full data.\n",
        "\"\"\"\n",
        "COLORS = {'red': (220, 40, 40), 'blue': (40, 70, 220),\n",
        "          'green': (40, 180, 70), 'yellow': (220, 210, 40)}\n",
        "names = list(COLORS)\n",
        "groups = []\n",
        "for group in range(6):\n",
        "    order = names[group % 4:] + names[:group % 4]\n",
        "    cards = []\n",
        "    for color in order:\n",
        "        pixels = [COLORS[color] for _ in range(64)]\n",
        "        cards.append(dict(label='Travel mug', pixels=pixels, truth=color))\n",
        "    groups.append(cards)\n",
        "\n",
        "def mean_rgb(pixels):\n",
        "    return tuple(sum(p[c] for p in pixels) / len(pixels) for c in range(3))\n",
        "\n",
        "def distance(a, b):\n",
        "    return sum((x - y) ** 2 for x, y in zip(a, b))\n",
        "\n",
        "text_hits = pixel_hits = 0\n",
        "for cards in groups:\n",
        "    for target in names:\n",
        "        text_choice = 0\n",
        "        pixel_choice = min(range(4), key=lambda i:\n",
        "            distance(mean_rgb(cards[i]['pixels']), COLORS[target]))\n",
        "        text_hits += cards[text_choice]['truth'] == target\n",
        "        pixel_hits += cards[pixel_choice]['truth'] == target\n",
        "print('cards', sum(map(len, groups)), 'queries', len(groups) * len(names))\n",
        "print('text_correct', text_hits, 'pixel_correct', pixel_hits)\n",
        "print('red_blue_squared_distance', distance(COLORS['red'], COLORS['blue']))\n",
        "assert (text_hits, pixel_hits) == (6, 24)\n",
        "\n",
        "\n",
        "rows = []\n",
        "for group,cards in enumerate(groups):\n",
        "    for target in names:\n",
        "        distances=[distance(mean_rgb(card['pixels']),COLORS[target]) for card in cards]\n",
        "        selected=min(range(4),key=lambda i:distances[i])\n",
        "        rows.append(dict(group=group,target=target,order=[c['truth'] for c in cards],text_choice=0,pixel_choice=selected,distances=distances))\n",
        "# Same RGB mean, different center: averaging discards spatial arrangement.\n",
        "red, blue = COLORS['red'], COLORS['blue']\n",
        "pattern_a = [red]*32 + [blue]*32\n",
        "pattern_b = [blue]*32 + [red]*32\n",
        "assert mean_rgb(pattern_a) == mean_rgb(pattern_b)\n",
        "assert pattern_a[0] != pattern_b[0]\n",
        "RESULTS=dict(colors=COLORS,queries=rows, correct=dict(text=text_hits,pixels=pixel_hits),collision=dict(mean=list(mean_rgb(pattern_a)),first_pixel_a=list(pattern_a[0]),first_pixel_b=list(pattern_b[0])))\n",
        "print('Spatial collision:',mean_rgb(pattern_a),'for two different pixel arrangements')\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cards 24 queries 24\n",
            "text_correct 6 pixel_correct 24\n",
            "red_blue_squared_distance 65700\n",
            "Spatial collision: (130.0, 55.0, 130.0) for two different pixel arrangements\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import base64\n",
        "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('figure.svg').read_bytes()).decode())\n"
      ]
    }
  ]
}