Memory needs a scope: a rule-change test inspired by Reflexion notebook

Python starts when you choose Run.

Memory needs a scope: a rule-change test inspired by Reflexion

A 30-episode trace shows why the conditions attached to a failure note matter more than the size of the notebook.

Deterministic note-retrieval extension; no LLM reflection or Reflexion score replication.

"""Memory needs a scope: a rule-change test inspired by Reflexion
Deterministic, standard-library experiment. RESULTS contains the full data.
"""
EPISODES = [(1 if i < 15 else 2, 'B' if i < 15 else 'A')
            for i in range(30)]

def evaluate(mode):
    notes = []
    wins = invalid = abstained = 0
    history = []
    for version, correct in EPISODES:
        applicable = [n for n in notes if mode == 'permanent'
                      or (mode == 'versioned' and n['version'] == version)]
        blocked = {n['avoid'] for n in applicable}
        lane = next((x for x in ('A', 'B') if x not in blocked), None)
        success = lane == correct
        wins += success
        if lane is None:
            abstained += 1
        elif not success:
            invalid += 1
            if mode != 'none':
                notes.append(dict(version=version, avoid=lane))
        history.append(int(success))
    assert wins + invalid + abstained == 30
    return wins, invalid, abstained, sum(history[:15]), sum(history[15:])

print('policy: wins invalid abstained first_half second_half')
for mode in ('none', 'permanent', 'versioned'):
    print(mode, *evaluate(mode))
assert evaluate('versioned')[0] == 29
assert evaluate('permanent')[2] == 14

rows = []
for mode in ('none','permanent','versioned'):
    notes = []
    for episode,(version,correct) in enumerate(EPISODES,1):
        active = [n for n in notes if mode == 'permanent' or (mode == 'versioned' and n['version'] == version)]
        blocked = {n['avoid'] for n in active}
        lane = next((x for x in ('A','B') if x not in blocked),None)
        outcome = 'abstained' if lane is None else ('completed' if lane==correct else 'invalid')
        rows.append(dict(policy=mode,episode=episode,version=version,correct=correct,choice=lane,blocked=sorted(blocked),outcome=outcome))
        if outcome=='invalid' and mode!='none': notes.append(dict(version=version,avoid=lane))
assert sum(r['outcome']=='completed' for r in rows if r['policy']=='versioned')==29
RESULTS=dict(episodes=rows, totals={p:dict(zip(('completed','invalid','abstained','first_half','second_half'),evaluate(p))) for p in ('none','permanent','versioned')})
print('Episode-level audit:',len(rows),'policy–episode records')

policy: wins invalid abstained first_half second_half
none 15 15 0 0 15
permanent 14 2 14 14 0
versioned 29 1 0 14 15
Episode-level audit: 90 policy–episode records

Python runs locally in this page. Saved results and figures are visible before execution.

Download full experiment notebook