Structured edit errors make recovery programmable notebook

Python starts when you choose Run.

Structured edit errors make recovery programmable

A transactional editor can preserve source state and expose a repairable error. Syntax acceptance still says nothing about intended behavior.

In-memory parser/interface test with scripted repairs; not SWE-agent model performance.

"""Structured edit errors make recovery programmable
Deterministic, standard-library experiment. RESULTS contains the full data.
"""
import ast
BASE = {f'f{i}.py': f'def f{i}():\n    return {i}\n' for i in range(5)}

def edit(files, name, start, end, replacement):
    lines = files[name].splitlines()
    if not 1 <= start <= end <= len(lines):
        return {'error': 'range', 'lines': len(lines)}
    candidate = '\n'.join(lines[:start-1] + [replacement] + lines[end:]) + '\n'
    try:
        ast.parse(candidate)
    except SyntaxError as exc:
        return {'error': 'syntax', 'line': exc.lineno}
    files[name] = candidate
    return {'ok': True}

for structured in (False, True):
    accepted = calls = 0
    for i, name in enumerate(BASE):
        for mistake in ('range', 'syntax'):
            files = BASE.copy()
            start = 99 if mistake == 'range' else 1
            replacement = f'def f{i}():' if mistake == 'range' else f'def f{i}()'
            for attempt in range(2):
                before = files[name]
                result = edit(files, name, start, start, replacement)
                if not structured and not result.get('ok'):
                    result = {'error': 'edit_failed'}
                calls += 1
                if result.get('ok'):
                    accepted += 1
                    break
                assert files[name] == before
                if structured:
                    if result['error'] == 'range':
                        start = min(1, result['lines'])
                    elif result['error'] == 'syntax':
                        replacement = f'def f{i}():'
    print('structured=' + str(structured), 'accepted', accepted, 'calls', calls)
    assert accepted == (10 if structured else 0)

rows=[]
for structured in (False,True):
    for i,name in enumerate(BASE):
        for mistake in ('range','syntax'):
            files=BASE.copy(); start=99 if mistake=='range' else 1
            replacement=f'def f{i}():' if mistake=='range' else f'def f{i}()'
            for attempt in (1,2):
                before=files[name]; response=edit(files,name,start,start,replacement)
                if not structured and not response.get('ok'): response={'error':'edit_failed'}
                rows.append(dict(policy='structured' if structured else 'generic',file=name,mistake=mistake,attempt=attempt,response=response,unchanged=files[name]==before))
                if response.get('ok'): break
                assert files[name]==before
                if structured:
                    if response['error']=='range': start=1
                    else: replacement=f'def f{i}():'
files=BASE.copy()
semantic_response=edit(files,'f0.py',2,2,'    return 999')
assert semantic_response.get('ok') and files['f0.py'] != BASE['f0.py']
parsed=ast.parse(files['f0.py'])
semantic_value=parsed.body[0].body[0].value.value
assert semantic_value==999
RESULTS=dict(attempts=rows,semantic_counterexample=dict(accepted_by_parser=True,expected_return=0,actual_return=semantic_value,semantic_test_passed=semantic_value==0))
print('Valid syntax / wrong behavior:',semantic_value,'!=',0)

structured=False accepted 0 calls 20
structured=True accepted 10 calls 20
Valid syntax / wrong behavior: 999 != 0

Python runs locally in this page. Saved results and figures are visible before execution.

Download full experiment notebook