"""Offline, synthetic scoped-retrieval evaluation. Python standard library only. Clone https://github.com/namankumar/agent-infrastructure and check out 4cd0fb407a66ecdc788506f348ad7aad44d3ba33. Then run: python3 scoped-retrieval-benchmark.txt --source /path/to/agent-infrastructure Creates only temporary databases, deleted on exit. Never opens a real workspace. Measures UTF-8 bytes of JSON structuredContent with ensure_ascii=False and default separators. Does not measure model tokens, latency, runtime memory or cost. """ import argparse,datetime,hashlib,json,platform,sys,tempfile from pathlib import Path PIN='4cd0fb407a66ecdc788506f348ad7aad44d3ba33' # Filled from the already published, byte-verified implementation receipt. HASHES={'monitor.py': 'be0dead80c303298f4b40a9a728df10743c94ea7fb0a35ab2a61b073ce22c3be', 'save_preferences.py': '91cbf70edc1e09eae514a670b83a4f23e34b21920c7ceeadcab692922874419a', 'reconciliation.py': '96bcfd1192893785ec3f411ceed96e484fff717fe3eba974bf2bac2802fd50c2', 'records.py': '468bb5a1870bd7762cf00f4c78520457ecf67bf96252c4649e7ea2b96ae82965', 'work.py': '9199c355b598a1508999f1b1ba6bf04260fffcbc2a0f8eb80f1c7dbc795f266e', 'app.py': '77c866d8875832d38d822495036d1509521ba1f7a7e42151dc54ba85e0b7ebb6', 'tables.py': 'b7d8f8ef86173ada55e1931fc15c8289790f5c37df48887bd353d270a024ad0d', 'boards.py': 'c2176c566a76e246e2750fb52b7a95ce93070e74b70cb869be9d6d206dced5bd', 'priorities.py': 'cd659624bf03e22f49cfc143a4da3156f5a1a70582aaf42e520051fa9510cb33', 'tests/test_plugin.py': '4ee86dc5497583522b7a50bab9e7157b62d8713139f120946802fa96a5620350', 'tests/test_priorities.py': '1354c835433b7096503955e8414fb3b2b84b827d223ed98da3075ca60d184c5b', 'tests/test_hosted_mcp.py': 'f632f3d01632332e7362f6f2164c69eb9f8bbe94ad133397bf33d40f90ec7814', 'tests/test_mcp_events.py': '7cd0189bdd0e41c48f10348790398ee0a6d4f4be5893f07b2b8faeb4e5324fa1', 'tests/test_store.py': '1bb8b821f5f5fb5c3e4b6bcab287340051830984a88bf5b235fd9142c0f65e6f', 'tests/test_http.py': 'd73b112dd16ac680525064e31f9e86d555a3160db071b2936b2fb0debee549c7', 'tests/test_private_mcp.py': 'cd815cf2320d684825417f178fd75750f96415376c066254247cbfab481dd626', 'plugin/server.py': '5fa255e965359c73f2576328f4f20b14b8b716f08e0a9a589afa61384d3d9947', 'plugin/setup_capture.py': '88d1b31647669ef6a9c3a4f872da7d6e5ca0d1d5f0f5e090d853a9260a1bb449', 'plugin/runtime/monitor.py': 'd51e40682b23997323b0faa41e404d2e4edb73ee174fbb1356710413a5f5624f', 'plugin/runtime/save_preferences.py': '91cbf70edc1e09eae514a670b83a4f23e34b21920c7ceeadcab692922874419a', 'plugin/runtime/capture_config.py': 'f7fed62860be1ab213f80e584ad5ce4e02757f8e2b3048113b17861ab48eaa18', 'plugin/runtime/reconciliation.py': '96bcfd1192893785ec3f411ceed96e484fff717fe3eba974bf2bac2802fd50c2', 'plugin/runtime/records.py': '468bb5a1870bd7762cf00f4c78520457ecf67bf96252c4649e7ea2b96ae82965', 'plugin/runtime/background_capture.py': 'ea38884493eda3157f8fa80528138ca39d648b8652d8db8d02e72a9b25ddb36b', 'plugin/runtime/work.py': '9199c355b598a1508999f1b1ba6bf04260fffcbc2a0f8eb80f1c7dbc795f266e', 'plugin/runtime/host_capture.py': 'ec85db6bb62f305cf2411d8ff54b72e0f6b17b56b2daaa2073b632155b16c72d', 'plugin/runtime/app.py': '6fb46cb851601a35942e88341a48be1d497ae15ac3ab0d212f684651ec872cbe', 'plugin/runtime/tables.py': 'b7d8f8ef86173ada55e1931fc15c8289790f5c37df48887bd353d270a024ad0d', 'plugin/runtime/boards.py': 'c2176c566a76e246e2750fb52b7a95ce93070e74b70cb869be9d6d206dced5bd', 'plugin/runtime/priorities.py': 'cd659624bf03e22f49cfc143a4da3156f5a1a70582aaf42e520051fa9510cb33', 'hosting/web.py': '938f487fe554d2bd87a4f2d0a9eecb37307e54fd199a622f726b8c68771d0098', 'hosting/events.py': 'a6321883ef6cde9c41b417650916bc3454b25b0ae832c0f14579d07b2cdd2c62'} def run(source): source=Path(source).resolve() for name,digest in HASHES.items(): if hashlib.sha256((source/name).read_bytes()).hexdigest()!=digest: raise SystemExit('Source differs from pinned implementation: '+name) sys.path.insert(0,str(source)) from plugin.server import Server rows=[] for unrelated_count in (10,100,500): with tempfile.TemporaryDirectory(prefix='mi-retrieval-fixture-') as tmp: server=Server(source,Path(tmp)/'synthetic.sqlite') def call(name,args=None): out=server.handle({'method':'tools/call','params':{'name':name,'arguments':args or {}}}) if out.get('isError'):raise AssertionError(out) return out['structuredContent'] def seed(i,text,title): return call('apply_extraction',{'batch':{'id':'batch-'+i,'sources':[{'id':'source-'+i,'label':'Synthetic fixture','text':text,'source_type':'user'}],'projects':[{'id':'project-'+i,'name':title,'objective':title,'source_id':'source-'+i,'quote':title}],'items':[{'id':'item-'+i,'project_id':'project-'+i,'kind':'task','title':title,'detail':'','evidence':[{'source_id':'source-'+i,'quote':title}]}]}}) seed('target','Resolve the Atlas workflow.','Resolve the Atlas workflow.') call('update_task',{'item_id':'item-target','version':1,'changes':{'status':'done'},'request_id':'close-target','source_id':'closed-target','user_statement':'I finished the Atlas workflow.'}) for i in range(unrelated_count):seed(str(i),'Unrelated synthetic work '+str(i)+'. '+('Filler evidence. '*50),'Unrelated synthetic work '+str(i)+'.') before=call('read_workspace');scoped=call('read_workspace',{'query':'Atlas workflow'});after=call('read_workspace') assert before==after,'Reading changed workspace state' expected_items=[x for x in before['state']['items'] if x['id']=='item-target'] assert scoped['state']['items']==expected_items and expected_items[0]['status']=='done' assert scoped['state']['evidence']==[x for x in before['state']['evidence'] if x['item_id']=='item-target'] expected_sources={x['source_id'] for x in scoped['state']['evidence']+scoped['state']['events']+scoped['state']['project_events']} expected_sources.add('source-target') assert {x['id'] for x in scoped['state']['sources']}==expected_sources assert not any(x['id'].startswith('item-') and x['id']!='item-target' for x in scoped['state']['items']) assert scoped['coverage']['workspace_complete'] is False full_bytes=len(json.dumps(before,ensure_ascii=False).encode('utf-8'));scoped_bytes=len(json.dumps(scoped,ensure_ascii=False).encode('utf-8')) rows.append({'unrelated_projects':unrelated_count,'relevant_projects':1,'relevant_tasks':1,'full_state_bytes':full_bytes,'scoped_read_bytes':scoped_bytes,'reduction_percent':round(100*(1-scoped_bytes/full_bytes),2),'closed_task_preserved':True,'relevant_evidence_preserved':True,'unrelated_tasks_excluded':True,'read_did_not_mutate_state':True}) return {'checked_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'implementation_commit':PIN,'python':platform.python_version(),'runner_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),'unit':'UTF-8 bytes of JSON structuredContent','fixture':'One completed relevant task and10/100/500unrelated projects, one task each, with synthetic filler evidence.','rows':rows,'limits':['Three synthetic fixture sizes, one query, one run per size; not a production workload.','Scoped reads filter a full internal snapshot; returned-payload reduction is not evidence of lower database work.','Byte counts are not token savings, latency, cost, task quality or security isolation.','Filtering a subject is not permission enforcement; unrelated content is excluded from this response, not an access-control claim.','Timestamp-bearing evidence can cause small byte differences on reproduction.']} if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('--source',required=True);a=p.parse_args();print(json.dumps(run(a.source),indent=2))