mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] harness: vs_bench grades zero-card trials from the backend's own dryrun-report
A trial whose card was reaped before the readback could not be verified either way, and excluding those rows was too charitable: it flipped openswarm 60.9% -> 100% and McNemar p=0.02 -> 1.0 purely on my read timing. The backend's own dryrun-report (composer/textboxes/filled) is the same bar the card readback applies, so it stands in when the card is gone. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ciregenz
co-authored by
Claude Opus 5
Claude Opus 5
parent
9f7035da6b
commit
1db0a5d04e
@@ -1,4 +1,87 @@
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 9.8, "infra": "", "why": "payload+host in card browser-3df6", "iter": 1, "ts": 1786173595.2}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 32.0, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 3 31.1s payload in editable on https://regex101.com/", "iter": 1, "ts": 1786173627.2}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 10.1, "infra": "", "why": "payload+host in card browser-a3ae", "iter": 1, "ts": 1786173637.3}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 31.6, "infra": "", "why": "30.5s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 1, "ts": 1786173668.9}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 9.7, "infra": "", "why": "payload+host in card browser-8d54", "iter": 1, "ts": 1786173714.9}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 71.1, "infra": "", "why": "11 69.5s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 1, "ts": 1786173786.0}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 8.1, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 1, "ts": 1786173794.1}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 32.7, "infra": "", "why": "h=Y claimed=Y steps= 3 31.3s payload in editable on https://www.deepl.com/en/translator", "iter": 1, "ts": 1786173826.8}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 10.6, "infra": "", "why": "payload+host in card browser-ef8c", "iter": 1, "ts": 1786173837.4}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 35.6, "infra": "", "why": "34.4s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 1, "ts": 1786173873.0}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 7.9, "infra": "", "why": "payload+host in card browser-e0a7", "iter": 1, "ts": 1786173880.9}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 37.7, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 4 36.8s payload in editable on https://regex101.com/", "iter": 1, "ts": 1786173918.6}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 42.2, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 1, "ts": 1786173960.8}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 50.6, "infra": "", "why": "db reach=Y claimed=Y steps= 4 49.6s payload in editable on https://www.onlinegdb.com/", "iter": 1, "ts": 1786174011.4}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 9.7, "infra": "", "why": "payload+host in card browser-1f68", "iter": 2, "ts": 1786174021.1}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 25.2, "infra": "", "why": " 3 23.8s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 2, "ts": 1786174046.4}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 18.2, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 2, "ts": 1786174064.5}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 31.3, "infra": "", "why": "h=Y claimed=Y steps= 3 29.6s payload in editable on https://www.deepl.com/en/translator", "iter": 2, "ts": 1786174095.8}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 8.2, "infra": "", "why": "payload+host in card browser-70f4", "iter": 2, "ts": 1786174104.0}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": false, "claimed": true, "wall_s": 49.0, "infra": "", "why": "] w3schools reach=n claimed=Y steps= 3 47.4s on host, payload absent from 2 editables", "iter": 2, "ts": 1786174153.0}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 5.7, "infra": "", "why": "payload+host in card browser-7036", "iter": 2, "ts": 1786174158.6}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 35.7, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 4 34.9s payload in editable on https://regex101.com/", "iter": 2, "ts": 1786174194.4}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 42.8, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 2, "ts": 1786174237.2}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 99.3, "infra": "", "why": "db reach=Y claimed=Y steps= 6 98.3s payload in editable on https://www.onlinegdb.com/", "iter": 2, "ts": 1786174336.5}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 9.9, "infra": "", "why": "payload+host in card browser-8944", "iter": 3, "ts": 1786174346.4}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 59.3, "infra": "", "why": " 9 57.8s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 3, "ts": 1786174405.7}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 7.8, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 3, "ts": 1786174413.5}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 26.8, "infra": "", "why": "h=Y claimed=Y steps= 3 25.6s payload in editable on https://www.deepl.com/en/translator", "iter": 3, "ts": 1786174440.3}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 10.3, "infra": "", "why": "payload+host in card browser-7ccd", "iter": 3, "ts": 1786174450.6}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 35.8, "infra": "", "why": "34.7s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 3, "ts": 1786174486.4}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 8.1, "infra": "", "why": "payload+host in card browser-669e", "iter": 3, "ts": 1786174494.5}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 29.6, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 3 28.5s payload in editable on https://regex101.com/", "iter": 3, "ts": 1786174524.0}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 42.2, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 3, "ts": 1786174566.2}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 54.7, "infra": "", "why": "db reach=Y claimed=Y steps= 4 53.9s payload in editable on https://www.onlinegdb.com/", "iter": 3, "ts": 1786174621.0}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 10.1, "infra": "", "why": "payload+host in card browser-cfa0", "iter": 4, "ts": 1786174631.1}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 29.3, "infra": "", "why": " 3 28.0s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 4, "ts": 1786174660.4}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 7.9, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 4, "ts": 1786174668.3}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 28.7, "infra": "", "why": "h=Y claimed=Y steps= 3 27.5s payload in editable on https://www.deepl.com/en/translator", "iter": 4, "ts": 1786174697.0}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 11.9, "infra": "", "why": "payload+host in card browser-5672", "iter": 4, "ts": 1786174708.9}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 37.7, "infra": "", "why": "36.6s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 4, "ts": 1786174746.6}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 7.9, "infra": "", "why": "payload+host in card browser-dca6", "iter": 4, "ts": 1786174754.5}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 32.1, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 3 31.3s payload in editable on https://regex101.com/", "iter": 4, "ts": 1786174786.6}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 40.2, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 4, "ts": 1786174826.8}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 51.2, "infra": "", "why": "db reach=Y claimed=Y steps= 4 49.9s payload in editable on https://www.onlinegdb.com/", "iter": 4, "ts": 1786174878.0}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 10.6, "infra": "", "why": "payload+host in card browser-5d88", "iter": 5, "ts": 1786174888.6}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 28.0, "infra": "", "why": " 3 26.3s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 5, "ts": 1786174916.6}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 13.8, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 5, "ts": 1786174930.4}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 28.2, "infra": "", "why": "h=Y claimed=Y steps= 3 26.6s payload in editable on https://www.deepl.com/en/translator", "iter": 5, "ts": 1786174958.6}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 10.0, "infra": "", "why": "payload+host in card browser-2c52", "iter": 5, "ts": 1786174968.5}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 69.7, "infra": "", "why": "68.5s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 5, "ts": 1786175038.2}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 7.7, "infra": "", "why": "payload+host in card browser-9ed1", "iter": 5, "ts": 1786175046.0}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 36.3, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 3 35.3s payload in editable on https://regex101.com/", "iter": 5, "ts": 1786175082.3}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 42.0, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 5, "ts": 1786175124.3}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 62.2, "infra": "", "why": "db reach=Y claimed=Y steps= 4 61.3s payload in editable on https://www.onlinegdb.com/", "iter": 5, "ts": 1786175186.4}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 13.9, "infra": "", "why": "payload+host in card browser-a8e1", "iter": 6, "ts": 1786175200.3}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 28.4, "infra": "", "why": " 3 26.7s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 6, "ts": 1786175228.8}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 15.4, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 6, "ts": 1786175244.1}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 26.1, "infra": "", "why": "h=Y claimed=Y steps= 3 24.6s payload in editable on https://www.deepl.com/en/translator", "iter": 6, "ts": 1786175270.2}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 10.8, "infra": "", "why": "payload+host in card browser-a721", "iter": 6, "ts": 1786175281.0}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 32.8, "infra": "", "why": "31.7s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 6, "ts": 1786175313.8}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 5.7, "infra": "", "why": "payload+host in card browser-4d28", "iter": 6, "ts": 1786175319.6}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 83.8, "infra": "", "why": "gex101 reach=Y claimed=Y steps=12 82.8s payload in editable on https://regex101.com/", "iter": 6, "ts": 1786175403.4}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 42.2, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 6, "ts": 1786175445.5}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 85.0, "infra": "", "why": "db reach=Y claimed=Y steps= 6 83.6s payload in editable on https://www.onlinegdb.com/", "iter": 6, "ts": 1786175530.5}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 10.1, "infra": "", "why": "payload+host in card browser-cd09", "iter": 7, "ts": 1786175540.6}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 28.9, "infra": "", "why": " 3 27.5s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 7, "ts": 1786175569.5}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 7.9, "infra": "", "why": "payload not read back from 0 new card(s)", "iter": 7, "ts": 1786175577.4}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 30.6, "infra": "", "why": "h=Y claimed=Y steps= 3 29.5s payload in editable on https://www.deepl.com/en/translator", "iter": 7, "ts": 1786175608.0}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 12.7, "infra": "", "why": "payload+host in card browser-f4f2", "iter": 7, "ts": 1786175620.7}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 59.8, "infra": "", "why": "58.4s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 7, "ts": 1786175680.5}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 7.6, "infra": "", "why": "payload+host in card browser-ec62", "iter": 7, "ts": 1786175688.1}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": false, "claimed": false, "wall_s": 72.3, "infra": "", "why": "[1/1] regex101 reach=n claimed=n steps=11 71.0s intended host never open", "iter": 7, "ts": 1786175760.4}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 40.0, "infra": "", "why": "payload not read back from 1 new card(s)", "iter": 7, "ts": 1786175800.4}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 59.1, "infra": "", "why": "db reach=Y claimed=Y steps= 4 58.4s payload in editable on https://www.onlinegdb.com/", "iter": 7, "ts": 1786175859.5}
|
||||
{"arm": "openswarm", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 9.8, "infra": "", "why": "payload+host in card browser-fd85", "iter": 8, "ts": 1786175869.3}
|
||||
{"arm": "browser-use", "site": "gtranslate", "reached": true, "claimed": true, "wall_s": 27.0, "infra": "", "why": " 3 25.6s payload in URL of https://translate.google.com/?sl=auto&tl=en&text=coverage%20", "iter": 8, "ts": 1786175896.3}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": true, "wall_s": 16.0, "infra": "", "why": "payload not read back from 1 new card(s)", "iter": 8, "ts": 1786175912.3}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": false, "claimed": false, "wall_s": 74.0, "infra": "", "why": "[1/1] deepl reach=n claimed=n steps=11 73.0s intended host never open", "iter": 8, "ts": 1786175986.4}
|
||||
{"arm": "openswarm", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 8.0, "infra": "", "why": "payload+host in card browser-1676", "iter": 8, "ts": 1786175994.4}
|
||||
{"arm": "browser-use", "site": "w3schools", "reached": true, "claimed": true, "wall_s": 87.6, "infra": "", "why": "86.8s payload in editable on https://www.w3schools.com/html/tryit.asp?filename=tryhtml_de", "iter": 8, "ts": 1786176082.0}
|
||||
{"arm": "openswarm", "site": "regex101", "reached": true, "claimed": true, "wall_s": 7.6, "infra": "", "why": "payload+host in card browser-2843", "iter": 8, "ts": 1786176089.6}
|
||||
{"arm": "browser-use", "site": "regex101", "reached": true, "claimed": true, "wall_s": 41.8, "infra": "", "why": "gex101 reach=Y claimed=Y steps= 3 41.0s payload in editable on https://regex101.com/", "iter": 8, "ts": 1786176131.4}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": true, "wall_s": 40.1, "infra": "", "why": "payload not read back from 1 new card(s)", "iter": 8, "ts": 1786176171.5}
|
||||
{"arm": "browser-use", "site": "onlinegdb", "reached": true, "claimed": true, "wall_s": 50.8, "infra": "", "why": "db reach=Y claimed=Y steps= 4 50.1s payload in editable on https://www.onlinegdb.com/", "iter": 8, "ts": 1786176222.3}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": false, "cards": 1, "wall_s": 241.0, "infra": "", "why": "payload not read back from 1 new card(s)", "iter": 9, "ts": 1786179041.2}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 24.7, "infra": "", "why": "h=Y claimed=Y steps= 3 23.3s payload in editable on https://www.deepl.com/en/translator", "iter": 9, "ts": 1786179065.9}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": false, "cards": 0, "wall_s": 240.8, "infra": "", "why": "from dryrun-report: armed=1 composer=0 opener=0 textboxes=0 filled=0 url=https://www.deepl", "iter": 10, "ts": 1786179348.0}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 28.1, "infra": "", "why": "h=Y claimed=Y steps= 3 26.9s payload in editable on https://www.deepl.com/en/translator", "iter": 10, "ts": 1786179376.1}
|
||||
{"arm": "openswarm", "site": "deepl", "reached": false, "claimed": false, "cards": 1, "wall_s": 240.9, "infra": "", "why": "payload not read back from 1 new card(s)", "iter": 11, "ts": 1786203768.6}
|
||||
{"arm": "browser-use", "site": "deepl", "reached": true, "claimed": true, "wall_s": 51.4, "infra": "", "why": "h=Y claimed=Y steps= 8 49.7s payload in editable on https://www.deepl.com/en/translator", "iter": 11, "ts": 1786203820.0}
|
||||
{"arm": "openswarm", "site": "onlinegdb", "reached": false, "claimed": false, "cards": 1, "wall_s": 240.9, "infra": "", "why": "payload not read back from 1 new card(s)", "iter": 11, "ts": 1786204061.0}
|
||||
|
||||
@@ -186,6 +186,10 @@ def run_openswarm(site: str, task: str, host: str) -> dict:
|
||||
# Verified from the CARD, never from the agent's reply: same bar as the other arm.
|
||||
cards = [c for c in ((req("GET", f"{base}/dashboards/{ds[0]['id']}").get("layout") or {})
|
||||
.get("browser_cards") or {}) if c not in p_before]
|
||||
# No new card at all means the run never opened one, which is NOT the same event as "opened a
|
||||
# page and failed to fill it" -- and the harness could not tell them apart, so 9 rows landed
|
||||
# in a false-success bucket that measured my own read timing. Recorded explicitly.
|
||||
p_ncards = len(cards)
|
||||
found = False
|
||||
for bid in cards:
|
||||
try:
|
||||
@@ -210,7 +214,29 @@ def run_openswarm(site: str, task: str, host: str) -> dict:
|
||||
req("DELETE", f"{base}/agents/sessions/{sid}", timeout=60)
|
||||
except Exception:
|
||||
pass
|
||||
# A trial with no readable card is graded from the BACKEND's own dryrun-report instead of being
|
||||
# excluded. Excluding them was too charitable and flipped the whole result (openswarm 60.9% ->
|
||||
# 100%, McNemar p=0.02 -> 1.0) purely on my read timing, and the log shows those rows are real
|
||||
# misses: deepl reports composer=0 textboxes=0 and onlinegdb composer=0 textboxes=2, both on the
|
||||
# correct URL. `filled=1` in that line is the product asserting it committed the payload, which
|
||||
# is the same bar the card readback applies, so it can stand in when the card is already reaped.
|
||||
if not infra and not reached and locals().get("p_ncards", 1) == 0:
|
||||
p_log = os.environ.get("OSW_LOG", "")
|
||||
p_saw = ""
|
||||
try:
|
||||
with open(p_log, errors="ignore") as fh:
|
||||
for ln in fh:
|
||||
if "[dryrun-report]" in ln and host in ln:
|
||||
p_saw = ln
|
||||
except OSError:
|
||||
p_saw = ""
|
||||
if p_saw:
|
||||
reached = " filled=1" in p_saw
|
||||
why = "from dryrun-report: " + p_saw.split("[dryrun-report]")[-1].strip()[:70]
|
||||
else:
|
||||
infra = "no card and no dryrun-report; unverifiable"
|
||||
return {"arm": "openswarm", "site": site, "reached": bool(reached), "claimed": bool(claimed),
|
||||
"cards": locals().get("p_ncards", -1),
|
||||
"wall_s": round(time.time() - t0, 1), "infra": infra, "why": why}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user