Onset: version control where nobody waits and nothing gets lost. The on command, its core and onset-server.
| 1 | #!/usr/bin/env node |
| 2 | // Load on an Onset host over HTTP. Agents land file edits with no workspace |
| 3 | // (one call each, its own file, on the latest version), as fast as they can, |
| 4 | // across projects; readers read files meanwhile. Prints lands a second, |
| 5 | // latencies and errors, then how long each project's git view took to catch up. |
| 6 | // Usage: ONSET_TOKEN=... node tests/load.mjs --host https://onset.coline.dev \ |
| 7 | // --projects o/a,o/b [--agents 50] [--seconds 60] [--readers 0] [--create] [--git on|off] [--fanout K] |
| 8 | // --agents is per project; --create makes missing projects; --git sets each one's git view first; |
| 9 | // --fanout puts agents' files in K folders (smaller folders to write again each land). |
| 10 |
| 11 | const args = Object.fromEntries( |
| 12 | process.argv.slice(2).reduce((out, a, i, all) => { |
| 13 | if (a.startsWith("--")) out.push([a.slice(2), all[i + 1]?.startsWith("--") || all[i + 1] === undefined ? "1" : all[i + 1]]); |
| 14 | return out; |
| 15 | }, []), |
| 16 | ); |
| 17 | const HOST = (args.host ?? "https://onset.coline.dev").replace(/\/+$/, ""); |
| 18 | const PROJECTS = (args.projects ?? "").split(",").filter(Boolean); |
| 19 | const AGENTS = Number(args.agents ?? 50); |
| 20 | const SECONDS = Number(args.seconds ?? 60); |
| 21 | const READERS = Number(args.readers ?? 0); |
| 22 | const FANOUT = Number(args.fanout ?? 0); |
| 23 | /** Agent `a`'s file. */ |
| 24 | const fileOf = (a) => (FANOUT > 0 ? `load/${RUN}/g${a % FANOUT}/agent-${a}.txt` : `load/${RUN}/agent-${a}.txt`); |
| 25 | const TOKEN = process.env.ONSET_TOKEN ?? process.env.VCS_TOKEN; |
| 26 | const RUN = `${Date.now().toString(36)}${process.pid.toString(36)}`; |
| 27 | if (!TOKEN || PROJECTS.length === 0) { |
| 28 | console.error("usage: ONSET_TOKEN=... node tests/load.mjs --host URL --projects o/a,o/b [--agents N] [--seconds S] [--readers N] [--create] [--git on|off]"); |
| 29 | process.exit(2); |
| 30 | } |
| 31 |
| 32 | const headers = { authorization: `Bearer ${TOKEN}`, "content-type": "application/json", "user-agent": "onset-load" }; |
| 33 | async function call(method, path, body) { |
| 34 | const started = performance.now(); |
| 35 | try { |
| 36 | const response = await fetch(`${HOST}/v1${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) }); |
| 37 | const text = await response.text(); |
| 38 | let json = null; |
| 39 | try { |
| 40 | json = JSON.parse(text); |
| 41 | } catch { |
| 42 | // Not JSON (a proxy's page). |
| 43 | } |
| 44 | return { status: response.status, json, ms: performance.now() - started }; |
| 45 | } catch (error) { |
| 46 | return { status: 0, json: { error: { code: "network", message: String(error?.cause?.code ?? error) } }, ms: performance.now() - started }; |
| 47 | } |
| 48 | } |
| 49 |
| 50 | function pct(sorted, p) { |
| 51 | return sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))] : 0; |
| 52 | } |
| 53 |
| 54 | for (const p of PROJECTS) { |
| 55 | const [owner, name] = p.split("/"); |
| 56 | let r = await call("GET", `/projects/${p}`); |
| 57 | if (r.status === 404 && args.create) { |
| 58 | r = await call("POST", "/projects", { owner, name }); |
| 59 | if (r.status !== 200) throw new Error(`creating ${p}: ${r.status} ${JSON.stringify(r.json)}`); |
| 60 | console.log(`created ${p}`); |
| 61 | } else if (r.status !== 200) throw new Error(`${p}: ${r.status} ${JSON.stringify(r.json)}`); |
| 62 | if (args.git) { |
| 63 | const s = await call("PUT", `/projects/${p}/settings`, { git: args.git }); |
| 64 | if (s.status !== 200) throw new Error(`${p} settings: ${s.status} ${JSON.stringify(s.json)}`); |
| 65 | } |
| 66 | } |
| 67 |
| 68 | const lands = []; |
| 69 | const reads = []; |
| 70 | const errors = new Map(); |
| 71 | const perProject = new Map(PROJECTS.map((p) => [p, 0])); |
| 72 | const deadline = Date.now() + SECONDS * 1000; |
| 73 | const samples = new Map(); |
| 74 | const count = (key, message) => { |
| 75 | errors.set(key, (errors.get(key) ?? 0) + 1); |
| 76 | if (message && !samples.has(key)) samples.set(key, message); |
| 77 | }; |
| 78 |
| 79 | async function agent(p, a) { |
| 80 | // Each agent edits its own file, on the version its last edit made (a file that's there needs one). |
| 81 | let base = null; |
| 82 | for (let i = 1; Date.now() < deadline; i++) { |
| 83 | const r = await call("POST", `/projects/${p}/edits`, { |
| 84 | title: `Load ${RUN} agent ${a} edit ${i}`, |
| 85 | files: { [fileOf(a)]: `${a} ${i}\n` }, |
| 86 | ...(base === null ? {} : { base }), |
| 87 | }); |
| 88 | if (r.status === 200 && r.json?.outcome === "landed") { |
| 89 | lands.push(r.ms); |
| 90 | perProject.set(p, perProject.get(p) + 1); |
| 91 | base = r.json.version?.n ?? base; |
| 92 | } else if (r.status === 200) count(r.json?.outcome ?? "?"); |
| 93 | else { |
| 94 | count(`${r.status} ${r.json?.error?.code ?? ""}`.trim(), r.json?.error?.message); |
| 95 | // Back off a little on errors, like a client would. |
| 96 | await new Promise((resolve) => setTimeout(resolve, 200)); |
| 97 | } |
| 98 | } |
| 99 | } |
| 100 |
| 101 | async function reader(p, k) { |
| 102 | while (Date.now() < deadline) { |
| 103 | const r = await call("GET", `/projects/${p}/file?path=${encodeURIComponent(fileOf(k % AGENTS))}`); |
| 104 | if (r.status === 200 || r.status === 404) reads.push(r.ms); |
| 105 | else { |
| 106 | count(`read ${r.status} ${r.json?.error?.code ?? ""}`.trim()); |
| 107 | await new Promise((resolve) => setTimeout(resolve, 200)); |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 |
| 112 | console.log(`${PROJECTS.length} projects × ${AGENTS} agents${READERS ? ` + ${READERS} readers` : ""} for ${SECONDS}s against ${HOST} (run ${RUN})`); |
| 113 | const started = Date.now(); |
| 114 | const work = []; |
| 115 | for (const p of PROJECTS) { |
| 116 | for (let a = 0; a < AGENTS; a++) work.push(agent(p, a)); |
| 117 | for (let k = 0; k < READERS; k++) work.push(reader(p, k)); |
| 118 | } |
| 119 | const tick = setInterval(() => { |
| 120 | const s = (Date.now() - started) / 1000; |
| 121 | process.stdout.write(` ${s.toFixed(0)}s: ${lands.length} landed (${(lands.length / s).toFixed(0)}/s), ${[...errors.values()].reduce((x, y) => x + y, 0)} not\n`); |
| 122 | }, 10_000); |
| 123 | await Promise.all(work); |
| 124 | clearInterval(tick); |
| 125 | const elapsed = (Date.now() - started) / 1000; |
| 126 |
| 127 | lands.sort((x, y) => x - y); |
| 128 | reads.sort((x, y) => x - y); |
| 129 | const ms = (v) => `${v.toFixed(0)} ms`; |
| 130 | console.log(); |
| 131 | console.log(`landed ${lands.length} in ${elapsed.toFixed(1)}s = ${(lands.length / elapsed).toFixed(0)} lands/s`); |
| 132 | console.log(`per project ${(lands.length / elapsed / PROJECTS.length).toFixed(1)} lands/s (busiest ${(Math.max(...perProject.values()) / elapsed).toFixed(1)})`); |
| 133 | console.log(`land latency p50 ${ms(pct(lands, 50))} p90 ${ms(pct(lands, 90))} p99 ${ms(pct(lands, 99))} max ${ms(lands.at(-1) ?? 0)}`); |
| 134 | if (reads.length) console.log(`read latency p50 ${ms(pct(reads, 50))} p90 ${ms(pct(reads, 90))} p99 ${ms(pct(reads, 99))} (${(reads.length / elapsed).toFixed(0)}/s)`); |
| 135 | console.log(`not landed ${errors.size ? [...errors].map(([k, v]) => `${k}: ${v}`).join(", ") : "none"}`); |
| 136 | for (const [k, m] of samples) console.log(` ${k}: ${m}`); |
| 137 |
| 138 | // The git view catching up. |
| 139 | const behind = new Map(); |
| 140 | const waitStart = Date.now(); |
| 141 | for (;;) { |
| 142 | const views = await Promise.all(PROJECTS.map((p) => call("GET", `/projects/${p}`))); |
| 143 | const lagging = views.filter((v, i) => { |
| 144 | if (v.json?.git_stopped) behind.set(PROJECTS[i], `paused: ${v.json.git_stopped}`); |
| 145 | return v.json?.git && !v.json?.git_stopped && v.json?.git_behind > 0; |
| 146 | }); |
| 147 | if (lagging.length === 0 || Date.now() - waitStart > 180_000) { |
| 148 | const off = views.filter((v) => !v.json?.git).length; |
| 149 | console.log( |
| 150 | `git view ${off === views.length ? "off" : `caught up ${((Date.now() - waitStart) / 1000).toFixed(1)}s after the run${lagging.length ? `; still behind: ${lagging.map((v) => `${v.json.owner}/${v.json.name} ${v.json.git_behind}`).join(", ")}` : ""}`}`, |
| 151 | ); |
| 152 | for (const [p, why] of behind) console.log(` ${p} ${why}`); |
| 153 | break; |
| 154 | } |
| 155 | await new Promise((resolve) => setTimeout(resolve, 1000)); |
| 156 | } |
| 157 |
