diff --git a/Cargo.lock b/Cargo.lock index 528d6c42..4dc9d909 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4909,6 +4909,7 @@ dependencies = [ "sqd-data-core", "sqd-data-source", "sqd-dataset", + "sqd-hotblocks-harness", "sqd-polars", "sqd-primitives", "sqd-query", @@ -4922,6 +4923,20 @@ dependencies = [ "zstd", ] +[[package]] +name = "sqd-hotblocks-harness" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "libc", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "sqd-hotblocks-retain" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 59935afe..a376b34e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/data-source", "crates/dataset", "crates/hotblocks", + "crates/hotblocks-harness", "crates/hotblocks-retain", "crates/polars", "crates/primitives", diff --git a/crates/data-client/src/reqwest/lines.rs b/crates/data-client/src/reqwest/lines.rs index b8addbf7..bb805a81 100644 --- a/crates/data-client/src/reqwest/lines.rs +++ b/crates/data-client/src/reqwest/lines.rs @@ -35,6 +35,10 @@ impl LineStream { } fn take_final_line(&mut self) -> Option { + // The buffer is emptied here, so the scan position must go with it: a body whose last + // line is unterminated would otherwise leave `unchecked_pos` past the end of an empty + // buffer, and the next poll would index out of bounds. + self.unchecked_pos = 0; let line = std::mem::take(&mut self.line); if line.is_empty() { None @@ -78,3 +82,43 @@ where } } } + +#[cfg(test)] +mod tests { + use futures::{executor::block_on, stream, StreamExt}; + + use super::*; + + fn lines(chunks: &[&'static str]) -> Vec { + let body = stream::iter( + chunks + .iter() + .map(|chunk| Ok::<_, std::io::Error>(Bytes::from_static(chunk.as_bytes()))) + .collect::>() + ); + block_on( + LineStream::new(body) + .map(|line| String::from_utf8(line.unwrap().to_vec()).unwrap()) + .collect() + ) + } + + #[test] + fn splits_lines_across_chunk_boundaries() { + assert_eq!(lines(&["{\"a\":1}\n{\"a\"", ":2}\n"]), ["{\"a\":1}", "{\"a\":2}"]); + } + + #[test] + fn an_unterminated_final_line_is_yielded_and_the_stream_ends() { + // The stream is polled once more after the final line: with the scan position left + // pointing past the emptied buffer, that poll used to index out of bounds and panic, + // taking the ingest task with it. + assert_eq!(lines(&["{\"a\":1}\n{\"a\":2}"]), ["{\"a\":1}", "{\"a\":2}"]); + } + + #[test] + fn an_empty_body_yields_nothing() { + assert!(lines(&[]).is_empty()); + assert!(lines(&[""]).is_empty()); + } +} diff --git a/crates/hotblocks-harness/Cargo.toml b/crates/hotblocks-harness/Cargo.toml new file mode 100644 index 00000000..a4307acd --- /dev/null +++ b/crates/hotblocks-harness/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sqd-hotblocks-harness" +version = "0.1.0" +edition = "2024" +description = "Black-box test harness for the sqd-hotblocks service: source simulator, reference model, client driver" + +[dependencies] +anyhow = { workspace = true } +axum = { workspace = true } +libc = "0.2" +reqwest = { workspace = true, features = ["gzip", "json"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["full"] } + +[lints] +workspace = true diff --git a/crates/hotblocks-harness/README.md b/crates/hotblocks-harness/README.md new file mode 100644 index 00000000..5eea9f91 --- /dev/null +++ b/crates/hotblocks-harness/README.md @@ -0,0 +1,142 @@ +# Hotblocks test harness + +A black-box test harness built against the **hotblocks behavioral specification** — the +implementation-free statement of what the service must do: state model, invariants (`INV-*`), +liveness (`LIV-*`), failure model (`FM-*`), read/write-path requirements (`RP-*` / `WP-*`), +observability (`OB-*`), and the conformance plan (test classes `CT-*`, gap register `GAP-*`). +Those identifiers are the vocabulary of this crate and are cited on every assertion; the spec +documents themselves are maintained separately and land in the repository on their own. + +The system under test is the real `sqd-hotblocks` **binary**, spawned as a child process and +driven over HTTP. Nothing here links against its internals: an assertion that cannot be made +through the binding is an assertion a client could not rely on either — and owning the process +is what makes the crash/restart and shutdown classes expressible at all. + +``` + script ──┬──▶ SourceSim ──HTTP──▶ [ sqd-hotblocks ] ──HTTP──▶ Client ──▶ validators + └──▶ Model ◀──────── compare at quiescence ────────────┘ +``` + +## Running + +```bash +cargo test -p sqd-hotblocks-harness # the harness's own unit tests (model, chain, simulator) +cargo test -p sqd-hotblocks --test ct1_happy_path # CT-1 — the Phase 0 exit criterion +cargo test -p sqd-hotblocks --test ct9_source_faults +``` + +The CT tests live in `crates/hotblocks/tests/` because only a test inside that package gets +`env!("CARGO_BIN_EXE_sqd-hotblocks")` — the path to the freshly built binary. Everything +reusable lives here, so a future soak or benchmark runner can use it outside `cargo test`. + +## The pieces + +| Module | What it is | Spec | +|---|---|---| +| [`sim`](src/sim.rs) | source simulator: scripted chain, fork signals, finality headers, fault knobs | 13 §7, DEF-12 | +| [`model`](src/model.rs) | the reference model — the oracle. Block-exact, well-formedness asserted after every transition | 12 §2 | +| [`driver`](src/driver.rs) | client: the read binding, the structural validators, the anchored follower and backfill scanner | 04 §7, 12 §4 | +| [`compare`](src/compare.rs) | quiescence comparator: diffs every observable, collects *all* violations before failing | 12 §1 | +| [`sut`](src/sut.rs) | process supervisor: config, spawn, readiness, SIGTERM, SIGKILL, restart-on-same-db | — | +| [`chain`](src/chain/) | kind-parametric payloads: `evm`, `solana`, `hyperliquid-fills` | DEF-5 | +| [`harness`](src/harness.rs) | glue: one script drives the simulator and the model in lockstep | 12 §1 | + +## What a test looks like + +```rust +let mut h = Harness::start(HarnessConfig::from_block( + env!("CARGO_BIN_EXE_sqd-hotblocks"), Arc::new(HlFills), 1_000 +)).await?; + +h.produce(50)?; // the source mints 50 blocks; the model EXTENDs by the same run +h.finalize_with_lag(5)?; // finality trails the tip, as on a real chain +h.settle().await?; // wait for the service to converge on the script +h.assert_conforms().await?; // HEAD, FINALIZED-HEAD, STATUS, METADATA, OB-1 gauges, + // and a full-window scan diffed against the model, block for block +``` + +`assert_conforms` fails with every divergence it found, not just the first, and a failing test +prints the service's own log tail (the HTTP access log filtered out — it is never the +interesting part). + +## Design decisions worth knowing + +**The simulator closes every response.** The service only commits a batch when the source's +response *ends* and it has caught up with what it was given (`MaybeOnHead`). A simulator holding +one long-lived push stream open would leave every block buffered and invisible — forever, and +silently. So `/stream` serves what exists now and closes. + +**And it long-polls.** The service re-requests the instant a response ends, so answering `204` +immediately would spin the ingest loop at full CPU. A request with nothing to serve is held for +`poll_timeout` (200 ms by default) — what a real source does. + +**Scans use `includeAllBlocks`.** Coverage is only recoverable by a client when at least one +block is emitted (RP-9); a filter-sparse query returning nothing tells the client nothing about +how far it got (GAP-8). Scanning with `include_all` sidesteps that, so the harness can always +advance. Do not "optimize" it away. + +**Numbering may be sparse.** Solana numbers blocks by time-based slots and a slot that produced +nothing leaves a hole, so contiguous *numbering* is not an invariant — being parent and child is +(`Block::parent_number`). The service agrees: it links batches and chunks by hash and never by +number, and a chunk's range starts at the position that was *requested*, not at the first block +that arrived. `Numbering::Sparse` exercises this end to end (`ct1_sparse_numbering`). + +**Quiescence is shorter than `P-QUIESCENCE`.** The spec proposes `2 × P-CLEANUP-PERIOD` (20 s), +which is about *space* observables converging after a deferred sweep. Chain-state observables +settle as soon as the write path commits, so the default here is 300 ms of stability. A +space-sensitive class (CT-7) must raise it back. + +**Tests run with `--rocksdb-disable-direct-io`.** Direct I/O behaves differently across the +platforms tests run on, and nothing structural depends on it. CT-6/CT-7 must drop that flag — +they are measuring the storage engine, not working around it. + +## What CT-1 covers today + +Run once per kind — `ct1_evm`, `ct1_solana` (on sparse slots), `ct1_hyperliquid_fills`: + +INV-1..3 (structural chain), INV-5/6 (watermark bounds), INV-7 (provenance — payload included), +INV-11 (append), INV-12 (finality monotone), INV-21/22 (response shape and completeness), +INV-23 (anchored ancestry across responses), INV-25 (progress), INV-27 (range honesty), +INV-30 (reporting), RP-5 (bounded wait), RP-9/RP-10 (coverage, client-driven continuation), +OB-1 (chain gauges). + +Each kind sets its own traps, and the oracle has to predict all of them: evm serves `timestamp` +in **seconds** while its transaction `nonce` is a plain number among hex strings; solana serves +eight collections that must all be present, `lamports` as a **string**, and a reward `pubkey` as +an **index** the service resolves into an account. A wrong guess there fails as if the service +were broken, which is why the emission oracle sits next to the source payload in the same file. + +## What it found on day one + +The first green-path run crashed the ingest task: a JSONL body whose final record carried no +trailing newline made `LineStream::take_final_line` leave its scan position past the end of an +emptied buffer, and the next poll indexed out of bounds. The dataset then parked for +`P-EPOCH-RETRY` (60 s) and crash-looped, because the source served the same body again. + +Fixed in `crates/data-client/src/reqwest/lines.rs`; pinned by a unit test there and by +`ct9_source_faults` end to end. It is the first entry of the fault corpus, and the reason +`SimFaults` exists. + +## Where the next phases plug in + +- **CT-2 (crash/restart)** — `Sut::crash()`, `Sut::stop()`, `Sut::restart()` already exist and + keep the same database directory and port across boots. What is missing is the kill-point + matrix. +- **CT-4 (fork/finality corpus)** — `Harness::fork()` and the model's `resolve_fork` / + `Finalize::IntegrityFault` are implemented and unit-tested; the follower implements the + normative CONFLICT recovery of 04 §7. What is missing is the scripts. +- **CT-5 (error taxonomy)** — `Model::predict_query` returns the outcome class for any query; + `Client::query` already classifies every response. What is missing is the request matrix. +- **CT-7 (soak/space)** — needs `OB-6`-style space accounting; the retention model transition + (`Model::retain`) is implemented but the comparator compares the first block *exactly*, which + is wrong once retention starts trimming: the service trims whole chunks, so its window may be + larger than the model's (legal under RS-3/RS-4, `P-RETENTION-SLACK`). Give the comparator that + tolerance before writing retention tests. +- **CT-9 (fuzz)** — `SimFaults` is the injection point. + +## Known open questions + +- With `includeAllBlocks: false`, the query engine appears to emit a header-only record for + blocks with no matching items (see `crates/query/fixtures/hyperliquid/queries/coin_fills`). + Whether that is the coverage carrier, an accident, or something else is unresolved — the model + predicts emissions only under `include_all` until it is. This is a CT-5/INV-22 question. diff --git a/crates/hotblocks-harness/src/chain/evm.rs b/crates/hotblocks-harness/src/chain/evm.rs new file mode 100644 index 00000000..6a423518 --- /dev/null +++ b/crates/hotblocks-harness/src/chain/evm.rs @@ -0,0 +1,236 @@ +//! `evm`: 14 required header fields, one required item collection and three optional ones whose +//! presence is the block's *data-availability mask*. +//! +//! Two traps this kind sets, both of which produce failures that look like service bugs: +//! its `timestamp` is in **seconds** (the service scales it), and a transaction's `nonce` is a +//! plain number while every other numeric-looking field is a hex string. + +use serde_json::{Value, json}; + +use super::Chain; +use crate::types::{Block, BlockNumber, block_hash}; + +pub struct Evm; + +const MINER: &str = "0x1111111111111111111111111111111111111111"; +const SENDER: &str = "0x2222222222222222222222222222222222222222"; +const CONTRACT: &str = "0x3333333333333333333333333333333333333333"; +const TRANSFER_TOPIC: &str = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; +/// The canonical keccak of an empty uncle list. +const EMPTY_UNCLES: &str = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"; + +/// Every other block carries a transaction, so a window mixes blocks that emit nothing at all +/// with blocks that emit a transaction and up to two logs. +fn tx_count(b: &Block) -> u32 { + (b.number % 2) as u32 +} + +fn log_count(b: &Block) -> u32 { + if tx_count(b) == 0 { 0 } else { (b.number % 3) as u32 } +} + +/// Distinct from any block hash: block numbers never reach this domain. +fn tx_hash(b: &Block, index: u32) -> String { + block_hash( + b.number.wrapping_mul(1_000_003).wrapping_add(u64::from(index)), + b.fork_id + ) +} + +impl Evm { + fn header(b: &Block) -> Value { + json!({ + "number": b.number, + "hash": b.hash, + "parentHash": b.parent_hash, + // Seconds. `Block::timestamp()` multiplies by 1000 on the way in, and the query + // renders the stored column back as seconds — so this is what the oracle predicts. + "timestamp": b.timestamp_ms / 1000 + }) + } + + fn source_header(b: &Block) -> Value { + let mut header = Self::header(b); + let o = header.as_object_mut().expect("header is an object"); + for (key, value) in [ + ("transactionsRoot", json!(block_hash(b.number, b.fork_id ^ 0x11))), + ("receiptsRoot", json!(block_hash(b.number, b.fork_id ^ 0x22))), + ("stateRoot", json!(block_hash(b.number, b.fork_id ^ 0x33))), + ("logsBloom", json!("0x00")), + ("sha3Uncles", json!(EMPTY_UNCLES)), + ("extraData", json!("0x")), + ("miner", json!(MINER)), + ("size", json!(1_000 + b.number % 100)), + ("gasLimit", json!("0x1c9c380")), + ("gasUsed", json!("0x5208")) + ] { + o.insert(key.into(), value); + } + header + } + + fn projected_tx(b: &Block, index: u32) -> Value { + json!({ + "transactionIndex": index, + "hash": tx_hash(b, index), + "from": SENDER, + // A plain number here, unlike every other numeric field of a transaction. + "nonce": b.number + }) + } + + fn source_tx(b: &Block, index: u32) -> Value { + let mut tx = Self::projected_tx(b, index); + let o = tx.as_object_mut().expect("transaction is an object"); + for (key, value) in [ + ("gas", json!("0x5208")), + ("cumulativeGasUsed", json!("0x5208")), + ("gasUsed", json!("0x5208")), + ("logsBloom", json!("0x00")) + ] { + o.insert(key.into(), value); + } + tx + } + + fn projected_log(b: &Block, index: u32) -> Value { + json!({ + "logIndex": index, + "transactionIndex": 0, + "address": CONTRACT, + "topics": [TRANSFER_TOPIC, block_hash(b.number, b.fork_id ^ (0x40 + index))], + "data": "0x" + }) + } + + fn source_log(b: &Block, index: u32) -> Value { + let mut log = Self::projected_log(b, index); + log.as_object_mut() + .expect("log is an object") + .insert("transactionHash".into(), json!(tx_hash(b, 0))); + log + } +} + +impl Chain for Evm { + fn config_kind(&self) -> &'static str { + "evm" + } + + fn storage_kind(&self) -> &'static str { + "evm" + } + + fn dialect(&self) -> &'static str { + "evm" + } + + fn source_block(&self, b: &Block) -> Value { + json!({ + "header": Self::source_header(b), + "transactions": (0..tx_count(b)).map(|i| Self::source_tx(b, i)).collect::>(), + // Presence, not content, is what the data-availability mask records: a block with + // `logs: []` and one with no `logs` key carry *different* masks, and the service + // cuts a chunk wherever the mask changes. Every block here declares all three, so + // the mask is constant and chunk boundaries stay driven by the batch, not the shape. + "logs": (0..log_count(b)).map(|i| Self::source_log(b, i)).collect::>(), + "traces": [], + "stateDiffs": [] + }) + } + + fn scan_query(&self, from: BlockNumber, to: Option, expected_parent: Option<&str>) -> Value { + super::scan_query( + self.dialect(), + from, + to, + expected_parent, + json!({ + "block": {"number": true, "hash": true, "parentHash": true, "timestamp": true}, + "transaction": {"transactionIndex": true, "hash": true, "from": true, "nonce": true}, + "log": {"logIndex": true, "transactionIndex": true, "address": true, "topics": true, "data": true} + }), + vec![("transactions", json!([{}])), ("logs", json!([{}]))] + ) + } + + fn expected_emission(&self, b: &Block) -> Value { + super::emission( + Self::header(b), + vec![ + ( + "transactions", + (0..tx_count(b)).map(|i| Self::projected_tx(b, i)).collect() + ), + ("logs", (0..log_count(b)).map(|i| Self::projected_log(b, i)).collect()), + ] + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(number: BlockNumber) -> Block { + Block { + number, + hash: block_hash(number, 0), + parent_number: number - 1, + parent_hash: block_hash(number - 1, 0), + timestamp_ms: 1_760_000_000_000 + number as i64 * 1000, + fork_id: 0 + } + } + + #[test] + fn the_source_serves_seconds_and_the_oracle_predicts_seconds() { + let b = block(1_001); + let source = Evm.source_block(&b); + let expected = Evm.expected_emission(&b); + assert_eq!(source["header"]["timestamp"], expected["header"]["timestamp"]); + assert_eq!(source["header"]["timestamp"].as_i64().unwrap(), b.timestamp_ms / 1000); + } + + #[test] + fn the_emission_oracle_is_the_source_payload_projected() { + let b = block(1_001); // odd → 1 transaction; 1001 % 3 == 2 → 2 logs + let source = Evm.source_block(&b); + let expected = Evm.expected_emission(&b); + + assert_eq!(expected["transactions"].as_array().unwrap().len(), 1); + assert_eq!(expected["logs"].as_array().unwrap().len(), 2); + + for collection in ["transactions", "logs"] { + for (src, exp) in source[collection] + .as_array() + .unwrap() + .iter() + .zip(expected[collection].as_array().unwrap()) + { + for (key, value) in exp.as_object().unwrap() { + assert_eq!(&src[key], value, "{collection}.{key} must come from the served payload"); + } + } + } + } + + #[test] + fn a_block_with_no_items_emits_a_bare_header() { + let b = block(1_000); // even → no transactions, and therefore no logs + let emitted = Evm.expected_emission(&b); + assert_eq!(emitted.as_object().unwrap().len(), 1); + assert!(emitted.get("transactions").is_none()); + } + + /// The mask is derived from *presence*, so it must not drift block to block. + #[test] + fn every_block_declares_the_same_optional_collections() { + for number in 1_000..1_010 { + let source = Evm.source_block(&block(number)); + for collection in ["logs", "traces", "stateDiffs"] { + assert!(source[collection].is_array(), "block {number} dropped {collection}"); + } + } + } +} diff --git a/crates/hotblocks-harness/src/chain/hl_fills.rs b/crates/hotblocks-harness/src/chain/hl_fills.rs new file mode 100644 index 00000000..77665ff7 --- /dev/null +++ b/crates/hotblocks-harness/src/chain/hl_fills.rs @@ -0,0 +1,165 @@ +//! `hyperliquid-fills`: one header, one flat item collection, no nesting. + +use serde_json::{Value, json}; + +use super::Chain; +use crate::types::{Block, BlockNumber}; + +pub struct HlFills; + +const USERS: [&str; 2] = [ + "0xcf0a36dec06e90263288100c11cf69828338e826", + "0xecb63caa47c7c4e77f60f1ce858cf28dc2b82b00" +]; +const COINS: [&str; 2] = ["BTC", "ETH"]; + +/// `number % 3` fills, so a window mixes empty and non-empty blocks — the filter-sparse shape +/// CT-5 (GAP-8) will lean on. +fn fill_count(b: &Block) -> u32 { + (b.number % 3) as u32 +} + +fn fill_seed(b: &Block, i: u32) -> u64 { + b.number + .wrapping_mul(7) + .wrapping_add(u64::from(i)) + .wrapping_add(u64::from(b.fork_id).wrapping_mul(1_000_003)) +} + +impl HlFills { + /// The fields a scan projects — the same set the emission oracle predicts. + fn projected_fill(b: &Block, i: u32) -> Value { + let s = fill_seed(b, i); + json!({ + "fillIndex": i, + "user": USERS[(s % 2) as usize], + "coin": COINS[(s % 2) as usize], + // Quarters are exact in binary floating point: source → parquet → response + // round-trips compare by equality, no epsilon. + "px": 100.0 + (s % 8) as f64 * 0.25, + "sz": 0.25 * ((s % 4) + 1) as f64, + "side": if s % 2 == 0 { "B" } else { "A" } + }) + } + + fn source_fill(b: &Block, i: u32) -> Value { + let s = fill_seed(b, i); + let mut fill = Self::projected_fill(b, i); + let o = fill.as_object_mut().expect("fill is an object"); + o.insert("time".into(), json!(b.timestamp_ms)); + o.insert("startPosition".into(), json!(0.0)); + o.insert("dir".into(), json!(if s % 2 == 0 { "Open Long" } else { "Open Short" })); + o.insert("closedPnl".into(), json!(0.0)); + o.insert("hash".into(), json!(format!("{}{:02x}", &b.hash[..64], i % 256))); + o.insert("oid".into(), json!(s)); + o.insert("crossed".into(), json!(s % 2 == 0)); + o.insert("fee".into(), json!(0.25)); + o.insert("tid".into(), json!(s.wrapping_mul(31))); + o.insert("feeToken".into(), json!("USDC")); + fill + } + + fn header(b: &Block) -> Value { + json!({ + "number": b.number, + "hash": b.hash, + "parentHash": b.parent_hash, + // Milliseconds at the source for this kind; evm/bitcoin serve seconds. + "timestamp": b.timestamp_ms + }) + } +} + +impl Chain for HlFills { + fn config_kind(&self) -> &'static str { + "hyperliquid-fills" + } + + fn storage_kind(&self) -> &'static str { + "hl-fills" + } + + fn dialect(&self) -> &'static str { + "hyperliquidFills" + } + + fn source_block(&self, b: &Block) -> Value { + json!({ + "header": Self::header(b), + "fills": (0..fill_count(b)).map(|i| Self::source_fill(b, i)).collect::>() + }) + } + + fn scan_query(&self, from: BlockNumber, to: Option, expected_parent: Option<&str>) -> Value { + super::scan_query( + self.dialect(), + from, + to, + expected_parent, + json!({ + "block": {"number": true, "hash": true, "parentHash": true, "timestamp": true}, + "fill": {"fillIndex": true, "user": true, "coin": true, "px": true, "sz": true, "side": true} + }), + // No predicates — selects every fill, so the scan checks payload provenance (INV-7). + vec![("fills", json!([{}]))] + ) + } + + fn expected_emission(&self, b: &Block) -> Value { + let fills = (0..fill_count(b)).map(|i| Self::projected_fill(b, i)).collect(); + super::emission(Self::header(b), vec![("fills", fills)]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::block_hash; + + fn block(number: BlockNumber, fork_id: u32) -> Block { + Block { + number, + hash: block_hash(number, fork_id), + parent_number: number - 1, + parent_hash: block_hash(number - 1, fork_id), + timestamp_ms: 1_700_000_000_000 + number as i64 * 1000, + fork_id + } + } + + #[test] + fn the_emission_oracle_is_the_source_payload_projected() { + let b = block(101, 0); // 101 % 3 == 2 fills + let source = HlFills.source_block(&b); + let expected = HlFills.expected_emission(&b); + + let source_fills = source["fills"].as_array().unwrap(); + let expected_fills = expected["fills"].as_array().unwrap(); + assert_eq!(source_fills.len(), 2); + assert_eq!(expected_fills.len(), 2); + + for (src, exp) in source_fills.iter().zip(expected_fills) { + for (k, v) in exp.as_object().unwrap() { + assert_eq!(&src[k], v, "projected field {k} must come from the served payload"); + } + } + assert_eq!(source["header"], expected["header"]); + } + + #[test] + fn a_fork_changes_the_payload_at_the_same_height() { + let a = HlFills.source_block(&block(101, 0)); + let b = HlFills.source_block(&block(101, 1)); + assert_ne!( + a, b, + "a replaced block must not be payload-identical to the one it replaced" + ); + } + + #[test] + fn empty_blocks_emit_a_bare_header() { + let b = block(102, 0); // 102 % 3 == 0 + assert_eq!(HlFills.expected_emission(&b).get("fills"), None); + assert_eq!(HlFills.source_block(&b)["fills"].as_array().unwrap().len(), 0); + } +} diff --git a/crates/hotblocks-harness/src/chain/mod.rs b/crates/hotblocks-harness/src/chain/mod.rs new file mode 100644 index 00000000..48ebe68b --- /dev/null +++ b/crates/hotblocks-harness/src/chain/mod.rs @@ -0,0 +1,76 @@ +//! Kind-parametric payloads (DEF-5): what the source serves, what a client asks for, what the +//! service must emit back. Everything above this module is kind-agnostic. + +mod evm; +mod hl_fills; +mod solana; + +pub use evm::Evm; +pub use hl_fills::HlFills; +use serde_json::Value; +pub use solana::Solana; + +use crate::types::{Block, BlockNumber}; + +pub trait Chain: Send + Sync + 'static { + /// The `kind` value of the dataset config (YAML). + fn config_kind(&self) -> &'static str; + + /// The kind string the service reports back in STATUS, which is *not* always the config one + /// (`hyperliquid-fills` in, `hl-fills` out). + fn storage_kind(&self) -> &'static str; + + /// The query dialect tag — the `type` field of a query body (IB-3). + fn dialect(&self) -> &'static str; + + /// One JSONL record of the source's `/stream` response (DEF-12 `Blocks`, IB §7). + fn source_block(&self, b: &Block) -> Value; + + /// A full-window scan. It sets `include_all`, so the last emitted block *is* the coverage + /// end `L` (RP-9) and the scanner can always advance; a filter-sparse query cannot (GAP-8). + fn scan_query(&self, from: BlockNumber, to: Option, expected_parent: Option<&str>) -> Value; + + /// What the service must emit for `b` in answer to [`Chain::scan_query`] — the INV-22 oracle. + fn expected_emission(&self, b: &Block) -> Value; +} + +/// Assemble a scan query out of the parts every dialect shares. +fn scan_query( + dialect: &str, + from: BlockNumber, + to: Option, + expected_parent: Option<&str>, + fields: Value, + items: Vec<(&str, Value)> +) -> Value { + let mut q = serde_json::json!({ + "type": dialect, + "fromBlock": from, + "includeAllBlocks": true, + "fields": fields + }); + let o = q.as_object_mut().expect("query is an object"); + if let Some(to) = to { + o.insert("toBlock".into(), serde_json::json!(to)); + } + if let Some(parent) = expected_parent { + o.insert("parentBlockHash".into(), serde_json::json!(parent)); + } + for (name, request) in items { + o.insert(name.into(), request); + } + q +} + +/// An emitted block: the header, plus each item collection that has items. The service omits an +/// empty collection rather than emitting `[]`. +fn emission(header: Value, collections: Vec<(&str, Vec)>) -> Value { + let mut block = serde_json::json!({ "header": header }); + let o = block.as_object_mut().expect("block is an object"); + for (name, items) in collections { + if !items.is_empty() { + o.insert(name.into(), Value::Array(items)); + } + } + block +} diff --git a/crates/hotblocks-harness/src/chain/solana.rs b/crates/hotblocks-harness/src/chain/solana.rs new file mode 100644 index 00000000..4286d57d --- /dev/null +++ b/crates/hotblocks-harness/src/chain/solana.rs @@ -0,0 +1,179 @@ +//! `solana`: the kind that makes the sparse-numbering case real. Blocks are numbered by +//! time-based **slots**, so `number - parent_number` is not always 1, and the header carries +//! `parentNumber` explicitly. +//! +//! Traps: every one of the eight collections must be present (an empty array, never an absent +//! key); a reward's `lamports` is a JSON *string*; and its `pubkey` is an *index* into the +//! block's `accounts`, which the service resolves — so the emission carries the account, not the +//! index. + +use serde_json::{Value, json}; + +use super::Chain; +use crate::types::{Block, BlockNumber}; + +pub struct Solana; + +const ACCOUNTS: [&str; 2] = [ + "11111111111111111111111111111111", + "Vote111111111111111111111111111111111111111" +]; + +/// `number % 3` rewards, so a window mixes empty and non-empty blocks. +fn reward_count(b: &Block) -> u32 { + (b.number % 3) as u32 +} + +fn lamports(b: &Block, index: u32) -> i64 { + (b.number as i64).wrapping_mul(7) + i64::from(index) + i64::from(b.fork_id) * 1_000_003 +} + +impl Solana { + fn header(b: &Block) -> Value { + json!({ + "number": b.number, + "hash": b.hash, + "parentNumber": b.parent_number, + "parentHash": b.parent_hash, + // Block height is informational here: nothing in the write path reads it, and the + // harness does not model it apart from the slot. + "height": b.number, + // Seconds, like evm. + "timestamp": b.timestamp_ms / 1000 + }) + } + + /// What the service emits: `pubkey` resolved from the index to the account itself. + fn projected_reward(b: &Block, index: u32) -> Value { + json!({ + "pubkey": ACCOUNTS[(index % 2) as usize], + "lamports": lamports(b, index).to_string(), + "postBalance": (lamports(b, index) + 1_000).to_string() + }) + } + + fn source_reward(b: &Block, index: u32) -> Value { + json!({ + "pubkey": index % 2, + "lamports": lamports(b, index).to_string(), + "postBalance": (lamports(b, index) + 1_000).to_string() + }) + } +} + +impl Chain for Solana { + fn config_kind(&self) -> &'static str { + "solana" + } + + fn storage_kind(&self) -> &'static str { + "solana" + } + + fn dialect(&self) -> &'static str { + "solana" + } + + fn source_block(&self, b: &Block) -> Value { + json!({ + "header": Self::header(b), + "transactions": [], + "instructions": [], + "logs": [], + "balances": [], + "tokenBalances": [], + "rewards": (0..reward_count(b)).map(|i| Self::source_reward(b, i)).collect::>(), + "accounts": ACCOUNTS + }) + } + + fn scan_query(&self, from: BlockNumber, to: Option, expected_parent: Option<&str>) -> Value { + super::scan_query( + self.dialect(), + from, + to, + expected_parent, + json!({ + "block": { + "number": true, "hash": true, "parentNumber": true, + "parentHash": true, "height": true, "timestamp": true + }, + "reward": {"pubkey": true, "lamports": true, "postBalance": true} + }), + vec![("rewards", json!([{}]))] + ) + } + + fn expected_emission(&self, b: &Block) -> Value { + let rewards = (0..reward_count(b)).map(|i| Self::projected_reward(b, i)).collect(); + super::emission(Self::header(b), vec![("rewards", rewards)]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::block_hash; + + /// Slot 1_003, whose parent is slot 1_000 — three slots produced nothing. + fn sparse_block(number: BlockNumber, parent_number: BlockNumber) -> Block { + Block { + number, + hash: block_hash(number, 0), + parent_number, + parent_hash: block_hash(parent_number, 0), + timestamp_ms: 1_760_000_000_000 + number as i64 * 1000, + fork_id: 0 + } + } + + #[test] + fn the_header_carries_the_parent_slot_across_a_hole() { + let b = sparse_block(1_003, 1_000); + let header = &Solana.source_block(&b)["header"]; + assert_eq!(header["number"], 1_003); + assert_eq!(header["parentNumber"], 1_000, "the hole is explicit in the payload"); + assert_eq!(Solana.expected_emission(&b)["header"], *header); + } + + #[test] + fn every_collection_is_present_even_when_empty() { + let source = Solana.source_block(&sparse_block(1_002, 1_000)); // 1002 % 3 == 0 rewards + for collection in [ + "transactions", + "instructions", + "logs", + "balances", + "tokenBalances", + "rewards", + "accounts" + ] { + assert!(source[collection].is_array(), "{collection} must be present"); + } + assert!( + Solana + .expected_emission(&sparse_block(1_002, 1_000)) + .get("rewards") + .is_none() + ); + } + + /// The source sends an index; the service stores the account it resolves to. + #[test] + fn a_reward_pubkey_is_an_index_at_the_source_and_an_account_in_the_emission() { + let b = sparse_block(1_004, 1_003); // 1004 % 3 == 2 rewards + let source = Solana.source_block(&b); + let emitted = Solana.expected_emission(&b); + + let source_rewards = source["rewards"].as_array().unwrap(); + let emitted_rewards = emitted["rewards"].as_array().unwrap(); + assert_eq!(source_rewards.len(), 2); + + for (src, exp) in source_rewards.iter().zip(emitted_rewards) { + let index = src["pubkey"].as_u64().unwrap() as usize; + assert_eq!(exp["pubkey"], ACCOUNTS[index], "the index must resolve to its account"); + assert_eq!(src["lamports"], exp["lamports"]); + assert!(src["lamports"].is_string(), "lamports crosses the wire as a string"); + } + } +} diff --git a/crates/hotblocks-harness/src/compare.rs b/crates/hotblocks-harness/src/compare.rs new file mode 100644 index 00000000..e06366f6 --- /dev/null +++ b/crates/hotblocks-harness/src/compare.rs @@ -0,0 +1,222 @@ +//! The quiescence comparator (spec 12 §1) — diffs every observable against the reference model, +//! at quiescence: no pending input, watermarks stable. That is what absorbs the two legitimate +//! sources of nondeterminism, batch boundaries and coverage cuts (INV-22). +//! +//! `P-QUIESCENCE` (14) proposes `2 × P-CLEANUP-PERIOD` (20s), but that bound is about *space* +//! observables converging after a deferred sweep. Chain state settles as soon as the write path +//! commits, so the default here is far shorter. CT-7 must raise it back. + +use std::time::{Duration, Instant}; + +use anyhow::{Result, bail}; +use serde_json::Value; + +use crate::{ + chain::Chain, + driver::{Client, Status}, + model::Model, + types::{BlockNumber, BlockRef} +}; + +#[derive(Clone, Debug)] +pub struct Quiescence { + /// How long the observables must hold still before they count as settled. + pub stable_for: Duration, + pub timeout: Duration, + pub poll: Duration +} + +impl Default for Quiescence { + fn default() -> Self { + Self { + stable_for: Duration::from_millis(300), + timeout: Duration::from_secs(30), + poll: Duration::from_millis(50) + } + } +} + +/// The triple that says "the write path has caught up with the script". +#[derive(Clone, Debug, PartialEq, Eq)] +struct Watermarks { + first: Option, + head: Option, + fin: Option +} + +impl Watermarks { + fn of_model(m: &Model) -> Self { + Self { + first: m.first(), + head: m.head(), + fin: m.fin.clone() + } + } + + fn of_status(s: &Status) -> Self { + match &s.data { + None => Self { + first: None, + head: None, + fin: None + }, + Some(d) => Self { + first: Some(d.first_block), + head: Some(BlockRef::new(d.last_block, d.last_block_hash.clone())), + fin: d.finalized_head.clone() + } + } + } +} + +/// Wait until the SUT's watermarks equal the model's and stay there. +pub async fn await_quiescence(client: &Client, model: &Model, q: &Quiescence) -> Result<()> { + let expected = Watermarks::of_model(model); + let deadline = Instant::now() + q.timeout; + let mut stable_since: Option = None; + + loop { + let observed = Watermarks::of_status(&client.status().await?); + if observed == expected { + let since = *stable_since.get_or_insert_with(Instant::now); + if since.elapsed() >= q.stable_for { + return Ok(()); + } + } else { + stable_since = None; + } + + if Instant::now() > deadline { + bail!( + "the service did not converge on the script within {:?}\n expected: {expected:?}\n observed: {observed:?}", + q.timeout + ); + } + tokio::time::sleep(q.poll).await; + } +} + +/// Diff every observable against the model, reporting *all* violations, not the first. +pub async fn assert_conforms(client: &Client, model: &Model, chain: &dyn Chain, dataset: &str) -> Result<()> { + let mut errs: Vec = Vec::new(); + + // --- watermark reads (RP-12/RP-14, INV-30) -------------------------------------------- + let head = client.head().await?; + if head != model.head() { + errs.push(format!("HEAD: expected {:?}, got {head:?}", model.head())); + } + let fin = client.finalized_head().await?; + if fin != model.fin { + errs.push(format!("FINALIZED-HEAD: expected {:?}, got {fin:?}", model.fin)); + } + + let status = client.status().await?; + if status.kind != chain.storage_kind() { + errs.push(format!( + "STATUS.kind: expected {:?}, got {:?}", + chain.storage_kind(), + status.kind + )); + } + let observed = Watermarks::of_status(&status); + let expected = Watermarks::of_model(model); + if observed != expected { + errs.push(format!("STATUS: expected {expected:?}, got {observed:?}")); + } + // INV-5/30: first <= fin <= head, in one consistent view. + if let Some(data) = &status.data { + if let Some(fin) = &data.finalized_head + && !(data.first_block <= fin.number && fin.number <= data.last_block) + { + errs.push(format!( + "INV-5: finalized head {} outside [{}, {}]", + fin.number, data.first_block, data.last_block + )); + } + let head_ts = model + .head() + .and_then(|h| model.blocks_in(h.number, h.number).first().map(|b| b.timestamp_ms)); + if data.last_block_timestamp != head_ts { + errs.push(format!( + "STATUS.lastBlockTimestamp: expected {head_ts:?}, got {:?}", + data.last_block_timestamp + )); + } + } + + let metadata = client.metadata().await?; + let start_block = metadata["start_block"].as_u64(); + if start_block != model.first() { + errs.push(format!( + "METADATA.start_block: expected {:?}, got {start_block:?}", + model.first() + )); + } + + // --- the OB-1 gauges ------------------------------------------------------------------ + let metrics = client.metrics().await?; + let gauge = |name: &str| metrics.get(name, Some(("dataset", dataset))); + if let (Some(first), Some(head)) = (model.first(), model.head()) { + for (name, expected) in [ + ("hotblocks_first_block", first as f64), + ("hotblocks_last_block", head.number as f64), + // Reported as 0, not absent, when nothing is finalized yet. + ( + "hotblocks_last_finalized_block", + model.fin.as_ref().map_or(0, |f| f.number) as f64 + ) + ] { + match gauge(name) { + Some(got) if got == expected => {} + got => errs.push(format!("OB-1 {name}: expected {expected}, got {got:?}")) + } + } + } + + // --- the window itself (INV-7/21/22/23/27) -------------------------------------------- + if let (Some(first), Some(head)) = (model.first(), model.head()) { + let emitted = client + .scan(chain, first, head.number, model.anchor.hash.as_deref()) + .await?; + let expected = model.emission(first, head.number, chain); + + if emitted.len() != expected.len() { + errs.push(format!( + "the scan of [{first}, {}] emitted {} blocks, the model predicts {}", + head.number, + emitted.len(), + expected.len() + )); + } + let mut shown = 0; + for (got, want) in emitted.iter().zip(&expected) { + if normalize(&got.raw) != normalize(want) { + shown += 1; + if shown <= 3 { + errs.push(format!( + "INV-22: block {} was emitted as {} but the model predicts {}", + got.number, got.raw, want + )); + } + } + } + if shown > 3 { + errs.push(format!("... and {} more block mismatches", shown - 3)); + } + } + + if errs.is_empty() { + Ok(()) + } else { + bail!("the service diverged from the model:\n - {}", errs.join("\n - ")) + } +} + +/// An emitted block may carry an empty item collection or omit it; both mean "no items". +fn normalize(v: &Value) -> Value { + let mut v = v.clone(); + if let Some(o) = v.as_object_mut() { + o.retain(|_, value| !matches!(value, Value::Array(a) if a.is_empty())); + } + v +} diff --git a/crates/hotblocks-harness/src/driver.rs b/crates/hotblocks-harness/src/driver.rs new file mode 100644 index 00000000..a21f9dfc --- /dev/null +++ b/crates/hotblocks-harness/src/driver.rs @@ -0,0 +1,526 @@ +//! The client driver: the read binding (spec 13 §2-§5), the kind-agnostic structural validators +//! (12 §4), and the two normative client algorithms of 04 §7 — the backfill scanner and the +//! anchored follower. +//! +//! Every response goes through [`validate`] before anything else looks at it: a violation there +//! is a bug in the service, whatever the test was asserting. + +use std::{ + collections::BTreeMap, + time::{Duration, Instant} +}; + +use anyhow::{Context, Result, bail, ensure}; +use serde::Deserialize; +use serde_json::Value; + +use crate::{ + chain::Chain, + types::{BlockNumber, BlockRef} +}; + +/// One block as the service emitted it (IB-4: one JSON object per record). +#[derive(Clone, Debug)] +pub struct Emitted { + pub number: BlockNumber, + pub hash: String, + pub parent_hash: String, + pub raw: Value +} + +impl Emitted { + pub fn as_ref(&self) -> BlockRef { + BlockRef::new(self.number, self.hash.clone()) + } +} + +/// IB-6 — the watermarks a query response carries. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Watermarks { + pub head: Option, + pub finalized: Option +} + +/// The error taxonomy of 04 §8, as it arrives over the binding. +#[derive(Debug)] +pub enum Outcome { + Ok { + blocks: Vec, + watermarks: Watermarks + }, + NoData { + watermarks: Watermarks + }, + Conflict { + hints: Vec + }, + Error { + status: u16, + body: String + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Status { + /// The *storage* kind, not always the config one (`hl-fills` vs `hyperliquid-fills`). + pub kind: String, + pub retention_strategy: Value, + pub data: Option +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StatusData { + pub first_block: BlockNumber, + pub last_block: BlockNumber, + pub last_block_hash: String, + pub last_block_timestamp: Option, + pub finalized_head: Option +} + +pub struct Client { + http: reqwest::Client, + base: String, + dataset: String +} + +impl Client { + pub fn new(base_url: impl Into, dataset: impl Into) -> Result { + let http = reqwest::Client::builder() + .gzip(true) + // A query may long-poll for `P-HEAD-WAIT` (5s) and then run for `P-QUERY-TIME` (10s). + .timeout(Duration::from_secs(60)) + .no_proxy() + .build()?; + Ok(Self { + http, + base: base_url.into(), + dataset: dataset.into() + }) + } + + fn url(&self, path: &str) -> String { + format!("{}/datasets/{}/{}", self.base, self.dataset, path) + } + + pub async fn head(&self) -> Result> { + Ok(self + .http + .get(self.url("head")) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + + pub async fn finalized_head(&self) -> Result> { + Ok(self + .http + .get(self.url("finalized-head")) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + + pub async fn status(&self) -> Result { + Ok(self + .http + .get(self.url("status")) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + + pub async fn metadata(&self) -> Result { + Ok(self + .http + .get(self.url("metadata")) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + + /// SET-RETENTION (DEF-9). Returns the status code: `403` unless the dataset is API-controlled. + pub async fn set_retention(&self, policy: &Value) -> Result { + let res = self.http.post(self.url("retention")).json(policy).send().await?; + Ok(res.status().as_u16()) + } + + /// The OB-* surface, parsed into `name{labels} -> value`. + pub async fn metrics(&self) -> Result { + let text = self + .http + .get(format!("{}/metrics", self.base)) + .send() + .await? + .error_for_status()? + .text() + .await?; + let mut out = BTreeMap::new(); + for line in text.lines() { + if line.starts_with('#') { + continue; + } + if let Some((key, value)) = line.rsplit_once(' ') + && let Ok(value) = value.parse::() + { + out.insert(key.to_string(), value); + } + } + Ok(Metrics(out)) + } + + pub async fn query(&self, body: &Value) -> Result { + self.query_at("stream", body).await + } + + pub async fn query_finalized(&self, body: &Value) -> Result { + self.query_at("finalized-stream", body).await + } + + async fn query_at(&self, path: &str, body: &Value) -> Result { + let res = self.http.post(self.url(path)).json(body).send().await?; + let status = res.status().as_u16(); + + // Headers must be read before the body consumes the response. + let watermarks = Watermarks { + head: header(&res, "x-sqd-head-number").and_then(|v| v.parse().ok()), + finalized: match ( + header(&res, "x-sqd-finalized-head-number").and_then(|v| v.parse().ok()), + header(&res, "x-sqd-finalized-head-hash") + ) { + (Some(number), Some(hash)) => Some(BlockRef::new(number, hash)), + (None, None) => None, + _ => bail!("IB-6: the finalized-head headers must be present as a pair") + } + }; + + Ok(match status { + 200 => Outcome::Ok { + blocks: parse_jsonl(&res.text().await?)?, + watermarks + }, + 204 => Outcome::NoData { watermarks }, + 409 => { + let body: ConflictBody = res.json().await.context("malformed CONFLICT payload")?; + ensure!( + !body.previous_blocks.is_empty(), + "RP-11: CONFLICT hints must be non-empty" + ); + Outcome::Conflict { + hints: body.previous_blocks + } + } + _ => Outcome::Error { + status, + body: res.text().await.unwrap_or_default() + } + }) + } + + /// The backfill scanner of 04 §7: walk `[from, to]` with anchored, budget-truncated responses + /// until the window is covered. + pub async fn scan( + &self, + chain: &dyn Chain, + from: BlockNumber, + to: BlockNumber, + expected_parent: Option<&str> + ) -> Result> { + let mut out: Vec = Vec::new(); + let mut cursor = from; + let mut parent = expected_parent.map(str::to_string); + let deadline = Instant::now() + Duration::from_secs(120); + + while cursor <= to { + ensure!( + Instant::now() < deadline, + "the scan of [{from}, {to}] did not converge: stuck at {cursor} after {} blocks", + out.len() + ); + let query = chain.scan_query(cursor, Some(to), parent.as_deref()); + match self.query(&query).await? { + Outcome::Ok { blocks, watermarks } => { + let bounds = Bounds { + from: cursor, + to: Some(to), + expected_parent: parent.clone(), + include_all: true + }; + validate(&blocks, &watermarks, &bounds)?; + // Under `include_all`, an empty success means zero coverage (INV-25, RP-9). + let last = blocks.last().context("INV-25: successful response covered no block")?; + cursor = last.number + 1; + parent = Some(last.hash.clone()); + out.extend(blocks); + } + Outcome::NoData { watermarks } => bail!( + "NO_DATA at {cursor} while scanning the committed window [{from}, {to}] \ + (watermarks: {watermarks:?})" + ), + Outcome::Conflict { hints } => bail!( + "unexpected CONFLICT at {cursor}: the scan follows the chain it is reading; \ + hints = {hints:?}" + ), + Outcome::Error { status, body } => bail!("query failed with {status}: {body}") + } + } + Ok(out) + } +} + +/// Prometheus text output, keyed by `name{labels}` exactly as exposed. +#[derive(Clone, Debug)] +pub struct Metrics(BTreeMap); + +impl Metrics { + pub fn get(&self, name: &str, label: Option<(&str, &str)>) -> Option { + self.0 + .iter() + .find(|(key, _)| { + let Some(rest) = key.strip_prefix(name) else { + return false; + }; + match label { + None => rest.is_empty(), + Some((k, v)) => rest.starts_with('{') && rest.contains(&format!("{k}=\"{v}\"")) + } + }) + .map(|(_, value)| *value) + } +} + +/// What a response is allowed to contain, per the request that produced it. +pub struct Bounds { + pub from: BlockNumber, + pub to: Option, + pub expected_parent: Option, + /// The query emits every covered block, so consecutive records must be parent and child — + /// whatever their numbers, which are sparse on a slot-numbered chain. + pub include_all: bool +} + +/// The structural validators of 12 §4 — the cheap 80% of INV-21/22/23, checkable without +/// understanding the kind. +pub fn validate(blocks: &[Emitted], watermarks: &Watermarks, bounds: &Bounds) -> Result<()> { + let Some(first) = blocks.first() else { return Ok(()) }; + + // Strictly ascending, unique, linked. + for w in blocks.windows(2) { + ensure!( + w[1].number > w[0].number, + "INV-21: emitted blocks are not strictly ascending ({} then {})", + w[0].number, + w[1].number + ); + if bounds.include_all || w[1].number == w[0].number + 1 { + ensure!( + w[1].parent_hash == w[0].hash, + "INV-2/23: block {} does not link to block {}", + w[1].number, + w[0].number + ); + } + } + + // Within the requested range and at or below the reported head. + ensure!( + first.number >= bounds.from, + "INV-27: block {} is below the requested from {}", + first.number, + bounds.from + ); + let last = blocks.last().expect("non-empty"); + if let Some(to) = bounds.to { + ensure!( + last.number <= to, + "INV-27: block {} is above the requested to {to}", + last.number + ); + } + if let Some(head) = watermarks.head { + ensure!( + last.number <= head, + "INV-27: block {} is above the reported head {head}", + last.number + ); + } + + // Anchored continuation: the first block links to what the client asserted. + if let Some(expected) = &bounds.expected_parent { + ensure!( + &first.parent_hash == expected, + "INV-23: block {} claims parent {} but the client anchored on {expected}", + first.number, + first.parent_hash + ); + } + + // Watermark coherence, whenever both are reported. + if let (Some(head), Some(fin)) = (watermarks.head, watermarks.finalized.as_ref()) { + ensure!( + fin.number <= head, + "INV-5/30: finalized head {} is above the head {head}", + fin.number + ); + } + Ok(()) +} + +/// The anchored follower of 04 §7, CONFLICT recovery included. Its local chain is what a correct +/// client would hold. +pub struct Follower { + pub local: Vec, + pub from: BlockNumber, + pub parent: Option, + pub rollbacks: u32 +} + +#[derive(Debug, PartialEq, Eq)] +pub enum FollowStep { + Applied(usize), + NoData, + RolledBack { to: BlockNumber } +} + +impl Follower { + pub fn new(from: BlockNumber, parent: Option) -> Self { + Self { + local: Vec::new(), + from, + parent, + rollbacks: 0 + } + } + + pub fn head(&self) -> Option { + self.local.last().map(Emitted::as_ref) + } + + /// One round trip of the client loop. + pub async fn step(&mut self, client: &Client, chain: &dyn Chain) -> Result { + let query = chain.scan_query(self.from, None, self.parent.as_deref()); + match client.query(&query).await? { + Outcome::Ok { blocks, watermarks } => { + let bounds = Bounds { + from: self.from, + to: None, + expected_parent: self.parent.clone(), + include_all: true + }; + validate(&blocks, &watermarks, &bounds)?; + let last = blocks.last().context("INV-25: successful response covered no block")?; + if let Some(tail) = self.local.last() { + ensure!( + blocks[0].parent_hash == tail.hash, + "INV-23: the follower's chain broke between responses at block {}", + blocks[0].number + ); + } + self.from = last.number + 1; + self.parent = Some(last.hash.clone()); + let n = blocks.len(); + self.local.extend(blocks); + Ok(FollowStep::Applied(n)) + } + Outcome::NoData { .. } => Ok(FollowStep::NoData), + Outcome::Conflict { hints } => { + self.rollbacks += 1; + // "a = highest hint the local chain has" (04 §7). + let anchor = hints + .iter() + .rev() + .find(|h| self.local.iter().any(|b| b.number == h.number && b.hash == h.hash)) + .cloned(); + match anchor { + Some(a) => { + self.local.retain(|b| b.number <= a.number); + self.from = a.number + 1; + self.parent = Some(a.hash); + Ok(FollowStep::RolledBack { to: self.from - 1 }) + } + None => { + // Nothing in common: walk back to the lowest hint and re-anchor there. + let lowest = hints.first().expect("hints are non-empty").clone(); + self.local.retain(|b| b.number < lowest.number); + self.from = lowest.number; + self.parent = None; + Ok(FollowStep::RolledBack { + to: lowest.number.saturating_sub(1) + }) + } + } + } + Outcome::Error { status, body } => bail!("the follower got {status}: {body}") + } + } + + /// Follow until the local chain reaches `target`, or give up. + pub async fn follow_to( + &mut self, + client: &Client, + chain: &dyn Chain, + target: BlockNumber, + timeout: Duration + ) -> Result<()> { + let deadline = Instant::now() + timeout; + while self.head().is_none_or(|h| h.number < target) { + ensure!( + Instant::now() < deadline, + "the follower did not reach block {target} within {timeout:?} (at {:?})", + self.head() + ); + self.step(client, chain).await?; + } + Ok(()) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ConflictBody { + previous_blocks: Vec +} + +fn header(res: &reqwest::Response, name: &str) -> Option { + res.headers().get(name)?.to_str().ok().map(str::to_string) +} + +/// IB-4 — a success body decodes as JSON Lines, one block object per record. +fn parse_jsonl(text: &str) -> Result> { + let mut out = Vec::new(); + for (i, line) in text.lines().enumerate() { + if line.is_empty() { + continue; + } + let raw: Value = serde_json::from_str(line) + .with_context(|| format!("IB-4: record {i} is not a JSON object: {:.120}", line))?; + let header = &raw["header"]; + let number = header["number"] + .as_u64() + .with_context(|| format!("record {i} has no header.number"))?; + let hash = header["hash"] + .as_str() + .with_context(|| format!("record {i} has no header.hash"))? + .to_string(); + let parent_hash = header["parentHash"] + .as_str() + .with_context(|| format!("record {i} has no header.parentHash"))? + .to_string(); + out.push(Emitted { + number, + hash, + parent_hash, + raw + }); + } + Ok(out) +} diff --git a/crates/hotblocks-harness/src/harness.rs b/crates/hotblocks-harness/src/harness.rs new file mode 100644 index 00000000..e468d4f8 --- /dev/null +++ b/crates/hotblocks-harness/src/harness.rs @@ -0,0 +1,175 @@ +//! The glue: one script drives the source simulator and the reference model in lockstep, the +//! real service sits between them as a black box. +//! +//! ```text +//! script ──┬──▶ SourceSim ──HTTP──▶ [ sqd-hotblocks ] ──HTTP──▶ Client ──▶ validators +//! └──▶ Model ◀────────────── compare at quiescence ───────┘ +//! ``` + +use std::{path::Path, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, bail}; + +use crate::{ + chain::Chain, + compare::{self, Quiescence}, + driver::{Client, Follower}, + model::{Finalize, Model}, + sim::{Numbering, SimConfig, SimDataset, SourceSim}, + sut::{DatasetSpec, Retention, Sut, SutConfig}, + types::{Anchor, BlockNumber, block_hash} +}; + +pub struct HarnessConfig { + pub bin: std::path::PathBuf, + pub dataset: String, + pub chain: Arc, + pub start_block: BlockNumber, + pub base_timestamp_ms: i64, + /// Dense (evm, hyperliquid) or sparse (Solana slots) block numbering. + pub numbering: Numbering, + pub retention: Retention, + /// Whether the service is told the anchor hash. If not, the anchor is `⊥` (DEF-7) and the + /// first block's parent is unverifiable. + pub anchored: bool, + /// How long the simulator holds a `/stream` request that has nothing to serve. + pub source_poll: Duration, + pub rust_log: String, + pub quiescence: Quiescence, + pub sut_args: Vec +} + +impl HarnessConfig { + /// One dataset, ingest pinned at `start_block`, anchored on the block below it. + pub fn from_block(bin: impl AsRef, chain: Arc, start_block: BlockNumber) -> Self { + let sut = SutConfig::new(bin.as_ref(), Vec::new()); + Self { + bin: sut.bin, + dataset: "ds0".to_string(), + chain, + start_block, + base_timestamp_ms: 1_760_000_000_000, + numbering: Numbering::Dense, + retention: Retention::FromBlock { + number: start_block, + parent_hash: Some(block_hash(start_block - 1, 0)) + }, + anchored: true, + source_poll: Duration::from_millis(200), + rust_log: "info".to_string(), + quiescence: Quiescence::default(), + sut_args: sut.args + } + } +} + +pub struct Harness { + pub sim: SourceSim, + pub sut: Sut, + pub client: Client, + pub model: Model, + pub chain: Arc, + pub dataset: String, + pub start_block: BlockNumber, + pub numbering: Numbering, + pub quiescence: Quiescence +} + +impl Harness { + pub async fn start(cfg: HarnessConfig) -> Result { + let sim = SourceSim::start(SimConfig { + datasets: vec![SimDataset { + id: cfg.dataset.clone(), + chain: cfg.chain.clone(), + start_block: cfg.start_block, + base_timestamp_ms: cfg.base_timestamp_ms, + numbering: cfg.numbering + }], + poll_timeout: cfg.source_poll + }) + .await + .context("failed to start the source simulator")?; + + let mut sut_cfg = SutConfig::new( + &cfg.bin, + vec![DatasetSpec { + id: cfg.dataset.clone(), + kind: cfg.chain.config_kind().to_string(), + retention: cfg.retention.clone(), + sources: vec![sim.base_url(&cfg.dataset)] + }] + ); + sut_cfg.args = cfg.sut_args; + sut_cfg.rust_log = cfg.rust_log; + let sut = Sut::start(sut_cfg).await?; + + let client = Client::new(sut.base_url(), &cfg.dataset)?; + let anchor_hash = cfg.anchored.then(|| sim.anchor_hash(&cfg.dataset)); + let model = Model::new(Anchor::new(cfg.start_block - 1, anchor_hash)); + + Ok(Self { + sim, + sut, + client, + model, + chain: cfg.chain, + dataset: cfg.dataset, + start_block: cfg.start_block, + numbering: cfg.numbering, + quiescence: cfg.quiescence + }) + } + + /// The source mints `n` more blocks; the model EXTENDs by the same run. + pub fn produce(&mut self, n: u32) -> Result<()> { + let blocks = self.sim.produce(&self.dataset, n); + self.model.extend(&blocks, None) + } + + /// The source reorgs: the suffix at `from` is replaced by `len` freshly-minted blocks. + pub fn fork(&mut self, from: BlockNumber, len: u32) -> Result<()> { + let blocks = self.sim.fork(&self.dataset, from, len)?; + self.model.replace(from, &blocks, None) + } + + /// The source declares block `number` final. + pub fn finalize(&mut self, number: BlockNumber) -> Result<()> { + let r = self.sim.finalize(&self.dataset, number)?; + match self.model.finalize(&r) { + Finalize::Applied | Finalize::Ignored => Ok(()), + Finalize::IntegrityFault => bail!("the script declared a finality that contradicts the model: {r}") + } + } + + /// Finality trailing the tip by `lag` *blocks*, as on a real chain. Blocks, not numbers: + /// on a slot-numbered chain `tip - lag` may name a slot that produced nothing. + pub fn finalize_with_lag(&mut self, lag: u64) -> Result<()> { + let Some(i) = self.model.span().checked_sub(1 + lag as usize) else { + return Ok(()); // not deep enough yet + }; + let number = self.model.seg[i].number; + self.finalize(number) + } + + /// Wait until the service has caught up with the script and holds still. On failure it + /// reports how the source was driven: "never asked" and "asked, threw the answer away" are + /// very different bugs. + pub async fn settle(&self) -> Result<()> { + compare::await_quiescence(&self.client, &self.model, &self.quiescence) + .await + .with_context(|| format!("the source saw: {:?}", self.sim.stats(&self.dataset))) + } + + /// Diff every observable against the model. + pub async fn assert_conforms(&self) -> Result<()> { + compare::assert_conforms(&self.client, &self.model, &*self.chain, &self.dataset).await + } + + /// An anchored follower positioned at the bottom of the window (04 §7). + pub fn follower(&self) -> Follower { + Follower::new( + self.model.first().unwrap_or(self.start_block), + self.model.anchor.hash.clone() + ) + } +} diff --git a/crates/hotblocks-harness/src/lib.rs b/crates/hotblocks-harness/src/lib.rs new file mode 100644 index 00000000..a93dd4c2 --- /dev/null +++ b/crates/hotblocks-harness/src/lib.rs @@ -0,0 +1,60 @@ +//! # Hotblocks test harness +//! +//! A black-box test harness built against the **hotblocks behavioral specification** — the +//! implementation-free statement of what the service must do. The `INV-*` / `RP-*` / `CT-*` / +//! `GAP-*` identifiers cited throughout this crate are its vocabulary; the spec documents +//! themselves are maintained separately. The pieces: +//! +//! | Piece | Spec | Module | +//! |---|---|---| +//! | source simulator (scripted chain, fork/finality signals) | 13 §7, DEF-12 | [`sim`] | +//! | reference model (the oracle) | 12 §2 | [`model`] | +//! | client driver + structural validators | 04 §7, 12 §4 | [`driver`] | +//! | quiescence comparator | 12 §1 | [`compare`] | +//! | SUT process supervisor (spawn / crash / restart) | — | [`sut`] | +//! | kind-parametric payloads | DEF-5 | [`chain`] | +//! | glue: script → simulator + model in lockstep | 12 §1 | [`harness`] | +//! +//! The system under test is the real `sqd-hotblocks` binary, driven over HTTP. Nothing here +//! links against its internals — an assertion that cannot be made through the binding is an +//! assertion a client could not rely on either. +//! +//! ## Writing a test +//! +//! Tests live in `crates/hotblocks/tests/`: only a test inside that package gets +//! `env!("CARGO_BIN_EXE_sqd-hotblocks")`, the `bin` path below. +//! +//! ```no_run +//! # use std::{path::Path, sync::Arc}; +//! # use sqd_hotblocks_harness::{chain::HlFills, harness::{Harness, HarnessConfig}}; +//! # async fn example(bin: &Path) -> anyhow::Result<()> { +//! let mut h = Harness::start(HarnessConfig::from_block(bin, Arc::new(HlFills), 1_000)).await?; +//! +//! h.produce(50)?; // the source mints 50 blocks; the model EXTENDs +//! h.finalize_with_lag(5)?; // finality trails the tip +//! h.settle().await?; // wait for the service to converge +//! h.assert_conforms().await?; // diff every observable against the model +//! # Ok(()) +//! # } +//! ``` + +pub mod chain; +pub mod compare; +pub mod driver; +pub mod harness; +pub mod model; +pub mod sim; +pub mod sut; +pub mod types; + +/// `P-CONFLICT-WINDOW` (14) — how many ancestors a CONFLICT/ForkSignal payload carries. +pub const P_CONFLICT_WINDOW: u64 = 100; + +pub use chain::{Chain, Evm, HlFills, Solana}; +pub use compare::Quiescence; +pub use driver::{Client, Emitted, FollowStep, Follower, Outcome}; +pub use harness::{Harness, HarnessConfig}; +pub use model::{Finalize, ForkResolution, Model, Predicted}; +pub use sim::{Numbering, SourceSim}; +pub use sut::{DatasetSpec, Retention, Sut, SutConfig}; +pub use types::{Anchor, Block, BlockNumber, BlockRef}; diff --git a/crates/hotblocks-harness/src/model.rs b/crates/hotblocks-harness/src/model.rs new file mode 100644 index 00000000..9c5c36a8 --- /dev/null +++ b/crates/hotblocks-harness/src/model.rs @@ -0,0 +1,878 @@ +//! The reference model — the oracle (spec 12 §2). One instance per dataset, fed the same script +//! the simulator executes. Transitions mirror the normative pseudocode one-for-one; +//! well-formedness (INV-1..6) is asserted after each. +//! +//! It is deliberately block-exact. Where the implementation may be coarser (chunk-granular +//! retention, RS-3's "keep *at least* k blocks"), the slack belongs in the comparator, not here. + +use anyhow::{Result, bail, ensure}; +use serde_json::Value; + +use crate::{ + chain::Chain, + types::{Anchor, Block, BlockNumber, BlockRef} +}; + +/// Outcome of a FINALIZE transition (WP §2.4). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Finalize { + Applied, + /// Stale or not-yet-applicable report; state unchanged (WP-7). + Ignored, + /// Finality contradicting stored data — must be alarmed, never accepted (WP-8, GAP-4/5). + IntegrityFault +} + +/// Outcome of resolving a `ForkSignal` (WP-6/WP-6b). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ForkResolution { + Resume { + from: BlockNumber, + parent_hash: Option + }, + /// The divergence is at or below the anchor — a destructive re-bootstrap, alarmed + /// and observable like every RESET (WP-6b, OB-9). + Reset { anchor: Anchor }, + /// The divergence is below the finalized prefix — unapplicable (FM-SRC-5). + IntegrityFault +} + +/// What a query must produce against this state (04 §3/§7). The coverage end `L` is the one free +/// variable left to the implementation (12 §2), so `Ok` reports only the bound it may not exceed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Predicted { + Ok { hi: BlockNumber }, + NoData, + RangeUnavailable, + Conflict { hints: Vec } +} + +/// DEF-6 — the abstract dataset state. +#[derive(Clone, Debug)] +pub struct Model { + pub seg: Vec, + pub anchor: Anchor, + pub fin: Option, + pub ver: u64 +} + +impl Model { + pub fn new(anchor: Anchor) -> Self { + Self { + seg: Vec::new(), + anchor, + fin: None, + ver: 0 + } + } + + pub fn first(&self) -> Option { + self.seg.first().map(|b| b.number) + } + + pub fn head(&self) -> Option { + self.seg.last().map(Block::as_ref) + } + + pub fn next(&self) -> BlockNumber { + self.seg.last().map_or(self.anchor.number + 1, |b| b.number + 1) + } + + pub fn span(&self) -> usize { + self.seg.len() + } + + /// DEF-16 — the chain hash at position `n`: the highest stored block at or below it, else + /// the anchor (whose hash may be `⊥`). `None` below the anchor — the position is outside + /// what this window knows. + pub fn hash_at(&self, n: BlockNumber) -> Option<&str> { + let i = self.seg.partition_point(|b| b.number <= n); + if i > 0 { + return Some(&self.seg[i - 1].hash); + } + if n >= self.anchor.number { + return self.anchor.hash.as_deref(); + } + None + } + + /// The stored block numbered exactly `n` — `⊥` at a hole (12 §2). + fn block_at(&self, n: BlockNumber) -> Option<&Block> { + self.seg + .binary_search_by_key(&n, |b| b.number) + .ok() + .map(|i| &self.seg[i]) + } + + fn stored_or_anchor(&self, r: &BlockRef) -> bool { + self.hash_at(r.number) == Some(r.hash.as_str()) + } + + /// INV-1..6 — must hold in every observable state. + fn wf(&self) { + for w in self.seg.windows(2) { + // Numbering may be sparse (Solana slots), so contiguity of *numbers* is not the + // invariant; being parent and child is. + assert!(w[1].number > w[0].number, "INV-1: segment is not ascending"); + assert_eq!( + w[1].parent_number, w[0].number, + "INV-1: a block is missing between {} and {}", + w[0].number, w[1].number + ); + assert_eq!(w[1].parent_hash, w[0].hash, "INV-2: segment is not linked"); + } + if let Some(first) = self.seg.first() { + assert!( + self.anchor.number < first.number, + "INV-3: the anchor does not sit below the window" + ); + if let Some(h) = &self.anchor.hash { + assert_eq!(&first.parent_hash, h, "INV-3: window does not link to its anchor"); + } + } + if let Some(fin) = &self.fin { + let first = self.first().expect("INV-5: finalized watermark without a window"); + let head = self.head().expect("INV-5: finalized watermark without a window").number; + assert!( + first <= fin.number && fin.number <= head, + "INV-5: finalized watermark {} outside [{first}, {head}]", + fin.number + ); + assert_eq!( + self.block_at(fin.number).map(|b| b.hash.as_str()), + Some(fin.hash.as_str()), + "INV-6: finalized watermark does not name the stored block" + ); + } + } + + /// WP §2.2 — append a linked run of blocks at the tail, optionally advancing `fin`. + pub fn extend(&mut self, blocks: &[Block], fin: Option<&BlockRef>) -> Result<()> { + ensure!(!blocks.is_empty(), "EXTEND with an empty run"); + valid_run(blocks)?; + // `next()` is a *request position*: with sparse numbering the run may start above it. + ensure!( + blocks[0].number >= self.next(), + "EXTEND at {} but the dataset expects {} or above", + blocks[0].number, + self.next() + ); + if let Some(head) = self.head() { + ensure!( + blocks[0].parent_number == head.number, + "EXTEND skips a block: {} is not the child of the head {head}", + blocks[0].number + ); + ensure!( + blocks[0].parent_hash == head.hash, + "EXTEND breaks linkage: block {} claims parent {} but the chain ends at {head}", + blocks[0].number, + blocks[0].parent_hash + ); + } else if let Some(anchor) = &self.anchor.hash { + ensure!( + &blocks[0].parent_hash == anchor, + "EXTEND breaks linkage: block {} does not link to the anchor", + blocks[0].number + ); + } + self.seg.extend_from_slice(blocks); + self.ver += 1; + self.apply_finality(fin)?; + self.wf(); + Ok(()) + } + + /// WP §2.3 — atomically substitute the suffix `>= from` (a fork). + pub fn replace(&mut self, from: BlockNumber, blocks: &[Block], fin: Option<&BlockRef>) -> Result<()> { + ensure!(!self.seg.is_empty(), "REPLACE on an empty window"); + ensure!(!blocks.is_empty(), "REPLACE with an empty run"); + valid_run(blocks)?; + ensure!( + blocks[0].number >= from, + "REPLACE run starts below the fork point {from}" + ); + let first = self.first().expect("seg is non-empty"); + // INV-14: a fork may not reach below the window ... + ensure!( + from >= first, + "INV-14: REPLACE at {from} is below the window floor {first}" + ); + // ... nor below the finalized prefix (INV-13). + if let Some(fin) = &self.fin { + ensure!( + from > fin.number, + "INV-13: REPLACE at {from} is at or below the finalized head {}", + fin.number + ); + } + let keep = self.seg.partition_point(|b| b.number < from); + { + // The run attaches to the last surviving block, or to the anchor if none survives. + match keep.checked_sub(1).map(|i| &self.seg[i]) { + Some(base) => ensure!( + blocks[0].parent_number == base.number && blocks[0].parent_hash == base.hash, + "REPLACE breaks linkage at {from}: block {} is not the child of {}", + blocks[0].number, + base.number + ), + None => { + if let Some(anchor) = &self.anchor.hash { + ensure!( + &blocks[0].parent_hash == anchor, + "REPLACE breaks linkage at {from}: block {} does not link to the anchor", + blocks[0].number + ); + } + } + } + } + self.seg.truncate(keep); + self.seg.extend_from_slice(blocks); + self.ver += 1; + self.apply_finality(fin)?; + self.wf(); + Ok(()) + } + + /// WP §2.4 — advance `fin`, monotone and clamped to the stored chain. + pub fn finalize(&mut self, r: &BlockRef) -> Finalize { + let outcome = self.finalize_inner(r); + if outcome == Finalize::Applied { + self.ver += 1; + self.wf(); + } + outcome + } + + fn finalize_inner(&mut self, r: &BlockRef) -> Finalize { + let Some(head) = self.head() else { + return Finalize::Ignored; + }; + let first = self.first().expect("seg is non-empty"); + let e = r.number.min(head.number); + if e < first { + return Finalize::Ignored; + } + if let Some(fin) = &self.fin + && e < fin.number + { + return Finalize::Ignored; + } + let at_e = if e == r.number { + // The report names an exact height: a hole there contradicts the stored chain the + // same way a hash mismatch does (WP §2.4), and a different stored hash is WP-8. + let Some(stored) = self.block_at(e) else { + return Finalize::IntegrityFault; + }; + if stored.hash != r.hash { + return Finalize::IntegrityFault; + } + stored.hash.clone() + } else { + // Clamped: `e = head.number`, and the head is a stored block. + head.hash + }; + if let Some(fin) = &self.fin + && e == fin.number + && at_e != fin.hash + { + return Finalize::IntegrityFault; + } + let new = BlockRef::new(e, at_e); + if self.fin.as_ref() == Some(&new) { + return Finalize::Ignored; + } + self.fin = Some(new); + Finalize::Applied + } + + /// A commit may compose EXTEND/REPLACE with FINALIZE (02 §6) — one version bump, not two. + fn apply_finality(&mut self, fin: Option<&BlockRef>) -> Result<()> { + let Some(r) = fin else { return Ok(()) }; + match self.finalize_inner(r) { + Finalize::Applied | Finalize::Ignored => Ok(()), + Finalize::IntegrityFault => bail!("INTEGRITY_FAULT: finality report {r} contradicts stored data") + } + } + + /// WP §2.5 — trim the prefix below `from`, moving the anchor up. + pub fn retain(&mut self, from: BlockNumber, hash: Option<&str>) { + let floor = self.first().unwrap_or(self.anchor.number + 1); + if from == floor || (self.seg.is_empty() && from < floor) { + // No-op — unless the hash contradicts the anchor, which is a WP-9 self-heal. + if let (Some(h), Some(anchor)) = (hash, self.anchor.hash.as_deref()) + && from == floor + && h != anchor + { + self.reset(Anchor::new(from - 1, Some(h.to_string()))); + } + return; + } + if from < floor { + // WP §2.5: a downward bound is a destructive re-bootstrap — history below the + // window cannot be re-acquired in place, so the window is discarded and + // re-ingested from `from` upward. Alarmed like every RESET (OB-9), never a + // silent widening. + self.reset(Anchor::new(from - 1, hash.map(str::to_string))); + return; + } + if !self.seg.is_empty() && from <= self.next() { + // The window's new anchor is the last block below `from` — with sparse numbering + // that is not necessarily the block numbered `from - 1`. + let drop = self.seg.partition_point(|b| b.number < from); + let new_anchor = self.seg[drop - 1].as_ref(); + if let Some(h) = hash + && h != new_anchor.hash + { + // The instruction names a block we do not have — re-anchor (WP-9). + self.reset(Anchor::new(from - 1, Some(h.to_string()))); + return; + } + self.seg.drain(..drop); + self.anchor = Anchor::new(new_anchor.number, Some(new_anchor.hash)); + // RS-2: the finalized watermark cannot survive below the window. + if let Some(fin) = &self.fin + && fin.number < from + { + self.fin = None; + } + self.ver += 1; + self.wf(); + } else { + self.reset(Anchor::new(from - 1, hash.map(str::to_string))); + } + } + + /// WP §2.6 — clear the window and re-anchor. Must be observable (OB-9). + pub fn reset(&mut self, anchor: Anchor) { + self.seg.clear(); + self.anchor = anchor; + self.fin = None; + self.ver += 1; + self.wf(); + } + + /// WP-6 — where a `ForkSignal` puts the write position. + pub fn resolve_fork(&self, hints: &[BlockRef]) -> Result { + ensure!(!hints.is_empty(), "ForkSignal with no hints"); + ensure!( + hints.windows(2).all(|w| w[0].number < w[1].number), + "ForkSignal hints are not ascending" + ); + let mut hints = hints; + if let Some(fin) = &self.fin { + let keep = hints.iter().position(|h| h.number >= fin.number); + let Some(keep) = keep else { + return Ok(ForkResolution::IntegrityFault); + }; + hints = &hints[keep..]; + if hints[0].number == fin.number && hints[0].hash != fin.hash { + return Ok(ForkResolution::IntegrityFault); + } + } + // WP-6b (direct evidence): a hint at the anchor position contradicting a known anchor + // hash puts the divergence at or below the anchor — RESET, never a replacement probe. + if let Some(anchor_hash) = &self.anchor.hash + && let Some(h) = hints.iter().find(|h| h.number == self.anchor.number) + && &h.hash != anchor_hash + { + return Ok(ForkResolution::Reset { + anchor: Anchor::new(self.anchor.number, Some(h.hash.clone())) + }); + } + let anchored = hints.iter().filter(|h| self.stored_or_anchor(h)).next_back(); + Ok(match anchored { + Some(m) => ForkResolution::Resume { + from: m.number + 1, + parent_hash: Some(m.hash.clone()) + }, + // No hint matches the stored chain. Under finality only the volatile suffix may + // be replaced (WP-6): the finalized block is common to both chains unless the + // source contradicts finality — repeated rejection here is FM-SRC-5, never RESET. + None => match &self.fin { + Some(fin) => ForkResolution::Resume { + from: fin.number + 1, + parent_hash: Some(fin.hash.clone()) + }, + // Full-window replacement *probe*; repeated WP-2 rejection at `first(D)` + // escalates to RESET per WP-6b. + None => ForkResolution::Resume { + from: self.first().unwrap_or(self.anchor.number + 1), + parent_hash: self.anchor.hash.clone() + } + } + }) + } + + /// 04 §3/§7 — the outcome class a query must produce against this state. + pub fn predict_query( + &self, + from: BlockNumber, + to: Option, + finalized_only: bool, + expected_parent: Option<&str> + ) -> Predicted { + let tip = if finalized_only { self.fin.clone() } else { self.head() }; + if let Some(first) = self.first() + && from < first + { + return Predicted::RangeUnavailable; + } + // RP-5b: the assertion names the watermark block's exact position — a definite fork + // on any chain, answered before the NO_DATA / long-poll path. + if let (Some(expected), Some(t)) = (expected_parent, &tip) + && from == t.number + 1 + && expected != t.hash + { + return Predicted::Conflict { + hints: self.conflict_hints(from - 1) + }; + } + let Some(hi) = tip.map(|t| t.number) else { + return Predicted::NoData; + }; + if from > hi { + return Predicted::NoData; + } + if let Some(expected) = expected_parent + && let Some(stored) = self.hash_at(from - 1) + && stored != expected + { + return Predicted::Conflict { + hints: self.conflict_hints(from - 1) + }; + } + Predicted::Ok { + hi: to.map_or(hi, |to| to.min(hi)) + } + } + + /// RP-11 — the ancestors a CONFLICT payload must carry, ending at `at`. + fn conflict_hints(&self, at: BlockNumber) -> Vec { + let hi = self.seg.partition_point(|b| b.number <= at); + let lo = hi.saturating_sub(crate::P_CONFLICT_WINDOW as usize); + self.seg[lo..hi].iter().map(Block::as_ref).collect() + } + + /// INV-22 — what the SUT must emit for coverage `[from, l]` under `include_all`. + /// The RP-9 coverage-end carrier and the boundary-marker exemption for *filtered* + /// queries are a documented follow-up (README, "known open questions"). + pub fn emission(&self, from: BlockNumber, l: BlockNumber, chain: &dyn Chain) -> Vec { + self.blocks_in(from, l) + .iter() + .map(|b| chain.expected_emission(b)) + .collect() + } + + pub fn blocks_in(&self, from: BlockNumber, to: BlockNumber) -> &[Block] { + if to < from { + return &[]; + } + let lo = self.seg.partition_point(|b| b.number < from); + let hi = self.seg.partition_point(|b| b.number <= to); + &self.seg[lo..hi] + } +} + +/// DEF-12 — a `Blocks` event must be an internally linked run of parents and children. +fn valid_run(blocks: &[Block]) -> Result<()> { + for w in blocks.windows(2) { + ensure!(w[1].number > w[0].number, "block run is not ascending"); + ensure!( + w[1].parent_number == w[0].number, + "block run has a hole: {} is not the parent of {}", + w[0].number, + w[1].number + ); + ensure!(w[1].parent_hash == w[0].hash, "block run is not linked"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::block_hash; + + /// A linked run over the given block numbers, which need not be consecutive. + fn chain_of(numbers: &[BlockNumber], fork_id: u32, anchor: (BlockNumber, &str)) -> Vec { + let mut out = Vec::new(); + let (mut parent_number, mut parent_hash) = (anchor.0, anchor.1.to_string()); + for &number in numbers { + let hash = block_hash(number, fork_id); + out.push(Block { + number, + hash: hash.clone(), + parent_number, + parent_hash, + timestamp_ms: 1_700_000_000_000 + number as i64 * 1000, + fork_id + }); + parent_number = number; + parent_hash = hash; + } + out + } + + /// A dense linked run on branch `fork_id`, starting at `from`. + fn run(from: BlockNumber, len: u64, fork_id: u32, parent: &str) -> Vec { + let numbers: Vec<_> = (from..from + len).collect(); + chain_of(&numbers, fork_id, (from - 1, parent)) + } + + fn model_at(start: BlockNumber, len: u64) -> (Model, Vec) { + let anchor_hash = block_hash(start - 1, 0); + let mut m = Model::new(Anchor::new(start - 1, Some(anchor_hash.clone()))); + let blocks = run(start, len, 0, &anchor_hash); + m.extend(&blocks, None).unwrap(); + (m, blocks) + } + + #[test] + fn extend_appends_and_tracks_watermarks() { + let (m, blocks) = model_at(100, 5); + assert_eq!(m.first(), Some(100)); + assert_eq!(m.head(), Some(blocks[4].as_ref())); + assert_eq!(m.next(), 105); + assert_eq!(m.span(), 5); + assert_eq!(m.fin, None); + } + + #[test] + fn extend_rejects_gaps_and_broken_linkage() { + let (mut m, _) = model_at(100, 5); + assert!( + m.extend(&run(106, 1, 0, "0xdead"), None).is_err(), + "gap must be rejected" + ); + assert!( + m.extend(&run(105, 1, 0, "0xdead"), None).is_err(), + "broken linkage must be rejected" + ); + } + + /// Solana numbers blocks by time-based slots: a slot that produced nothing leaves a hole, + /// and the window is still one chain. Numbering carries no invariant — the parent does. + #[test] + fn sparse_numbering_is_still_one_chain() { + let anchor = block_hash(99, 0); + let mut m = Model::new(Anchor::new(99, Some(anchor.clone()))); + let blocks = chain_of(&[100, 103, 104, 109], 0, (99, &anchor)); + m.extend(&blocks, None).unwrap(); + + assert_eq!(m.first(), Some(100)); + assert_eq!(m.head().unwrap().number, 109); + assert_eq!(m.span(), 4); + assert_eq!(m.hash_at(103), Some(blocks[1].hash.as_str())); + assert_eq!( + m.hash_at(101), + Some(blocks[0].hash.as_str()), + "a hole carries the preceding block's hash (DEF-16)" + ); + assert_eq!(m.hash_at(99), Some(anchor.as_str()), "at the anchor position: its hash"); + assert_eq!(m.hash_at(42), None, "below the anchor the chain hash is undefined"); + assert_eq!(m.blocks_in(101, 105).len(), 2, "103 and 104"); + + // A run whose first block is not the child of the head is a hole in the *chain* — that + // is the violation, whatever the numbers say. + let orphan = chain_of(&[112], 0, (110, &block_hash(110, 0))); + assert!(m.extend(&orphan, None).is_err()); + + // Trimming re-anchors on the last block below the cut, not on `from - 1`. + m.retain(104, None); + assert_eq!(m.first(), Some(104)); + assert_eq!(m.anchor, Anchor::new(103, Some(block_hash(103, 0)))); + + // A fork point may fall in a hole; the run attaches to the last surviving block. + let fork = chain_of(&[106, 107], 1, (104, &block_hash(104, 0))); + m.replace(105, &fork, None).unwrap(); + assert_eq!(m.head().unwrap().number, 107); + assert_eq!(m.span(), 3, "104, then the two forked blocks"); + } + + #[test] + fn anchor_with_unknown_hash_accepts_any_first_block() { + let mut m = Model::new(Anchor::new(99, None)); + assert!(m.extend(&run(100, 3, 0, "0xwhatever"), None).is_ok()); + assert_eq!(m.first(), Some(100)); + } + + #[test] + fn finalize_is_monotone_clamped_and_hash_checked() { + let (mut m, blocks) = model_at(100, 5); + + // Above the head: clamped to the head (WP §2.4). + assert_eq!(m.finalize(&BlockRef::new(200, block_hash(200, 0))), Finalize::Applied); + assert_eq!(m.fin, Some(blocks[4].as_ref())); + + // Stale report: ignored, not an error (WP-7). + assert_eq!(m.finalize(&BlockRef::new(101, block_hash(101, 0))), Finalize::Ignored); + assert_eq!(m.fin, Some(blocks[4].as_ref())); + + // A report naming a block we do not have is an integrity fault (WP-8) — this is the + // check GAP-4 says the implementation skips below the head. + let (mut m, _) = model_at(100, 5); + assert_eq!( + m.finalize(&BlockRef::new(102, block_hash(102, 7))), + Finalize::IntegrityFault + ); + assert_eq!(m.fin, None, "a faulted report must not mutate state"); + } + + #[test] + fn finalize_on_a_hole_is_an_integrity_fault() { + let anchor = block_hash(99, 0); + let mut m = Model::new(Anchor::new(99, Some(anchor.clone()))); + m.extend(&chain_of(&[100, 103, 104], 0, (99, &anchor)), None).unwrap(); + // WP §2.4: a report naming a height that is a hole in the stored chain contradicts + // the chain exactly like a hash mismatch. + assert_eq!( + m.finalize(&BlockRef::new(101, block_hash(101, 0))), + Finalize::IntegrityFault + ); + assert_eq!(m.fin, None); + } + + #[test] + fn replace_forks_above_finality_only() { + let (mut m, blocks) = model_at(100, 5); + assert_eq!(m.finalize(&BlockRef::new(102, block_hash(102, 0))), Finalize::Applied); + + // Below the finalized head — INV-13. + let bad = run(102, 2, 1, &blocks[0].hash); + assert!(m.replace(102, &bad, None).is_err()); + + // Above it — accepted, and the head moves back by one. + let good = run(103, 2, 1, &blocks[2].hash); + m.replace(103, &good, None).unwrap(); + assert_eq!(m.span(), 5); + assert_eq!(m.head().unwrap().number, 104); + assert_eq!(m.head().unwrap().hash, block_hash(104, 1)); + assert_eq!(m.fin.as_ref().unwrap().number, 102, "finality survives a fork above it"); + } + + #[test] + fn replace_below_the_window_is_rejected() { + let (mut m, _) = model_at(100, 5); + assert!( + m.replace(99, &run(99, 2, 1, "0xzz"), None).is_err(), + "INV-14 fork floor" + ); + } + + #[test] + fn retain_trims_moves_the_anchor_and_drops_stale_finality() { + let (mut m, blocks) = model_at(100, 10); + m.finalize(&BlockRef::new(101, block_hash(101, 0))); + + m.retain(105, None); + + assert_eq!(m.first(), Some(105)); + assert_eq!(m.anchor, Anchor::new(104, Some(blocks[4].hash.clone())), "INV-18"); + assert_eq!(m.fin, None, "RS-2: finality below the window is dropped"); + assert_eq!(m.head().unwrap().number, 109); + + // Trimming at the floor is a no-op. + m.retain(105, None); + assert_eq!(m.first(), Some(105)); + } + + #[test] + fn retain_below_the_window_is_a_destructive_reset() { + let (mut m, _) = model_at(100, 10); + m.retain(105, None); + // WP §2.5: lowering the bound cannot re-acquire history — the window is discarded + // and re-ingested from the new bound (alarmed, OB-9); it never widens in place. + m.retain(100, None); + assert!(m.seg.is_empty()); + assert_eq!(m.anchor, Anchor::new(99, None)); + assert_eq!(m.next(), 100); + } + + #[test] + fn retain_above_the_head_resets() { + let (mut m, _) = model_at(100, 5); + m.retain(200, Some("0xnew")); + assert!(m.seg.is_empty()); + assert_eq!(m.anchor, Anchor::new(199, Some("0xnew".to_string()))); + assert_eq!(m.next(), 200); + } + + #[test] + fn retain_with_a_conflicting_hash_self_heals() { + let (mut m, _) = model_at(100, 10); + m.retain(105, Some("0xnot-our-block")); + assert!(m.seg.is_empty(), "WP-9: a contradicting instruction re-anchors"); + assert_eq!(m.anchor, Anchor::new(104, Some("0xnot-our-block".to_string()))); + } + + #[test] + fn resolve_fork_picks_the_highest_known_hint() { + let (m, blocks) = model_at(100, 5); + let hints = vec![ + blocks[1].as_ref(), // known + blocks[2].as_ref(), // known — the highest + BlockRef::new(103, block_hash(103, 9)), // a block from another branch + BlockRef::new(104, block_hash(104, 9)), + ]; + assert_eq!( + m.resolve_fork(&hints).unwrap(), + ForkResolution::Resume { + from: 103, + parent_hash: Some(blocks[2].hash.clone()) + } + ); + } + + #[test] + fn resolve_fork_with_no_known_hint_falls_back_to_the_window_floor() { + // `fin = ⊥`: a full-window replacement probe is the only fallback left (WP-6). + let (m, _) = model_at(100, 5); + let hints = vec![ + BlockRef::new(101, block_hash(101, 9)), + BlockRef::new(102, block_hash(102, 9)), + ]; + assert_eq!( + m.resolve_fork(&hints).unwrap(), + ForkResolution::Resume { + from: 100, + parent_hash: Some(block_hash(99, 0)) + } + ); + } + + #[test] + fn resolve_fork_with_no_known_hint_replaces_the_volatile_suffix_when_finalized() { + let (mut m, blocks) = model_at(100, 5); + m.finalize(&blocks[2].as_ref()); + let hints = vec![ + BlockRef::new(103, block_hash(103, 9)), + BlockRef::new(104, block_hash(104, 9)), + ]; + assert_eq!( + m.resolve_fork(&hints).unwrap(), + ForkResolution::Resume { + from: 103, + parent_hash: Some(blocks[2].hash.clone()) + }, + "WP-6: the fallback under finality is fin + 1, not the window floor" + ); + } + + #[test] + fn resolve_fork_contradicting_the_anchor_resets() { + let (m, _) = model_at(100, 5); + let hints = vec![ + BlockRef::new(99, block_hash(99, 9)), + BlockRef::new(100, block_hash(100, 9)), + ]; + assert_eq!( + m.resolve_fork(&hints).unwrap(), + ForkResolution::Reset { + anchor: Anchor::new(99, Some(block_hash(99, 9))) + }, + "WP-6b: divergence at or below the anchor is a re-bootstrap, not a probe" + ); + } + + #[test] + fn resolve_fork_below_finality_is_an_integrity_fault() { + let (mut m, blocks) = model_at(100, 5); + m.finalize(&blocks[3].as_ref()); + let hints = vec![ + BlockRef::new(101, block_hash(101, 9)), + BlockRef::new(102, block_hash(102, 9)), + ]; + assert_eq!( + m.resolve_fork(&hints).unwrap(), + ForkResolution::IntegrityFault, + "FM-SRC-5" + ); + } + + #[test] + fn predict_query_covers_the_error_taxonomy() { + let (mut m, blocks) = model_at(100, 5); + m.finalize(&blocks[2].as_ref()); + + assert_eq!( + m.predict_query(99, None, false, None), + Predicted::RangeUnavailable, + "RP-4" + ); + assert_eq!(m.predict_query(105, None, false, None), Predicted::NoData, "RP-5"); + assert_eq!(m.predict_query(100, None, false, None), Predicted::Ok { hi: 104 }); + assert_eq!(m.predict_query(100, Some(102), false, None), Predicted::Ok { hi: 102 }); + assert_eq!( + m.predict_query(100, None, true, None), + Predicted::Ok { hi: 102 }, + "RP-6" + ); + assert_eq!(m.predict_query(103, None, true, None), Predicted::NoData, "RP-6"); + + // Anchored: the parent of 103 is block 102. + assert_eq!( + m.predict_query(103, None, false, Some(&blocks[2].hash)), + Predicted::Ok { hi: 104 } + ); + let Predicted::Conflict { hints } = m.predict_query(103, None, false, Some("0xwrong")) else { + panic!("RP-11: a mismatching expected_parent must conflict") + }; + assert_eq!(hints.last().unwrap(), &blocks[2].as_ref(), "hints end at from - 1"); + assert!( + hints.windows(2).all(|w| w[0].number < w[1].number), + "hints are ascending" + ); + } + + #[test] + fn predict_query_conflicts_at_the_tip_before_no_data() { + let (mut m, blocks) = model_at(100, 5); // head = 104 + + // RP-5b: `from = head + 1` with a mismatching assertion is a definite fork — CONFLICT, + // not a long-poll into NO_DATA. + let Predicted::Conflict { hints } = m.predict_query(105, None, false, Some("0xwrong")) else { + panic!("RP-5b: a mismatching assertion at head + 1 must conflict") + }; + assert_eq!(hints.last().unwrap(), &blocks[4].as_ref(), "hints end at the tip"); + + // One position further out the assertion names an un-ingested block: NO_DATA (RP-5). + assert_eq!(m.predict_query(106, None, false, Some("0xwrong")), Predicted::NoData); + + // Finalized-only: the watermark block is `fin`. + m.finalize(&blocks[2].as_ref()); + let Predicted::Conflict { .. } = m.predict_query(103, None, true, Some("0xwrong")) else { + panic!("RP-5b applies to the finalized watermark too") + }; + assert_eq!( + m.predict_query(103, None, true, Some(&blocks[2].hash)), + Predicted::NoData, + "a matching assertion at fin + 1 long-polls as usual" + ); + } + + #[test] + fn predict_query_evaluates_anchors_across_holes() { + // Blocks 100 and 105: an assertion for `from = 103` names `hash_at(102)`, which is + // block 100 (DEF-16) — a hole never makes the assertion unevaluable (GAP-21 is the + // implementation failing exactly this). + let anchor = block_hash(99, 0); + let mut m = Model::new(Anchor::new(99, Some(anchor.clone()))); + let blocks = chain_of(&[100, 105], 0, (99, &anchor)); + m.extend(&blocks, None).unwrap(); + + assert_eq!( + m.predict_query(103, None, false, Some(&blocks[0].hash)), + Predicted::Ok { hi: 105 } + ); + let Predicted::Conflict { hints } = m.predict_query(103, None, false, Some("0xwrong")) else { + panic!("RP-11: the assertion is evaluated against the preceding block across a hole") + }; + assert_eq!( + hints.last().unwrap(), + &blocks[0].as_ref(), + "hints end at the nearest stored position below from - 1" + ); + } +} diff --git a/crates/hotblocks-harness/src/sim.rs b/crates/hotblocks-harness/src/sim.rs new file mode 100644 index 00000000..3c95bedb --- /dev/null +++ b/crates/hotblocks-harness/src/sim.rs @@ -0,0 +1,628 @@ +//! The source simulator — a scriptable chain generator speaking the source side of the binding +//! (spec 13 §7, DEF-12). It is the truth of a run: whatever it served is what the service must +//! converge to. +//! +//! Two facts about the service shape it, and undoing either silently breaks every test: +//! +//! 1. It commits a batch only when the source's response *ends* (`MaybeOnHead`), so each +//! `/stream` request serves what exists now and closes. Hold the stream open and the blocks +//! stay buffered and invisible, forever. +//! 2. It re-requests the instant a response ends, so a request with nothing to serve is held for +//! `poll_timeout`. Answer `204` at once and the ingest loop spins at full CPU. + +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{Arc, Mutex}, + time::{Duration, Instant} +}; + +use anyhow::{Context, Result, ensure}; +use axum::{ + Json, Router, + body::{Body, Bytes}, + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post} +}; +use serde::Deserialize; +use serde_json::json; +use tokio::sync::watch; + +use crate::{ + P_CONFLICT_WINDOW, + chain::Chain, + types::{Block, BlockNumber, BlockRef, Rng, block_hash} +}; + +pub struct SimConfig { + pub datasets: Vec, + /// How long a `/stream` request with nothing to serve is held before answering `NO_DATA`. + pub poll_timeout: Duration +} + +pub struct SimDataset { + pub id: String, + pub chain: Arc, + pub start_block: BlockNumber, + pub base_timestamp_ms: i64, + pub numbering: Numbering +} + +/// How block numbers advance. Solana numbers blocks by time-based *slots*, and a slot that +/// produces nothing leaves a hole — so the numbering carries no invariant, the parent pointer +/// does (INV-1/INV-2). evm and hyperliquid number densely. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Numbering { + #[default] + Dense, + /// Deterministic holes: 0 to 2 slots skipped between blocks. + Sparse +} + +impl Numbering { + fn skipped_after(self, number: BlockNumber) -> u64 { + match self { + Numbering::Dense => 0, + Numbering::Sparse => Rng::new(number).below(3) + } + } +} + +/// Source-side faults the service must survive (FM-SRC-*, CT-4/CT-9). +/// +/// FUTURE: the rest of the repertoire lands here — stalls, disconnects, malformed JSON, broken +/// linkage, regressive watermarks, conflicting hints. +#[derive(Clone, Copy, Debug, Default)] +pub struct SimFaults { + /// Serve a JSONL body whose final record is not newline-terminated. + pub unterminated_final_line: bool +} + +/// Counters a test can assert on (how the SUT actually drove the source). +#[derive(Clone, Copy, Debug, Default)] +pub struct SimStats { + pub stream_requests: u64, + pub blocks_served: u64, + pub fork_signals: u64, + pub no_data: u64, + /// The service asked for history below the source's first block — always a defect. + pub below_history: u64 +} + +pub struct SourceSim { + shared: Arc, + addr: SocketAddr, + server: tokio::task::JoinHandle<()> +} + +struct Shared { + datasets: Mutex>, + /// Bumped after every mutation; wakes the long-polling `/stream` handlers. + bump: watch::Sender, + poll_timeout: Duration +} + +impl SourceSim { + pub async fn start(cfg: SimConfig) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + + let datasets = cfg + .datasets + .into_iter() + .map(|d| (d.id.clone(), DatasetSim::new(d))) + .collect(); + + let shared = Arc::new(Shared { + datasets: Mutex::new(datasets), + bump: watch::channel(0).0, + poll_timeout: cfg.poll_timeout + }); + + let app = Router::new() + .route("/{ds}/stream", post(stream)) + .route("/{ds}/head", get(head)) + .route("/{ds}/finalized-head", get(finalized_head)) + .with_state(shared.clone()); + + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + Ok(Self { shared, addr, server }) + } + + /// The `data_sources` URL — the service appends `/stream`, `/head`, ... to it. + pub fn base_url(&self, dataset: &str) -> String { + format!("http://{}/{}", self.addr, dataset) + } + + /// The hash of the block below the dataset's first block (DEF-7). + pub fn anchor_hash(&self, dataset: &str) -> String { + self.with(dataset, |d| d.anchor_hash.clone()) + } + + /// Extend the canonical chain by `n` blocks; returns them so the model can EXTEND too. + pub fn produce(&self, dataset: &str, n: u32) -> Vec { + let blocks = self.with(dataset, |d| d.produce(n)); + self.bump(); + blocks + } + + /// Replace the canonical suffix at `from` with `len` freshly-minted blocks (a reorg). + pub fn fork(&self, dataset: &str, from: BlockNumber, len: u32) -> Result> { + let blocks = self.try_with(dataset, |d| d.fork(from, len))?; + self.bump(); + Ok(blocks) + } + + /// Declare `number` (and everything below it) final. + pub fn finalize(&self, dataset: &str, number: BlockNumber) -> Result { + let r = self.try_with(dataset, |d| d.finalize(number))?; + self.bump(); + Ok(r) + } + + pub fn tip(&self, dataset: &str) -> Option { + self.with(dataset, |d| d.chain.last().map(Block::as_ref)) + } + + pub fn stats(&self, dataset: &str) -> SimStats { + self.with(dataset, |d| d.stats) + } + + /// Turn a source-side fault on or off (FM-SRC-*). + pub fn inject_fault(&self, dataset: &str, f: impl FnOnce(&mut SimFaults)) { + self.with(dataset, |d| f(&mut d.faults)); + } + + fn with(&self, dataset: &str, f: impl FnOnce(&mut DatasetSim) -> T) -> T { + let mut guard = self.shared.datasets.lock().expect("simulator state is poisoned"); + let d = guard + .get_mut(dataset) + .unwrap_or_else(|| panic!("unknown dataset '{dataset}'")); + f(d) + } + + fn try_with(&self, dataset: &str, f: impl FnOnce(&mut DatasetSim) -> Result) -> Result { + self.with(dataset, f) + } + + fn bump(&self) { + self.shared.bump.send_modify(|v| *v += 1); + } +} + +impl Drop for SourceSim { + fn drop(&mut self) { + self.server.abort(); + } +} + +struct DatasetSim { + chain_kind: Arc, + start: BlockNumber, + base_ts: i64, + anchor_hash: String, + numbering: Numbering, + chain: Vec, + fin: Option, + fork_id: u32, + next_fork_id: u32, + faults: SimFaults, + stats: SimStats +} + +enum Reply { + Blocks { blocks: Vec, fin: Option }, + Fork(Vec), + NoData(Option) +} + +impl DatasetSim { + fn new(cfg: SimDataset) -> Self { + Self { + chain_kind: cfg.chain, + start: cfg.start_block, + base_ts: cfg.base_timestamp_ms, + anchor_hash: block_hash(cfg.start_block - 1, 0), + numbering: cfg.numbering, + chain: Vec::new(), + fin: None, + fork_id: 0, + next_fork_id: 1, + faults: SimFaults::default(), + stats: SimStats::default() + } + } + + fn tip_number(&self) -> Option { + self.chain.last().map(|b| b.number) + } + + /// The number the next produced block will carry. Under [`Numbering::Sparse`] it skips + /// slots, so it is *not* `tip + 1`. + fn next_number(&self) -> BlockNumber { + match self.chain.last() { + None => self.start, + Some(last) => last.number + 1 + self.numbering.skipped_after(last.number) + } + } + + /// The anchor, then every block of the canonical chain. `None` for a slot that produced none. + fn hash_at(&self, n: BlockNumber) -> Option<&str> { + if n + 1 == self.start { + return Some(&self.anchor_hash); + } + self.chain + .binary_search_by_key(&n, |b| b.number) + .ok() + .map(|i| self.chain[i].hash.as_str()) + } + + /// The last canonical hash at or below `n`. Unlike [`Self::hash_at`], this crosses empty + /// slots on sparse-numbered chains. Positions above the tip remain unknown. + fn chain_hash_at(&self, n: BlockNumber) -> Option<&str> { + if self.chain.last().is_some_and(|tip| n > tip.number) { + return None; + } + let i = self.chain.partition_point(|b| b.number <= n); + if i > 0 { + return Some(&self.chain[i - 1].hash); + } + (n.checked_add(1) == Some(self.start)).then_some(self.anchor_hash.as_str()) + } + + fn produce(&mut self, n: u32) -> Vec { + let mut out = Vec::with_capacity(n as usize); + for _ in 0..n { + let number = self.next_number(); + let (parent_number, parent_hash) = match self.chain.last() { + Some(last) => (last.number, last.hash.clone()), + None => (self.start - 1, self.anchor_hash.clone()) + }; + let block = Block { + number, + hash: block_hash(number, self.fork_id), + parent_number, + parent_hash, + timestamp_ms: self.base_ts + (number - self.start) as i64 * 1000, + fork_id: self.fork_id + }; + self.chain.push(block.clone()); + out.push(block); + } + out + } + + fn fork(&mut self, from: BlockNumber, len: u32) -> Result> { + ensure!( + from >= self.start, + "fork at {from} is below the source's first block {}", + self.start + ); + ensure!(from <= self.next_number(), "fork at {from} is above the source's chain"); + ensure!( + self.fin.as_ref().is_none_or(|f| f.number < from), + "the script forks at or below the source's own finalized head — an equivocating source \ + belongs to the CT-4 fault corpus, not to a well-formed script" + ); + let keep = self.chain.partition_point(|b| b.number < from); + self.chain.truncate(keep); + self.fork_id = self.next_fork_id; + self.next_fork_id += 1; + Ok(self.produce(len)) + } + + fn finalize(&mut self, number: BlockNumber) -> Result { + let hash = self + .hash_at(number) + .with_context(|| format!("cannot finalize block {number}: the source does not have it"))? + .to_string(); + let r = BlockRef::new(number, hash); + self.fin = Some(r.clone()); + Ok(r) + } + + /// RP-11/DEF-12 — the ancestors of the divergence point, ascending. + fn hints_ending_at(&self, n: BlockNumber) -> Vec { + let hi = self.chain.partition_point(|b| b.number <= n); + if hi == 0 { + // Nothing of ours at or below the divergence: all we know is the anchor. + return vec![BlockRef::new(self.start - 1, self.anchor_hash.clone())]; + } + let lo = hi.saturating_sub(P_CONFLICT_WINDOW as usize); + self.chain[lo..hi].iter().map(Block::as_ref).collect() + } + + fn respond(&mut self, req: &StreamReq) -> Reply { + self.stats.stream_requests += 1; + + let parent_pos = req.from_block.saturating_sub(1); + if let Some(asserted) = &req.parent_block_hash { + match self.chain_hash_at(parent_pos) { + Some(known) if known == asserted => {} + Some(_) => { + self.stats.fork_signals += 1; + return Reply::Fork(self.hints_ending_at(parent_pos)); + } + None => { + // Above our tip: it claims blocks we do not have — disagree, hint at our tip. + if let Some(tip) = self.tip_number() + && parent_pos > tip + { + self.stats.fork_signals += 1; + return Reply::Fork(self.hints_ending_at(tip)); + } + // Below our history: unverifiable, serve as-is (the `⊥`-anchor case). + } + } + } + + if req.from_block < self.start { + self.stats.below_history += 1; + return Reply::NoData(self.fin.clone()); + } + let Some(tip) = self.tip_number() else { + self.stats.no_data += 1; + return Reply::NoData(self.fin.clone()); + }; + if req.from_block > tip { + self.stats.no_data += 1; + return Reply::NoData(self.fin.clone()); + } + + let lo = self.chain.partition_point(|b| b.number < req.from_block); + let blocks = self.chain[lo..].to_vec(); + self.stats.blocks_served += blocks.len() as u64; + Reply::Blocks { + blocks, + fin: self.fin.clone() + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StreamReq { + from_block: BlockNumber, + #[serde(default)] + parent_block_hash: Option +} + +async fn stream(State(shared): State>, Path(ds): Path, body: Bytes) -> Response { + let req: StreamReq = match serde_json::from_slice(&body) { + Ok(req) => req, + Err(err) => return (StatusCode::BAD_REQUEST, format!("bad stream request: {err}")).into_response() + }; + + let mut bump = shared.bump.subscribe(); + let deadline = Instant::now() + shared.poll_timeout; + + loop { + // Mark the version seen *before* reading the state, or a racing mutation is lost. + bump.borrow_and_update(); + + let reply = { + let mut guard = shared.datasets.lock().expect("simulator state is poisoned"); + let Some(d) = guard.get_mut(&ds) else { + return (StatusCode::NOT_FOUND, format!("unknown dataset '{ds}'")).into_response(); + }; + match d.respond(&req) { + Reply::Blocks { blocks, fin } => { + let mut body = String::new(); + for block in &blocks { + body.push_str(&d.chain_kind.source_block(block).to_string()); + body.push('\n'); + } + // A fault: well-formed sources terminate every record. + if d.faults.unterminated_final_line { + body.pop(); + } + Some(jsonl(body, fin.as_ref())) + } + Reply::Fork(hints) => { + Some((StatusCode::CONFLICT, Json(json!({ "previousBlocks": hints }))).into_response()) + } + Reply::NoData(fin) => { + if Instant::now() >= deadline { + Some(no_data(fin.as_ref())) + } else { + None + } + } + } + }; + + if let Some(reply) = reply { + return reply; + } + + let wait = deadline.saturating_duration_since(Instant::now()); + let _ = tokio::time::timeout(wait, bump.changed()).await; + } +} + +async fn head(State(shared): State>, Path(ds): Path) -> Response { + watermark(&shared, &ds, |d| d.chain.last().map(Block::as_ref)) +} + +async fn finalized_head(State(shared): State>, Path(ds): Path) -> Response { + watermark(&shared, &ds, |d| d.fin.clone()) +} + +fn watermark(shared: &Shared, ds: &str, f: impl FnOnce(&DatasetSim) -> Option) -> Response { + let guard = shared.datasets.lock().expect("simulator state is poisoned"); + match guard.get(ds) { + Some(d) => Json(f(d)).into_response(), + None => (StatusCode::NOT_FOUND, format!("unknown dataset '{ds}'")).into_response() + } +} + +fn jsonl(body: String, fin: Option<&BlockRef>) -> Response { + finality_headers(Response::builder().status(StatusCode::OK), fin) + .header("content-type", "text/plain") + .body(Body::from(body)) + .expect("a well-formed response") +} + +fn no_data(fin: Option<&BlockRef>) -> Response { + finality_headers(Response::builder().status(StatusCode::NO_CONTENT), fin) + .body(Body::empty()) + .expect("a well-formed response") +} + +/// All-or-nothing: the client rejects a response carrying one half of the pair. +fn finality_headers(builder: axum::http::response::Builder, fin: Option<&BlockRef>) -> axum::http::response::Builder { + match fin { + Some(fin) => builder + .header("x-sqd-finalized-head-number", fin.number) + .header("x-sqd-finalized-head-hash", fin.hash.as_str()), + None => builder + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::chain::HlFills; + + const DS: &str = "ds0"; + const START: BlockNumber = 1_000; + + async fn sim() -> SourceSim { + SourceSim::start(SimConfig { + datasets: vec![SimDataset { + id: DS.to_string(), + chain: Arc::new(HlFills), + start_block: START, + base_timestamp_ms: 1_760_000_000_000, + numbering: Numbering::Dense + }], + poll_timeout: Duration::from_millis(100) + }) + .await + .unwrap() + } + + /// The service's data client is the only consumer that matters, so the simulator is tested + /// against what that client actually does with a response — not against what it looks like. + #[tokio::test] + async fn serves_a_terminated_jsonl_stream_with_finality_headers() { + let sim = sim().await; + sim.produce(DS, 3); + sim.finalize(DS, START + 1).unwrap(); + + let res = reqwest::Client::new() + .post(format!("{}/stream", sim.base_url(DS))) + .json(&json!({"fromBlock": START, "parentBlockHash": sim.anchor_hash(DS)})) + .send() + .await + .unwrap(); + + assert_eq!(res.status(), 200); + assert_eq!(res.headers()["x-sqd-finalized-head-number"], "1001"); + assert!(res.headers().contains_key("x-sqd-finalized-head-hash")); + // The client reads the body as a stream and only commits when it *ends*: a response + // without a length that terminates is a service that never sees the blocks. + assert!(res.content_length().is_some(), "the body must be length-delimited"); + + let body = res.text().await.unwrap(); + let lines: Vec<&str> = body.lines().collect(); + assert_eq!(lines.len(), 3, "one JSON object per block"); + let first: serde_json::Value = serde_json::from_str(lines[0]).unwrap(); + assert_eq!(first["header"]["number"], 1000); + assert_eq!(first["header"]["parentHash"], sim.anchor_hash(DS)); + + // Every record is newline-terminated, the last one included. This is not cosmetic: an + // unterminated final record panics the service's line reader (GAP-19), so a simulator + // that got this wrong would fail every test for the wrong reason. + assert!(body.ends_with('\n')); + } + + #[tokio::test] + async fn disagrees_with_a_wrong_parent_by_a_fork_signal() { + let sim = sim().await; + sim.produce(DS, 5); + + let res = reqwest::Client::new() + .post(format!("{}/stream", sim.base_url(DS))) + .json(&json!({"fromBlock": START + 3, "parentBlockHash": "0xnot-our-block"})) + .send() + .await + .unwrap(); + + assert_eq!(res.status(), 409); + let body: serde_json::Value = res.json().await.unwrap(); + let hints = body["previousBlocks"].as_array().unwrap(); + assert!(!hints.is_empty(), "the client rejects an empty hint list"); + assert_eq!( + hints.last().unwrap()["number"], + START + 2, + "hints end at the disputed position" + ); + } + + #[tokio::test] + async fn validates_parent_hash_across_a_sparse_numbering_hole() { + let sim = SourceSim::start(SimConfig { + datasets: vec![SimDataset { + id: DS.to_string(), + chain: Arc::new(HlFills), + start_block: START, + base_timestamp_ms: 1_760_000_000_000, + numbering: Numbering::Sparse + }], + poll_timeout: Duration::from_millis(100) + }) + .await + .unwrap(); + let blocks = sim.produce(DS, 10); + let [parent, child] = blocks + .windows(2) + .find(|w| w[1].number > w[0].number + 1) + .expect("the deterministic sparse sequence contains a hole") + else { + unreachable!() + }; + + let wrong = reqwest::Client::new() + .post(format!("{}/stream", sim.base_url(DS))) + .json(&json!({"fromBlock": child.number, "parentBlockHash": "0xwrong"})) + .send() + .await + .unwrap(); + assert_eq!(wrong.status(), StatusCode::CONFLICT); + + let correct = reqwest::Client::new() + .post(format!("{}/stream", sim.base_url(DS))) + .json(&json!({"fromBlock": child.number, "parentBlockHash": parent.hash})) + .send() + .await + .unwrap(); + assert_eq!(correct.status(), StatusCode::OK); + } + + #[tokio::test] + async fn holds_a_request_it_cannot_serve_and_then_answers_no_data() { + let sim = sim().await; + let started = Instant::now(); + + let res = reqwest::Client::new() + .post(format!("{}/stream", sim.base_url(DS))) + .json(&json!({"fromBlock": START})) + .send() + .await + .unwrap(); + + assert_eq!(res.status(), 204); + assert!( + started.elapsed() >= Duration::from_millis(100), + "the request must long-poll" + ); + } +} diff --git a/crates/hotblocks-harness/src/sut.rs b/crates/hotblocks-harness/src/sut.rs new file mode 100644 index 00000000..876eb1d0 --- /dev/null +++ b/crates/hotblocks-harness/src/sut.rs @@ -0,0 +1,322 @@ +//! The system under test — the real `sqd-hotblocks` binary, driven as a black box. +//! +//! Owning the process, not a library handle, is what makes the crash-recovery (CT-2) and +//! shutdown (GAP-17) classes expressible at all. + +use std::{ + fs::{File, OpenOptions}, + io::Write, + mem::MaybeUninit, + os::unix::process::CommandExt, + path::{Path, PathBuf}, + process::{ExitStatus, Stdio}, + time::{Duration, Instant} +}; + +use anyhow::{Context, Result, bail, ensure}; +use tempfile::TempDir; +use tokio::process::{Child, Command}; + +const TARGET_NOFILE_LIMIT: libc::rlim_t = 8_192; +const MIN_NOFILE_LIMIT: libc::rlim_t = 1_024; + +/// The dataset-config retention strategies (see `hotblocks/src/dataset_config.rs`). +#[derive(Clone, Debug)] +pub enum Retention { + /// Ingest starts exactly here — the deterministic choice for structural tests. + FromBlock { number: u64, parent_hash: Option }, + /// A moving window of `n` blocks; the start position is probed from the source's head. + Head(u64), + /// Driven at runtime via `POST /datasets/{id}/retention` (External, DEF-9). The dataset + /// ingests nothing until the first instruction arrives. + Api, + /// Never trim. On an empty database this dataset never ingests anything at all. + None +} + +#[derive(Clone, Debug)] +pub struct DatasetSpec { + pub id: String, + pub kind: String, + pub retention: Retention, + pub sources: Vec +} + +#[derive(Clone, Debug)] +pub struct SutConfig { + /// Path to the built binary — tests get it from `env!("CARGO_BIN_EXE_sqd-hotblocks")`. + pub bin: PathBuf, + pub datasets: Vec, + pub args: Vec, + pub rust_log: String, + /// Generous by design: serving is gated on full initialization (GAP-7). A test that cares + /// measures [`Sut::last_startup`] instead. + pub startup_timeout: Duration +} + +pub struct Sut { + cfg: SutConfig, + dir: TempDir, + port: u16, + child: Option, + /// How long the last boot took to accept connections (SLI-5). + pub last_startup: Duration +} + +#[derive(Debug)] +pub struct ShutdownReport { + pub status: ExitStatus, + pub took: Duration +} + +impl Sut { + /// Write the config, start the process, wait until it accepts connections. + pub async fn start(cfg: SutConfig) -> Result { + let dir = TempDir::new().context("failed to create the SUT working directory")?; + let mut sut = Self { + port: free_port()?, + cfg, + dir, + child: None, + last_startup: Duration::ZERO + }; + sut.write_config()?; + sut.boot().await?; + Ok(sut) + } + + pub fn base_url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + pub fn db_dir(&self) -> PathBuf { + self.dir.path().join("db") + } + + pub fn log_path(&self) -> PathBuf { + self.dir.path().join("sut.log") + } + + /// Boot a stopped process again against the same database — the CT-2 restart primitive. + pub async fn restart(&mut self) -> Result<()> { + if self.child.is_some() { + self.stop().await?; + } + self.boot().await + } + + /// SIGKILL: the process loses everything not committed (CN-6, `P-DUR-PROCESS = 0`). + pub async fn crash(&mut self) -> Result<()> { + let Some(child) = self.child.as_mut() else { + return Ok(()); + }; + child.kill().await.context("failed to kill the SUT")?; + self.child = None; + Ok(()) + } + + /// SIGTERM and wait for the drain-and-exit (LIV-12, `P-SHUTDOWN`). + pub async fn stop(&mut self) -> Result { + let Some(mut child) = self.child.take() else { + bail!("the SUT is not running") + }; + let pid = child.id().context("the SUT has no pid")? as i32; + let started = Instant::now(); + + // SAFETY: `pid` names a child of this process that has not been reaped. + unsafe { libc::kill(pid, libc::SIGTERM) }; + + match tokio::time::timeout(Duration::from_secs(60), child.wait()).await { + Ok(status) => Ok(ShutdownReport { + status: status.context("failed to wait for the SUT")?, + took: started.elapsed() + }), + Err(_) => { + child.kill().await.ok(); + bail!("the SUT did not exit within 60s of SIGTERM") + } + } + } + + /// The tail of the service log, minus the HTTP access log — the harness's own polling makes + /// that the loudest thing in there and never the interesting one. + pub fn log_tail(&self, lines: usize) -> String { + let Ok(log) = std::fs::read_to_string(self.log_path()) else { + return "".to_string(); + }; + let all: Vec<&str> = log + .lines() + .filter(|line| !line.contains(r#""target":"http_request""#)) + .collect(); + all[all.len().saturating_sub(lines)..].join("\n") + } + + async fn boot(&mut self) -> Result<()> { + let started = Instant::now(); + let log = OpenOptions::new() + .create(true) + .append(true) + .open(self.log_path()) + .context("failed to open the SUT log")?; + + let mut command = Command::new(&self.cfg.bin); + command + .arg("--datasets") + .arg(self.config_path()) + .arg("--db") + .arg(self.db_dir()) + .arg("--port") + .arg(self.port.to_string()) + .args(&self.cfg.args) + .env("RUST_LOG", &self.cfg.rust_log) + .stdin(Stdio::null()) + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)) + .kill_on_drop(true); + configure_nofile_limit(&mut command)?; + + let child = command + .spawn() + .with_context(|| format!("failed to spawn {}", self.cfg.bin.display()))?; + + self.child = Some(child); + self.await_ready(started).await?; + self.last_startup = started.elapsed(); + Ok(()) + } + + async fn await_ready(&mut self, started: Instant) -> Result<()> { + let http = reqwest::Client::builder().timeout(Duration::from_secs(2)).build()?; + let deadline = started + self.cfg.startup_timeout; + loop { + if let Some(status) = self.child.as_mut().expect("just spawned").try_wait()? { + bail!( + "the SUT exited during startup with {status}\n--- log tail ---\n{}", + self.log_tail(60) + ); + } + if let Ok(res) = http.get(self.base_url()).send().await + && res.status().is_success() + { + return Ok(()); + } + if Instant::now() > deadline { + bail!( + "the SUT did not accept connections within {:?} (LIV-5a)\n--- log tail ---\n{}", + self.cfg.startup_timeout, + self.log_tail(60) + ); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + fn config_path(&self) -> PathBuf { + self.dir.path().join("datasets.yaml") + } + + /// Renders `BTreeMap` as the service's YAML loader expects it + /// (`serde_yaml` with `singleton_map_recursive`; see `hotblocks/src/dataset_config.rs`). + fn write_config(&self) -> Result<()> { + let mut yaml = String::new(); + for ds in &self.cfg.datasets { + yaml.push_str(&format!("{}:\n", ds.id)); + yaml.push_str(&format!(" kind: {}\n", ds.kind)); + match &ds.retention { + Retention::FromBlock { number, parent_hash } => { + yaml.push_str(" retention_strategy:\n FromBlock:\n"); + yaml.push_str(&format!(" number: {number}\n")); + match parent_hash { + Some(hash) => yaml.push_str(&format!(" parent_hash: \"{hash}\"\n")), + None => yaml.push_str(" parent_hash: null\n") + } + } + Retention::Head(n) => yaml.push_str(&format!(" retention_strategy:\n Head: {n}\n")), + Retention::Api => yaml.push_str(" retention_strategy: Api\n"), + Retention::None => yaml.push_str(" retention_strategy: None\n") + } + yaml.push_str(" data_sources:\n"); + for src in &ds.sources { + yaml.push_str(&format!(" - \"{src}\"\n")); + } + } + File::create(self.config_path())? + .write_all(yaml.as_bytes()) + .context("failed to write the dataset config") + } +} + +/// Raise the SUT's soft descriptor limit without changing the test runner or the hard limit. +/// EVM chunk processors keep one temporary file per physical Arrow buffer, so the macOS default +/// of 256 descriptors is insufficient even for a single small chunk. +fn configure_nofile_limit(command: &mut Command) -> Result<()> { + let mut limits = MaybeUninit::::uninit(); + // SAFETY: `limits` points to writable storage for one `rlimit`, and `getrlimit` initializes + // it on success before `assume_init` is called. + if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, limits.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()).context("failed to read RLIMIT_NOFILE"); + } + // SAFETY: the successful `getrlimit` call above initialized `limits`. + let limits = unsafe { limits.assume_init() }; + let child_soft_limit = limits.rlim_max.min(TARGET_NOFILE_LIMIT); + ensure!( + child_soft_limit >= MIN_NOFILE_LIMIT, + "the SUT needs RLIMIT_NOFILE >= {MIN_NOFILE_LIMIT}, but the hard limit is {}", + limits.rlim_max + ); + + // SAFETY: the closure only calls the async-signal-safe `setrlimit` syscall between `fork` + // and `exec`, does not access shared state, and preserves the inherited hard limit. + unsafe { + command.as_std_mut().pre_exec(move || { + let child_limits = libc::rlimit { + rlim_cur: child_soft_limit, + rlim_max: limits.rlim_max + }; + if libc::setrlimit(libc::RLIMIT_NOFILE, &child_limits) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(()) +} + +impl Drop for Sut { + fn drop(&mut self) { + // The child dies with the handle (`kill_on_drop`); a failing test needs the log. + if std::thread::panicking() { + eprintln!( + "\n--- SUT log tail ({}) ---\n{}\n", + self.log_path().display(), + self.log_tail(80) + ); + } + } +} + +/// Ask the kernel for a free port and hand it to the child. The gap before the child binds it is +/// a race in principle, not in practice. +fn free_port() -> Result { + let listener = std::net::TcpListener::bind("127.0.0.1:0").context("no free port")?; + Ok(listener.local_addr()?.port()) +} + +impl SutConfig { + pub fn new(bin: impl AsRef, datasets: Vec) -> Self { + Self { + bin: bin.as_ref().to_path_buf(), + datasets, + args: vec![ + "--data-cache-size".into(), + "16".into(), + // Direct I/O differs across the platforms tests run on and nothing structural + // depends on it. CT-6/CT-7 must drop this flag — they measure the engine. + "--rocksdb-disable-direct-io".into(), + ], + rust_log: "info".into(), + startup_timeout: Duration::from_secs(60) + } + } +} diff --git a/crates/hotblocks-harness/src/types.rs b/crates/hotblocks-harness/src/types.rs new file mode 100644 index 00000000..7f1b6ac4 --- /dev/null +++ b/crates/hotblocks-harness/src/types.rs @@ -0,0 +1,115 @@ +//! Primitives shared by the simulator, the model and the driver (spec 02). + +use serde::{Deserialize, Serialize}; + +pub type BlockNumber = u64; + +/// DEF-3 — a block reference `⟨number, hash⟩`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlockRef { + pub number: BlockNumber, + pub hash: String +} + +impl BlockRef { + pub fn new(number: BlockNumber, hash: impl Into) -> Self { + Self { + number, + hash: hash.into() + } + } +} + +impl std::fmt::Display for BlockRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}#{}", self.number, &self.hash[..self.hash.len().min(10)]) + } +} + +/// DEF-4. Only the structural part is materialized: the payload is a pure function of +/// `(number, fork_id)`, so the simulator and the model derive identical items independently. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Block { + pub number: BlockNumber, + pub hash: String, + /// The number of the parent block, which is *not* always `number - 1`: Solana numbers + /// blocks by time-based slots and a slot that produced nothing leaves a hole. This is the + /// pointer that makes the window one chain; the numbering does not. + pub parent_number: BlockNumber, + pub parent_hash: String, + pub timestamp_ms: i64, + /// The branch it was minted on: a fork mints a fresh id, so replaced blocks differ in hash. + pub fork_id: u32 +} + +impl Block { + pub fn as_ref(&self) -> BlockRef { + BlockRef::new(self.number, self.hash.clone()) + } +} + +/// DEF-7. `hash = None` is the spec's `⊥`: linkage below the window is unverifiable. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Anchor { + pub number: BlockNumber, + pub hash: Option +} + +impl Anchor { + pub fn new(number: BlockNumber, hash: Option) -> Self { + Self { number, hash } + } +} + +/// Deterministic hash of `(number, fork_id)` — hashes are opaque strings compared by equality (DEF-2). +pub fn block_hash(number: BlockNumber, fork_id: u32) -> String { + let mut s = + splitmix64(number.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ u64::from(fork_id).wrapping_mul(0xD1B5_4A32_D192_ED03)); + let mut out = String::with_capacity(66); + out.push_str("0x"); + for _ in 0..4 { + s = splitmix64(s); + out.push_str(&format!("{s:016x}")); + } + out +} + +/// Deterministic PRNG for randomized scripts. +#[derive(Clone, Debug)] +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Self(seed) + } + + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + splitmix64(self.0) + } + + /// Uniform-ish in `[0, n)`; `n` must be non-zero. + pub fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } +} + +fn splitmix64(x: u64) -> u64 { + let mut z = x; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hashes_are_deterministic_and_branch_dependent() { + assert_eq!(block_hash(100, 0), block_hash(100, 0)); + assert_ne!(block_hash(100, 0), block_hash(100, 1)); + assert_ne!(block_hash(100, 0), block_hash(101, 0)); + assert_eq!(block_hash(100, 0).len(), 66); + } +} diff --git a/crates/hotblocks/Cargo.toml b/crates/hotblocks/Cargo.toml index 37667ae6..302aab88 100644 --- a/crates/hotblocks/Cargo.toml +++ b/crates/hotblocks/Cargo.toml @@ -34,5 +34,10 @@ tracing = { workspace = true, features = ["valuable"] } tracing-subscriber = { workspace = true, features = ["env-filter", "json", "valuable"] } url = { workspace = true, features = ["serde"] } +[dev-dependencies] +anyhow = { workspace = true } +sqd-hotblocks-harness = { path = "../hotblocks-harness" } +tokio = { workspace = true, features = ["full"] } + [lints] -workspace = true +workspace = true diff --git a/crates/hotblocks/spec/01-overview.md b/crates/hotblocks/spec/01-overview.md new file mode 100644 index 00000000..9d4d5d06 --- /dev/null +++ b/crates/hotblocks/spec/01-overview.md @@ -0,0 +1,101 @@ +# 01 — Overview + +## 1. What Hotblocks is + +Hotblocks is a **real-time block database**. For each configured **dataset** it maintains a +bounded, contiguous, fork-aware **window** of a blockchain — typically the most recent +portion, close to the chain tip — and serves **range queries** over the blocks and their +contents (transactions, logs, instructions, etc., depending on the chain family). + +It is the "hot" complement to an archival store: archives hold the deep, immutable history; +Hotblocks holds the volatile tip where blocks arrive continuously, forks and rollbacks +happen, and freshness is measured in fractions of a second. + +The system is a single service instance hosting many datasets concurrently (tens of +datasets is the normal operating point). Datasets are independent chains: different +networks, different chain families ("kinds"), different retention policies. + +## 2. Actors + +| Actor | Role | +|---|---| +| **Block sources** | Upstream providers that stream blocks, fork signals, and finality signals for a dataset. Several redundant sources MAY feed one dataset. | +| **Query clients** | Consumers issuing ranged queries, watermark reads, and continuation/fork-recovery round trips. Clients are typically automated pollers following the chain tip. | +| **Retention controller** | An external agent that (for datasets in the *External* retention mode) instructs the service which lower bound of history to keep. | +| **Operator** | Configures datasets (kind, sources, retention policy), deploys, restarts, monitors. | + +## 3. Design goals + +- **G1 — Freshness.** A block published by a healthy source becomes queryable quickly; the + visible head tracks the source tip with bounded lag ([LIV-1](07-liveness.md), SLI-1). +- **G2 — Fork correctness.** The stored window is always a single consistent chain; forks + replace a suffix atomically; clients anchored by a parent hash can never silently read + across a reorg ([INV-1..3](06-invariants.md), [INV-23](06-invariants.md)). +- **G3 — Bounded resources.** Storage, memory, and startup work are bounded functions of + configured retention — never of total chain history or uptime ([RS-6](09-retention-and-space.md), PF-*). +- **G4 — Isolation.** Datasets are logically independent; one dataset's load, faults, or + maintenance must not corrupt — and must not indefinitely stall — another + ([INV-35/36](06-invariants.md), [LIV-8](07-liveness.md)). +- **G5 — Self-healing.** Crashes, source misbehavior, and transient overload are absorbed + without operator intervention; unrecoverable conditions are surfaced loudly rather than + retried silently forever ([INV-40..44](06-invariants.md), [FM-*](08-failure-model.md)). +- **G6 — Testability.** Every guarantee in this spec is stated so that a black-box harness + with a scripted source and client can verify it ([12](12-conformance-tdd.md)). + +## 4. Non-goals (explicit) + +- **NG1 — Not an archive.** History outside the retention window is not available and its + absence is a defined error, not a defect. **Retention dominates finality**: finalized + blocks are deleted when the window moves past them ([RS-2](09-retention-and-space.md)). +- **NG2 — No multi-fork serving.** At any instant a dataset presents exactly one chain. + Competing forks are never simultaneously queryable; the service converges to what its + sources present as canonical ([DEF-6](02-data-model.md)). +- **NG3 — Not a consensus participant.** The service does not validate consensus rules, + signatures, or state transitions. It validates *structural* consistency (numbering, + parent-hash linkage, finality monotonicity) and otherwise trusts its sources + (see Trust model, §5). +- **NG4 — No cross-dataset transactions.** No operation spans datasets atomically. +- **NG5 — No server-side query cursors.** Continuation is client-driven and stateless on + the server ([RP-10](04-read-path.md)). +- **NG6 — No exactly-once source consumption.** Source delivery is at-least-once; + ingestion is idempotent with respect to redelivery ([WP-16](03-write-path.md)). + +## 5. Trust model + +- **Sources are semi-trusted.** The service verifies everything it can verify locally: + ascending numbering, parent linkage against its own stored chain, finality + monotonicity, and cross-source agreement where multiple sources exist. It cannot verify + that a hash corresponds to a *valid* block, so the *content* of blocks and the *choice* + of canonical fork and finality are trusted from sources. A source that violates the + structural rules MUST be rejected/quarantined per [FM-SRC-*](08-failure-model.md); it + MUST NOT be able to corrupt stored state or crash the process. +- **Clients are untrusted.** Any client input — malformed, oversized, adversarial — MUST + yield a defined error and MUST NOT affect other clients' correctness, crash the process, + or wedge ingestion ([FM-1](08-failure-model.md)). +- **The operator is trusted** but fallible: configuration mistakes MUST fail loudly and + safely at startup rather than silently corrupt or delete the wrong data + ([INV-43/44](06-invariants.md)). + +## 6. Lifecycle at a glance + +``` + ┌──────────────────────────── service instance ───────────────────────────┐ + sources ──blocks──▶ │ per-dataset ingestion ──▶ atomic transitions ──▶ committed chain window │ ──▶ query clients + fork sigs │ validation (extend/replace/ (snapshot reads) │ (ranged queries, + finality │ arbitration finalize/retain) │ watermarks) + │ │ + retention ─────────▶│ retention policy ──▶ logical trim ──▶ deferred physical reclamation │ ──▶ observability + controller └─────────────────────────────────────────────────────────────────────────┘ (metrics, health) +``` + +A dataset's life: **created** (kind fixed forever) → **window established** (per retention +policy) → steady state of *extend / replace-on-fork / finalize / retain* transitions → +possibly **reset** (self-healing on unrecoverable divergence) → **dropped** (removed from +configuration). The service's life: **boot** (recover, validate, optional reader-free +maintenance) → **ready** (serving + ingesting) → **draining shutdown**. + +## 7. Reading guide + +The formal core is 02 (state) → 03/04 (transitions and reads) → 06/07 (what must always / +eventually hold). 05, 08, 09 elaborate cross-cutting guarantees. 10/11 make the operational +qualities testable. 12 turns all of it into a test plan; 13/14 pin the concrete surface. diff --git a/crates/hotblocks/spec/02-data-model.md b/crates/hotblocks/spec/02-data-model.md new file mode 100644 index 00000000..1bf87f9b --- /dev/null +++ b/crates/hotblocks/spec/02-data-model.md @@ -0,0 +1,206 @@ +# 02 — Data model + +This document defines the abstract state on which all other documents operate. Nothing here +prescribes a storage layout; it prescribes what must be *representable* and what the +observable structure of the data is. + +## 1. Primitive types + +**DEF-1 (Block number).** A natural number `n ∈ ℕ`, strictly increasing along a chain. It does +**not** necessarily increase by exactly 1: Solana numbers blocks by time-based **slots**, and a +slot that produced no block leaves a hole. Numbering therefore carries no structural guarantee — +`parent_number` (DEF-4) does. + +**DEF-2 (Hash).** An opaque, non-empty string. Hashes are compared by exact string +equality; the service assigns no meaning to their content. All sources of a dataset MUST +use one consistent representation (a representation mismatch between sources is a source +fault, [FM-SRC-7](08-failure-model.md)). + +**DEF-3 (Block reference / Ref).** A pair `⟨number, hash⟩` identifying one block. + +**DEF-4 (Block).** A tuple: + +``` +Block = ⟨ number : ℕ, + hash : Hash, + parent_number : ℕ, — the number of the parent block + parent_hash : Hash, + payload : Payload(kind), + time : Timestamp | ⊥ ⟩ +``` + +`parent_number = number − 1` on a densely-numbered chain (evm, hyperliquid); on a slot-numbered +one (Solana) it may sit further down. Always `parent_number < number`; a block violating this +is structurally invalid ([WP-2](03-write-path.md)). `⟨parent_number, parent_hash⟩` is what makes +a run of blocks a chain — the numbering alone does not (DEF-1). + +`time` is informational; no correctness property depends on it (clock-free correctness, +[CN-8](05-consistency-and-durability.md)). + +**DEF-5 (Kind and payload schema).** Every dataset has an immutable **kind** — a chain +family identifier (e.g. `evm`, `solana`, `bitcoin`, `tron`, `hyperliquid-fills`, +`hyperliquid-replica-cmds`). A kind defines: + +- the **header** attributes of a block; +- a fixed set of named **item collections** (e.g. transactions, logs, traces, state diffs, + instructions), where each **item** is a record carrying (at least) the number of the + block it belongs to, plus kind-specific attributes; +- the **query dialect**: which filter predicates and field selections are expressible; +- optional **availability**: an item collection MAY be absent for sub-ranges of the chain + (the source did not provide it). Availability is a property of stored data and is + reported per query ([RP-8](04-read-path.md)). + +The internal encoding of payloads is out of scope; §6 of [12-conformance-tdd.md](12-conformance-tdd.md) +defines the kind-agnostic structural checks a harness can still perform. + +## 2. Dataset state + +**DEF-6 (Dataset).** A dataset `D` is the tuple: + +``` +D = ⟨ kind : Kind, — immutable after creation + seg : ⟨b_1 … b_n⟩, n ≥ 0, — the stored chain segment (window) + anchor : ⟨number : ℕ ∪ {−1}, hash : Hash | ⊥⟩, — a position below seg (DEF-7) + fin : Ref | ⊥, — finalized watermark + ret : RetentionPolicy, + ver : ℕ ⟩ — commit version, increments on every transition +``` + +Derived values (defined when `seg` is non-empty): + +``` +first(D) = b_1.number head(D) = ⟨b_n.number, b_n.hash⟩ +next(D) = head(D).number + 1 (if seg empty: anchor.number + 1) +span(D) = n (window size in blocks) +``` + +A dataset state is **well-formed** iff it satisfies the structural invariants +INV-1 … INV-7 of [06-invariants.md](06-invariants.md). Every externally observable state +MUST be well-formed. + +**DEF-7 (Anchor).** `anchor` names a position below `first(D)` — on a densely-numbered +chain exactly `first(D) − 1`; on a slot-numbered chain possibly lower (a RETAIN may land +on a hole, and the next stored block may sit above it — INV-3). When its hash is +known, it carries `hash_at(anchor.number)` (DEF-16): on a densely-numbered chain the hash of +the block at that position; on a slot-numbered chain the hash of the nearest block at or +below it (the position itself may be a hole). The anchor is how a window "remembers" its +connection to the chain below the window. `anchor.hash = ⊥` is permitted (e.g. a window +starting at genesis — `anchor = ⟨−1, ⊥⟩` — or an operator-supplied start without a hash); +when it is `⊥`, linkage of `b_1` cannot be verified and is accepted as-is. + +**DEF-16 (Preceding block; chain hash at a position).** The **preceding block** of a +position `n` is the highest stored block numbered below `n` — the block anything starting +at `n` must link to. Its hash is `hash_at(n − 1)`: + +``` +hash_at(p) = hash of the highest stored block with number ≤ p, + or anchor.hash when no stored block lies at or below p and p ≥ anchor.number; + undefined below the anchor +``` + +On a densely-numbered chain the preceding block of `n` is simply block `n − 1`; on a +slot-numbered chain it is the nearest block below `n`, wherever it sits. Everywhere the +protocol anchors a client or a source to a position — the anchor (DEF-7), +`expected_parent` (DEF-13), fork-signal hints (DEF-12), CONFLICT hints +([RP-11](04-read-path.md)), continuation cursors ([RP-10](04-read-path.md)) — a pair +`⟨p, h⟩` reads "`h` is the hash of the preceding block of `p + 1`" (`h = hash_at(p)`), +**not** "a block numbered `p` has hash `h`". The two readings coincide exactly on +densely-numbered chains. + +**DEF-8 (Finalized watermark).** `fin`, when defined, designates a stored block +(`first ≤ fin.number ≤ head.number`, hash matching the stored block — INV-5/6) that the +sources have declared irreversible. Blocks at or below `fin.number` form the **finalized +prefix**; blocks above it are the **volatile suffix**, subject to replacement by forks. + +**DEF-9 (Retention policy).** One of: + +| Policy | Meaning | +|---|---| +| `Window(k)` | Keep (at least) all stored blocks in the last `k` **positions** below `next(D)`; trim the rest as the head advances. `k` counts numbers, not stored blocks: on a slot-numbered chain the window holds ≤ `k` blocks. `k ≥ 1`. | +| `Pinned(from, hash?)` | Keep everything from block `from` upward; `hash?` optionally asserts the anchor at `from − 1`. | +| `External` | The lower bound is set at runtime by the retention controller via the SET-RETENTION operation. Until first set: unbounded, and a dataset that is *empty* at activation defers ingestion until the first instruction (WP-5). | +| `Unbounded` | Never trim. | + +**DEF-10 (Version order).** For one dataset, committed states are totally ordered by +`ver`. "Later state" always means greater `ver`, never a comparison of head numbers (a +fork can lower the head number while increasing `ver`). + +## 3. Snapshots + +**DEF-11 (Snapshot).** An immutable, well-formed dataset state as of some committed +version. Every read operation is evaluated against exactly one snapshot +([INV-20](06-invariants.md)). Snapshots of different datasets are independent; no +operation observes a cross-dataset "consistent cut", and none is guaranteed to exist. + +## 4. Source-facing events + +Ingestion consumes an abstract event stream per dataset (how it is transported is a +binding concern): + +**DEF-12 (Source events).** + +| Event | Content | Meaning | +|---|---|---| +| `Blocks(⟨b…⟩)` | contiguous, internally linked run of blocks | candidate extension of the chain at the position the service requested | +| `ForkSignal(hints)` | non-empty ascending list of `⟨position, hash⟩` pairs, read per DEF-16 (a hint at a stored position of the source's chain is a true block ref; at a hole it carries the preceding block's hash) | the source disagrees with the service's requested position; `hints` name recent positions of the source's canonical chain near the divergence point | +| `FinalitySignal(ref)` | a Ref | the source declares `ref` (and everything below it on its chain) final | +| `OnTip` | — | the source believes the service has caught up with its tip | + +Delivery is at-least-once, per source; ordering is guaranteed only within one source's +stream between reconnects. Multiple sources are reconciled by arbitration +([WP-4](03-write-path.md)). + +## 5. Query-facing objects + +**DEF-13 (Query).** A read request: + +``` +Query = ⟨ dialect : Kind, — must match the dataset kind + from : ℕ, — inclusive lower bound + to : ℕ | ⊥, — inclusive upper bound; ⊥ = open-ended + expected_parent : Hash | ⊥, — asserted hash of from's preceding block (DEF-16) + finalized_only : bool, — restrict to the finalized prefix + selection : kind-specific filters (item requests) and field projections, + include_all : bool ⟩ — emit headers even for blocks with no matches +``` + +**DEF-14 (Coverage).** The result of a successful query is defined by its **coverage**: +a contiguous range `[from, L]` of block numbers, `L ≥ from`, fully processed against the +query's snapshot. The response *emits* the subset of covered blocks required by +`selection`/`include_all`; coverage — not emission — is what the client uses to make +progress ([RP-9](04-read-path.md)). + +**DEF-15 (Watermark reads).** Point reads returning the current committed `head`, `fin`, +retention policy, and dataset status. Defined in [04-read-path.md §6](04-read-path.md). + +## 6. Transitions (summary) + +The complete write-side vocabulary; semantics in [03-write-path.md](03-write-path.md): + +| Transition | Effect on ⟨seg, anchor, fin⟩ | +|---|---| +| `CREATE(kind, ret)` | new empty dataset with an initial anchor per policy | +| `EXTEND(B, f?)` | append blocks at the tail; optionally advance `fin` | +| `REPLACE(from, B, f?)` | fork: atomically substitute the suffix `≥ from` with `B` | +| `FINALIZE(r)` | advance `fin` (monotone, clamped to the stored chain) | +| `RETAIN(from, h?)` | trim the prefix `< from`; move the anchor up | +| `RESET(a)` | clear the segment, re-anchor at `a` (self-healing / re-targeting) | +| `DROP` | remove the dataset and all its data | + +Each transition is atomic and increments `ver` exactly once — except CREATE, which +*establishes* `ver = 0`: it starts the version sequence rather than extending one. A +single commit MAY compose `EXTEND`/`REPLACE` with `FINALIZE` (blocks and a finality +advance arriving together); invariants are evaluated at commit points only. + +## 7. Terminology cross-reference + +| Term used elsewhere | Defined as | +|---|---| +| window | `seg` (DEF-6) | +| head / first block | derived values of DEF-6 | +| finalized prefix / volatile suffix | DEF-8 | +| anchor | DEF-7 | +| preceding block / `hash_at(p)` | DEF-16 | +| coverage | DEF-14 | +| fork hints | `ForkSignal.hints` (DEF-12), also the payload of the CONFLICT error (RP-11) | +| batch | the unit of one `EXTEND`/`REPLACE` commit (bounded by P-BATCH-ROWS / P-BATCH-BYTES) | diff --git a/crates/hotblocks/spec/03-write-path.md b/crates/hotblocks/spec/03-write-path.md new file mode 100644 index 00000000..65cbe4b5 --- /dev/null +++ b/crates/hotblocks/spec/03-write-path.md @@ -0,0 +1,268 @@ +# 03 — Write path (ingestion and transitions) + +This document specifies how a dataset's state advances: what is accepted from sources, how +it is validated, and the exact pre/postconditions of every transition. All requirements are +per dataset unless stated otherwise. + +## 1. Ingestion loop (abstract) + +For each dataset the service runs a conceptual loop: + +``` +loop: + pos ← resume position = ⟨ next(D), expected parent hash ⟩ + where expected parent hash = head(D).hash if seg ≠ ∅ else anchor.hash (may be ⊥) + (= hash_at(next(D) − 1), DEF-16) + request blocks from sources starting at pos + react to source events: + Blocks(B) → validate, accumulate, commit EXTEND / REPLACE batches + ForkSignal(hints) → resolve rollback point, resume from it (next commit is a REPLACE) + FinalitySignal(r) → commit FINALIZE (possibly composed with a batch commit) + OnTip → flush any accumulated partial batch + apply retention policy after commits (RETAIN when the policy demands it) +``` + +- **WP-1** The service MUST always request continuation from its own committed state + (`next(D)` and the corresponding expected parent hash), never from transient in-memory + positions that could disagree with committed state. +- **WP-2 (Validation).** Before commit, a candidate run of blocks MUST be verified to be: + internally linked (`b_{i+1}.parent_number = b_i.number ∧ b_{i+1}.parent_hash = b_i.hash`) + and correctly attached at the front (per the transition preconditions in §3). Since + `parent_number < number` (DEF-4), linkage implies strictly ascending numbers — but an + implementation that checks only *hash* linkage MUST verify ascending numbers separately: + hashes are opaque, source-controlled strings (DEF-2), so hash linkage alone implies no + number order (GAP-20). A run failing validation MUST be discarded without any + state change, and the offending source penalized ([FM-SRC-4](08-failure-model.md)). +- **WP-3 (Batching).** Blocks MAY be accumulated and committed in batches. A batch is + bounded by `P-BATCH-ROWS` rows / `P-BATCH-BYTES` bytes and MUST be flushed no later + than: the bound being reached, an `OnTip` event, a `ForkSignal`, or an item-availability + change. Batching MUST NOT reorder or drop blocks. (Freshness consequence: the head + advances in batch-sized steps; see HZ-6 and SLI-11.) +- **WP-4 (Multi-source arbitration).** With multiple sources, the service MUST present the + effect of a single coherent source: duplicate deliveries deduplicated, positions below + `next(D)` ignored, and a `ForkSignal` acted upon only when corroborated — by a majority + of sources, by all currently live sources, or after `P-FORK-CONSENSUS` time without + contradiction. Arbitration MUST NOT interleave blocks from sources on different forks + into one run (WP-2 makes such a run unommittable). +- **WP-5 (Initial window).** On first activation, a dataset's initial anchor is + established per policy: `Pinned(from, h?)` → `anchor = ⟨from − 1, h?⟩` (`from = 0` ⇒ + `h? = ⊥`: there is no position below genesis to assert a hash against — DEF-7, and no + wire representation of position `−1`, IB §3); + `Window(k)` → probe the sources' current tip `T` and set + `anchor = ⟨max(0, T − k) − 1 …⟩` semantics such that roughly the last `k` blocks will be + ingested; `External`/`Unbounded` → resume from the existing window when the dataset is + non-empty; an **empty** dataset has no defined start position and MUST idle (no source + consumption) until one is established — by the first SET-RETENTION instruction + (External) or by reconfiguration (Unbounded). Idling is not a fault; STATUS reports + the empty state. Probing MUST NOT block the rest of the service (LIV-5) and MUST be + retried with bounded backoff while sources are unavailable. + +## 2. Transition catalog — preconditions and effects + +Notation: unprimed = state before, primed = state after. Every transition increments +`ver` by one and is atomically visible (INV-10). `seg[x‥]` / `seg[‥x]` denote the suffix +from number `x` / prefix below number `x`. + +### 2.1 CREATE(kind, ret) + +- Pre: no dataset with this identity, or an existing dataset with the **same** kind + (idempotent re-attach). An existing dataset with a different kind MUST be refused + (INV-43); the service MUST NOT silently reinterpret or overwrite it. +- Post, fresh create: `⟨kind, seg=∅, anchor per WP-5, fin=⊥, ret, ver=0⟩`. +- Post, re-attach: the persisted committed state is preserved in every field — `seg`, + `anchor`, `fin`, `ver` (INV-40); only `ret` is taken from configuration and validated + against that state (WP-9 / INV-43). Re-attach is **not** a destructive path (INV-44): + an attach that loses or resets stored state is a defect, not CREATE semantics. + +### 2.2 EXTEND(B, f?) + +- Pre: `B ≠ ∅` and valid per WP-2, and: + - `seg ≠ ∅` ⇒ `B[0].number ≥ next(D)` ∧ `B[0].parent_number = head(D).number` ∧ + `B[0].parent_hash = head(D).hash`; + - `seg = ∅` ⇒ `B[0].number ≥ anchor.number + 1` ∧ + (`anchor.hash ≠ ⊥` ⇒ `B[0].parent_hash = anchor.hash`). +- Post: `seg' = seg ⧺ B`; `anchor' = anchor`; `fin'` per composed FINALIZE if `f?` given, + else `fin`. + +(`=` in place of `≥` on a densely-numbered chain; on a slot-numbered one the first block +of a run may sit above the requested position — `next(D)` is the lowest *admissible* +position, not a promised block number, DEF-1.) + +### 2.3 REPLACE(from, B, f?) — fork application + +- Pre: `seg ≠ ∅`; `B ≠ ∅` and valid per WP-2; `B[0].number ≥ from`; and + - **fork floor (finality):** `fin = ⊥ ∨ from > fin.number` — MUST (INV-13/14); + - **fork floor (window):** `from ≥ first(D)` — MUST (INV-14). A required rollback below + `first(D)` is not representable as REPLACE; it MUST be handled as RESET (§2.6) and + surfaced as an event ([OB-9](11-observability.md)); + - linkage: `from > first(D)` ⇒ `B[0].parent_hash` = the hash of `from`'s preceding + block (`hash_at(from − 1)`, DEF-16 — `from − 1` itself may be a hole on a + slot-numbered chain); `from = first(D)` ⇒ (`anchor.hash ≠ ⊥` ⇒ + `B[0].parent_hash = anchor.hash`). +- Post: `seg' = seg[‥from] ⧺ B`; `anchor' = anchor`; `fin'` per composed FINALIZE or `fin`. +- `REPLACE(next(D), B)` degenerates to `EXTEND(B)`. + +**WP-6 (Fork resolution).** Upon an arbitrated `ForkSignal(hints)` the service MUST +compute the resume point as follows: consider hints with `number ≥ fin.number` only — if +none exist, or the hint at `fin.number` carries a different hash than `fin.hash`, the +signal contradicts finality and MUST be treated as a source integrity fault +([FM-SRC-5](08-failure-model.md)), not applied. Otherwise resume from +`⟨m + 1, hint(m).hash⟩` where `m` is the highest hint number whose `⟨number, hash⟩` equals +the stored chain (or the anchor). If no hint matches stored state, the fallback MUST +respect the finality floor: + +- `fin ≠ ⊥` → resume from `⟨fin.number + 1, fin.hash⟩`: the finalized block is common to + both chains unless the source contradicts finality, so only the volatile suffix is + replaced (a full-window replacement would violate the fork floor anyway, INV-14). If + the source then repeatedly fails to link at this position, that *is* a finality + contradiction: after `P-SOURCE-STRIKES` consecutive rejections it MUST be classified as + FM-SRC-5 (fault + alarm, keep serving) — never RESET; +- `fin = ⊥` → resume from `⟨first(D), anchor.hash⟩` (full-window replacement); WP-6b + governs escalation if the divergence turns out to lie below the window. + +The subsequent commit is `REPLACE(m + 1, …)` (resp. `REPLACE(fin.number + 1, …)`, +`REPLACE(first(D), …)`). The resume point MAY be conservatively deeper than the optimal +`m + 1` by at most one storage batch (an implementation that matches hints only at batch +boundaries): the replacement then re-commits blocks identical to those it replaces, which +is correctness-neutral. It MUST NOT be shallower than `m + 1` and MUST NOT cross the +floors of §2.3. + +**WP-6b (Divergence below the window → RESET).** WP-6's fallback cannot represent a fork +deeper than the window; the service MUST detect that case and escalate to RESET (alarmed, +OB-9) instead of retrying a replacement that can never link: + +- **direct evidence:** an arbitrated hint at position `anchor.number` whose hash + contradicts `anchor.hash ≠ ⊥` — the divergence is at or below the anchor; + `RESET(⟨anchor.number, hint hash⟩)`; +- **indirect evidence:** after a full-window-replacement resume (reachable only when + `fin = ⊥` — WP-6 fallback), the source's runs + repeatedly fail attachment at `first(D)` (`B[0].parent_hash ≠ anchor.hash`, WP-2). + After `P-SOURCE-STRIKES` consecutive such rejections the condition MUST be classified + as a below-window divergence — `RESET(⟨anchor.number, ⊥⟩)` — not retried silently + forever (LIV-9b, GAP-3/GAP-5). + +**WP-7 (Fork visibility window).** Between acting on a `ForkSignal` and committing the +first `REPLACE`, the pre-fork state remains the committed, visible state — the service +MUST NOT truncate without replacement. Consequences: orphaned blocks remain readable to +un-anchored clients during this interval; anchored clients are protected by the CONFLICT +protocol (RP-11). The interval is bounded by LIV-9. + +### 2.4 FINALIZE(r) + +Let `e = min(r.number, head(D).number)` (a finality report above the head is clamped to +the head; the excess is not forgotten by sources and will re-arrive). + +- Pre: `seg ≠ ∅` (else the report is deferred/ignored); `e ≥ first(D)` (a report below the + window is ignored); **hash verification:** when `e = r.number` there MUST be a stored + block at height `e` with hash equal to `r.hash` — a report naming a height that is a + hole in the stored chain (slot-numbered chains, DEF-1) contradicts the stored chain + exactly like a hash mismatch and MUST be treated as a source integrity fault + (FM-SRC-5, WP-8); when clamped (`e < r.number`), the stored head is taken as finalized + on the strength of the source's claim about its descendant. +- Monotonicity: if `fin ≠ ⊥ ∧ e < fin.number` → ignore (no transition). If + `e = fin.number` with a different hash → source integrity fault (FM-SRC-5), no + transition. +- Post: `fin' = ⟨e, stored hash at e⟩`. + +**WP-8** A finality report whose hash contradicts the stored block at the same height +(either `fin` itself or the block at `e`) MUST NOT be applied and MUST raise an integrity +alarm — silently dropping it hides either a source fault or a wrong stored chain. + +### 2.5 RETAIN(from, h?) + +Trims history below `from`; the inverse direction (lowering `from`) is not a transition — +history cannot be re-acquired through RETAIN. + +- Case `from = first(D)` (or `seg = ∅` and `from ≤ anchor.number + 1`): no-op (except + optional anchor-hash validation per WP-9). +- Case `from < first(D)` — the bound moves *down*: history below the window cannot be + re-acquired in place (there is no downward backfill), so in self-healing modes + (`Window`, `External` runtime application) the instruction MUST be executed as + `RESET(⟨from − 1, h?⟩)`: the window is discarded and re-ingested from `from` upward. + Like every RESET this MUST be observable (OB-9) — a downward `SET-RETENTION` is a + destructive re-bootstrap, not a widening, and controllers MUST treat it as such + (FM-OP-4). During boot validation of a `Pinned` policy whose `from` lies below the + stored window, the service MUST refuse the dataset (INV-43) rather than destroy data — + symmetric with WP-9. *Finality note:* like every RESET (§2.6) and like upward trims + (RS-2), this discards the finalized prefix and clears `fin` — retention dominates + finality. It is **not** a rollback below `fin`: the fork floor (INV-13/14) is + untouched — *sources* can never replace anything at or below `fin` — and this path is + reachable only through an explicit retention instruction (operator-class actor, + FM-OP-4), never through source input. +- Case `first(D) < from ≤ next(D)`: + - Post: `seg' = seg[from‥]`; `anchor' = ⟨from − 1, hash of from's preceding block in + the pre-trim state⟩` (= `hash_at(from − 1)`, DEF-16). The hash MUST be carried over + from previously stored data, never recomputed or dropped (INV-18); + - `fin' = fin` if `fin.number ≥ from`, else `fin' = ⊥` (retention dominates finality, + RS-2). +- Case `from > next(D)`: equivalent to `RESET(⟨from − 1, h?⟩)` — the whole window is + discarded and the dataset awaits future blocks. +- **WP-9 (Mismatch policy).** If `h?` is provided and disagrees with the stored chain's + `hash_at(from − 1)` (DEF-16): in self-healing modes (`Window`, `External` runtime + application) the service MUST `RESET(⟨from − 1, h?⟩)`; during boot validation of a + `Pinned` policy it MUST refuse the dataset (INV-43) rather than destroy data. +- **WP-10 (Trigger, `Window(k)`).** After every commit that advances `next(D)`, if + `next(D) − first(D) > k` (`k` counts positions — DEF-9) the service MUST trim such that + the availability floor RS-3 + (at least the last `k` blocks) always holds and the excess bound RS-4 + (`first(D) ≥ next(D) − k − P-RETENTION-SLACK`, eventually) is met. Trimming MAY be + batch-granular. +- **WP-11 (External application).** A SET-RETENTION instruction accepted through the API + MUST be applied (the corresponding RETAIN committed) within `P-RETENTION-APPLY`, and be + observable via the retention read afterwards. Acceptance and application are distinct + points: the acknowledgement means the instruction is recorded and scheduled, not that + the trim has happened. Between the two, GET-RETENTION reports the *instructed* bound — + an acknowledged instruction is never silently forgotten (CN-9; violated today, GAP-28) — + and `ver` advances with the RETAIN/RESET commit itself, not at acceptance. What + instruction payloads other than a block bound mean for an External dataset (policy-mode + changes, e.g. the binding's `"None"`) is unspecified, and the current behavior diverges + from the binding's reading (GAP-35). + +### 2.6 RESET(a) + +- Post: `seg' = ∅`, `anchor' = a`, `fin' = ⊥`. +- RESET is the only self-healing transition that discards the volatile *and* finalized + parts of a window together. It MUST be committed atomically, MUST be surfaced as an + event/observable (OB-9) — never silent — and MUST only occur through the defined + triggers: WP-6b (fork deeper than the window), WP-9 (retention mismatch), a downward + retention bound (§2.5), operator action (INV-44). + +### 2.7 DROP + +- Removes the dataset and all stored data. Triggered by operator/configuration only + (INV-44). Space follows RS-5/RS-6 (logical removal immediate, physical reclamation + deferred). A DROP followed by CREATE of the same identity MUST behave as a fresh dataset. + +## 3. Commit and acknowledgement + +- **WP-12 (Commit point).** A transition is *committed* when a read issued afterwards can + observe it. All invariant evaluation, version numbering, and watermark reporting refer + to commit points. +- **WP-13 (No partial blocks).** No reader may ever observe a block with a subset of its + items, a batch with a subset of its blocks, or metadata referring to absent payloads + (INV-10). This MUST hold across crashes (INV-40). +- **WP-14 (Write acknowledgement to sources).** The only acknowledgement a source receives + is the position of the next request (WP-1). Therefore the service MUST tolerate + redelivery of anything not yet durable, and deduplicate blocks below `next(D)` + (**WP-16, idempotency**: redelivery of already-stored blocks causes no state change and + no error). +- **WP-15 (Single writer).** At most one writer per dataset store may exist. If a second + concurrent controller of the same dataset is detected (state changed under the writer's + feet), the writer MUST stop mutating that dataset and raise an alarm (FM-OP-3) — both + writers continuing is forbidden. + +## 4. Ingestion error handling + +- **WP-17 (Transient vs integrity).** Failures MUST be classified: *transient* (source + unreachable, timeouts, storage temporarily saturated) → retry with bounded backoff + (`P-SOURCE-BACKOFF`, `P-EPOCH-RETRY`), unbounded in count; *integrity* (finality + contradiction, unremovable validation failure, config/state mismatch) → stop retrying + blindly, surface a distinct alarm state within `P-ALARM` (LIV-9, OB-9), keep serving + reads. +- **WP-18 (Robustness).** No content of any source event — arbitrary bytes, sizes, + encodings, value ranges — may terminate the process, corrupt other datasets, or wedge + the write path permanently (FM-1). Malformed events are rejected per WP-2/WP-17. +- **WP-19 (Restart continuity).** After any restart, ingestion MUST resume from the + recovered committed state per WP-1 — including the recovered anchor hash being exactly + the committed one (INV-40); resuming from a wrong expected hash manifests as a + perpetual source rejection loop and is a defect, not a source fault. diff --git a/crates/hotblocks/spec/04-read-path.md b/crates/hotblocks/spec/04-read-path.md new file mode 100644 index 00000000..7f24bda6 --- /dev/null +++ b/crates/hotblocks/spec/04-read-path.md @@ -0,0 +1,210 @@ +# 04 — Read path (queries and watermarks) + +This document specifies every read-side operation: the ranged query (live and +finalized-only), watermark/status reads, and the retention admin operation. The wire +mapping lives in [13-interface-binding.md](13-interface-binding.md). + +## 1. Operations + +| Operation | Input | Output | +|---|---|---| +| `QUERY` | DEF-13 query, `finalized_only = false` | streamed block emissions + response watermarks, or error | +| `QUERY-FINALIZED` | DEF-13 query, `finalized_only = true` | same, restricted to the finalized prefix | +| `HEAD` | dataset | `head(D)` Ref or "none" | +| `FINALIZED-HEAD` | dataset | `fin(D)` Ref or "none" | +| `STATUS` | dataset | kind, retention policy, `first`, `head` (+hash, time), `fin` | +| `GET-RETENTION` / `SET-RETENTION` | dataset (+policy) | current policy / acceptance | + +## 2. Query admission + +- **RP-1 (Validation).** A query MUST be rejected with `MALFORMED_REQUEST` if structurally + invalid: unparsable, unknown fields, `to < from`, more than `P-MAX-ITEM-REQ` item + requests, or dialect-specific rule violations. Validation failures MUST be detected + before any partial response bytes are emitted. +- **RP-2 (Dialect gate).** A query whose dialect differs from the dataset kind MUST fail + with `KIND_MISMATCH`. A dialect that is expressible but not supported by this service + MUST fail with `UNSUPPORTED_QUERY`. Neither may crash the process or degrade other + requests (FM-1). +- **RP-3 (Admission control).** Under resource exhaustion the service MUST refuse new + query work with `OVERLOADED` (retryable) rather than queue unboundedly or collapse + (PF-9). Admission decisions are made before the success/error status is committed. + +## 3. Effective range and waiting + +Let the query's snapshot be `S` (taken per INV-20 after any waiting completes). + +- Live query: `hi(S) = head(S).number` (undefined if `seg = ∅`); finalized-only: + `hi(S) = fin(S).number` (undefined if `fin = ⊥`). An undefined `hi` behaves as + `from > hi` (RP-5/RP-6). Effective range: `[from, min(to, hi(S))]` with `to = ∞` when + omitted. +- **RP-4 (Below the window).** If `from < first(S)` the query MUST fail with + `RANGE_UNAVAILABLE` (not silently serve a later sub-range). The error SHOULD state the + lowest available block. +- **RP-5 (Bounded wait / long-poll).** If `from > hi(S)` at admission, the service MUST + wait up to `P-HEAD-WAIT` for the watermark to reach `from`. If reached, proceed with a + fresh snapshot; else respond `NO_DATA` (empty, retryable). `NO_DATA` responses SHOULD + carry current watermarks so pollers can pace themselves. The number of concurrent + waiters MAY be capped (`P-WAITERS`; overflow → `OVERLOADED`). +- **RP-5b (Tip conflict precedes waiting).** Exception to RP-5. Let `T` be the watermark + block itself (`head(S)` for a live query, `fin(S)` for finalized-only). If + `expected_parent ≠ ⊥` and `from = T.number + 1` and `expected_parent ≠ T.hash`, the + service MUST respond `CONFLICT` immediately (hints per RP-11) instead of waiting: + client cursor and watermark reference the *same exact position* (`from − 1 = + T.number`), so the mismatch is a definite fork on any chain, and the immediate signal + is how a tip-follower learns of a reorg without waiting out `P-HEAD-WAIT`. This + exception MUST NOT be generalized to `from > T.number + 1`: there `expected_parent` + names a position the service may simply not have ingested yet (a client ahead of a + lagging service), a mismatch against `hash_at(from − 1) = T.hash` is not evidence of a + fork, and `NO_DATA` remains the correct response. +- **RP-6 (Queries without a watermark).** `QUERY-FINALIZED` when `fin = ⊥` — and likewise + any query on an empty dataset (`seg = ∅`, live `hi` undefined) — behaves as `from > hi`: + wait, then `NO_DATA`. Neither is an internal error. + +## 4. Execution and coverage + +- **RP-7 (Snapshot execution).** The whole response is computed against the single + snapshot `S` (INV-20): one chain, one `fin`, one availability view — regardless of + concurrent writes, forks, or trims. +- **RP-8 (Item availability).** Availability is range-based (DEF-5) and a response MUST + never silently omit items of a selected collection. If the selection references an item + collection absent at the *start* of the effective range, the query MUST fail with + `ITEM_UNAVAILABLE` identifying the collection. If the collection becomes unavailable + *inside* the effective range, coverage MUST end before the availability boundary (an + early stop per RP-9); the continuation query then starts inside the unavailable range + and fails as above. Coverage never crosses an availability boundary of a selected + collection. +- **RP-9 (Coverage contract).** A successful response has coverage `[from, L]` with: + - `from ≤ L ≤ min(to, hi(S))` — never beyond the effective range (INV-27); + - coverage is gap-free: every block in `[from, L]` was evaluated (INV-21/22); + - **progress:** if the effective range is non-empty then `L ≥ from` — a successful + response always covers at least one block (INV-25); + - the response MAY stop early (`L < min(to, hi(S))`) due to budgets + (`P-RESP-WEIGHT`, `P-QUERY-TIME`) — early stop is normal, not an error; + - **coverage MUST be recoverable by the client — the carrier is emission itself:** the + block at the coverage end (the highest stored block ≤ `L`; = block `L` itself when + `L` is a stored block) MUST be emitted, as a **header-only record** when it matches + nothing — even with `include_all` unset. Consequently the client always advances + with `L` = the last emitted block's number, and a successful response over an + effective range containing at least one stored block emits at least one block. The + service MAY additionally emit other covered non-matching blocks as header-only + records (**boundary markers**); which extra markers appear MAY depend on physical + layout and is exempt from INV-22 determinism — clients MUST NOT attach meaning to a + header-only record beyond its header fields and the coverage it witnesses. A + zero-emission success is legal only when the effective range contains no stored + block at all, and it asserts coverage of the *entire* effective range (GAP-8 tracks + the residual hazards of that case). +- **RP-10 (Continuation is client-driven).** The server keeps no cursor. The client + continues with `from' = L + 1` and `expected_parent` = the hash of its last applied + block (= `hash_at(L)`, DEF-16) — see §7. Consecutive + responses MAY come from different snapshots; cross-response consistency is exactly the + anchored-ancestry guarantee (INV-23), nothing more. + +## 5. Emission contract + +For coverage `[from, L]` against snapshot `S`: + +- Emitted blocks are those in coverage that match the selection (plus all covered blocks + when `include_all` is set), plus the RP-9 coverage-end record and any header-only + boundary markers of covered blocks; emitted in strictly ascending number order; each + emitted at most once (INV-21). +- Each emitted block carries its header (per projection) and, per item collection, **all** + items of that block matching the item requests, projected to the selected fields — + never a subset (INV-22). Kind-specific "include related items" rules are deterministic + functions of `(S, query)`. +- Every emitted item belongs to its enclosing block (`item.block_number = block.number`) + (INV-21). +- Truncation granularity is the block: a block is emitted whole or not at all (INV-21). + If even the first covered block exceeds the response budget, it MUST still be emitted + (progress, INV-25). +- Emission content is deterministic given `(S, query, coverage)` (INV-22); coverage + itself MAY vary run to run (budgets are time-based). + +## 6. Watermark and status reads + +- **RP-12 (Honesty).** Every watermark read returns values of some committed state + (INV-30) — never a forecast, never a mix of two states for the (number, hash) pair of + one watermark. +- **RP-13 (Freshness/monotonicity).** Reads are real-time monotonic in version order + (INV-31): a read started after a commit completes reflects that commit or a later one. + Head *numbers* are not monotonic (forks); versions are. +- **RP-14 (Status).** STATUS reports a consistent view: kind, retention policy, `first`, + `head` (number, hash, time when known), `fin` — all from one snapshot. + +## 7. Fork handling — the CONFLICT protocol + +- **RP-11 (Anchored queries).** When `expected_parent ≠ ⊥`, it asserts the hash of + `from`'s **preceding block** on `S`'s chain (`hash_at(from − 1)`, DEF-16): + - if `hash_at(from − 1) ≠ ⊥` on `S` and differs from `expected_parent`, the query MUST + fail with `CONFLICT` and MUST NOT emit any blocks. Equivalently: the first stored + block at or above `from` must carry `parent_hash = expected_parent` (INV-2 makes the + two readings coincide); + - the `CONFLICT` payload is a non-empty ascending list of **hints**: entries + `⟨p, hash_at(p)⟩` of `S`'s chain ending at position `from − 1` (or the nearest stored + position below), containing at least that final entry and SHOULD contain up to + `P-CONFLICT-WINDOW` predecessors. A hint at a stored position is a true block ref; a + hint at a hole position carries the hash of that position's preceding block (DEF-16); + - if `hash_at(from − 1) = ⊥` (window edge, unknown anchor hash), the assertion is + accepted as-is. +- **Client recovery algorithm (normative for clients, the basis of conformance tests):** + +``` +cursor = (from, parent_hash | ⊥) +loop: + r = QUERY(from = cursor.from, expected_parent = cursor.parent_hash, …) + case OK(emitted, L): apply(emitted); cursor = (L + 1, own hash_at(L)) + # = last emitted block's hash; unchanged when nothing emitted + case NO_DATA: wait / re-poll + case CONFLICT(hints): a = highest hint h with local_chain.has(h) + if a exists: unapply local blocks > a.number; cursor = (a.number + 1, a.hash) + else: unapply local blocks ≥ lowest(hints).number; + cursor = (lowest(hints).number, ⊥) # walk further back / restart + case RANGE_UNAVAILABLE: local history is older than the window — restart from STATUS.first + case OVERLOADED: backoff, retry +``` + +- `local_chain.has(⟨p, h⟩)` is evaluated per DEF-16: the client's own `hash_at(p)` equals + `h` — its highest applied block at or below `p` has hash `h` (an exact block match on a + densely-numbered chain). +- The protocol guarantees: a client that always supplies `expected_parent` never applies + two blocks from different forks at adjacent heights without an intervening explicit + rollback (INV-23). Reorg recovery cost is bounded by reorg depth / + `P-CONFLICT-WINDOW` round trips. + +## 8. Error taxonomy + +Every failure of a read operation MUST map to exactly one class (INV-26); classes are +stable API surface. An error response carries no block data. + +| Class | Trigger | Retryable | +|---|---|---| +| `MALFORMED_REQUEST` | RP-1 violations | no — fix the request | +| `UNSUPPORTED_QUERY` | dialect expressible but not served here | no | +| `KIND_MISMATCH` | dialect ≠ dataset kind | no | +| `UNKNOWN_DATASET` | dataset not configured | no | +| `RANGE_UNAVAILABLE` | `from < first` (window moved past it) | no — re-anchor upward | +| `ITEM_UNAVAILABLE` | selection touches an absent item collection (RP-8) | no | +| `NO_DATA` | `from` above watermark after bounded wait (RP-5/6) | yes — poll | +| `CONFLICT` | anchored-ancestry mismatch (RP-11) | yes — after re-anchoring | +| `OVERLOADED` | admission control (RP-3) / waiter cap | yes — backoff | +| `FORBIDDEN` | SET-RETENTION on a non-External dataset | no | +| `INTERNAL` | anything else; MUST be a defect, tracked by OB-5 error budget | maybe | + +- **RP-15 (Streaming truncation caveat).** Once a success status is committed and + streaming has begun, a mid-stream failure MUST terminate the stream such that the bytes + already sent remain a valid, well-formed prefix per §5 (a decodable response whose + emitted blocks satisfy all response invariants). The truncation is then observationally + identical to a budget stop; clients rely on RP-9/RP-10, not on transport signals, for + completeness. Such truncations MUST be counted observably (OB-4) and SHOULD be signaled + in-band where the binding allows. +- **RP-16 (No partial-then-error).** An operation MUST NOT emit block data and then + report an error class for the same response; errors are decided at admission / + first-computation time (before the success status), truncation afterwards (RP-15). + +## 9. Resource bounds (read side) + +- **RP-17** Per-response resource use is bounded: emission weight by `P-RESP-WEIGHT` + (+ one block), buffering by `P-RESP-FLUSH`, execution wall time by `P-QUERY-TIME` + (LIV-3). Request size is bounded by `P-BODY-LIMIT`. +- **RP-18** A slow or disconnected client MUST cost only its own bounded buffers; its + query MUST be cancelled/abandoned promptly after disconnect and MUST NOT pin snapshots + or execution slots beyond `P-QUERY-TIME` (CN-7, HZ-9). diff --git a/crates/hotblocks/spec/05-consistency-and-durability.md b/crates/hotblocks/spec/05-consistency-and-durability.md new file mode 100644 index 00000000..f164f656 --- /dev/null +++ b/crates/hotblocks/spec/05-consistency-and-durability.md @@ -0,0 +1,89 @@ +# 05 — Consistency and durability + +This document defines the consistency model binding the write path (03) to the read path +(04), and what survives failures. The corresponding invariants are INV-10, INV-20, +INV-30/31, INV-35/36, INV-40…44. + +## 1. Commit model + +- **CN-1 (Total order per dataset).** All transitions of one dataset are totally ordered; + each committed state carries a version `ver` (DEF-10). There is no cross-dataset order. +- **CN-2 (Atomic visibility).** Readers observe committed states only. No intermediate + state — a partially applied batch, a truncated-but-not-yet-replaced fork, a half-trimmed + window — is ever observable (INV-10). "Observable" includes every read operation and + every crash-recovered state. + +## 2. Isolation + +- **CN-3 (Snapshot isolation for queries).** Each query executes against one committed + state (INV-20). Concurrent transitions never affect a running query's results; a query + never blocks a transition and a transition never blocks admitted queries (performance + coupling aside — see LIV-8). +- **CN-4 (Real-time monotonic reads).** If any read reflecting version `v` completes + before another read of the same dataset starts (wall-clock order, any clients), the + later read reflects `v' ≥ v` (INV-31). Together with CN-1 this makes watermark reads + linearizable per dataset. +- **CN-5 (Watermark coherence).** All values inside one response's watermark set (head, + fin, first) MUST come from one committed state. Response metadata MAY additionally + expose a fresher head *number* than the query snapshot, but never an older one, and + never a mixed (number, hash) pair. + +## 3. Durability + +Durability is two-tiered; both tiers preserve well-formedness — the difference is only +*how much of the recent suffix of commits* survives. + +- **CN-6 (Process-crash durability).** After a process crash or kill (host survives), + recovery MUST restore, per dataset, exactly the last committed state + (`P-DUR-PROCESS = 0` lost commits). +- **CN-6b (System-crash durability).** After a host/OS/power failure, recovery MUST + restore, per dataset, *some* committed state whose age is bounded by `P-DUR-SYSTEM` + (bounded suffix loss). Never a state that was not committed; never a mixture of two + states (INV-40). +- **CN-7 (Snapshot lifetime).** Read snapshots are short-lived — bounded by + `P-QUERY-TIME` — so resources pinned by MVCC (superseded data kept alive for readers) + are bounded (HZ-9). No API hands out unbounded-lifetime snapshots. +- **CN-8 (Clock independence).** Correctness (all INV-*) MUST NOT depend on wall-clock + values: not on block timestamps, not on host clock monotonicity across restarts. Clocks + parameterize only budgets, backoffs, and observability. + +## 4. Recovery contract + +- **CN-9 (Recovery = committed state).** The recovered state per dataset is a committed + state in every field — segment, anchor (number *and* hash), fin, retention, kind. + Derived/cached values MUST be reconstructed to exactly the committed values; a + reconstruction that differs from what was committed (e.g. a wrong recovered anchor + hash) violates INV-40 even if the segment itself is intact. +- **CN-10 (Recovery independence).** Datasets recover independently; one dataset's + recovery failure (corrupt state, config mismatch) MUST NOT prevent recovery and serving + of others (INV-36). The failed dataset enters a distinct alarmed state (OB-9). +- **CN-11 (Recovery idempotence).** Crash during recovery, followed by another recovery, + converges: repeated crash/recover cycles do not accumulate damage or unbounded residue + (INV-42). Build residue from interrupted writes (data written but never committed) is + invisible (CN-2) and MUST eventually be collected (RS-10). +- **CN-12 (Format compatibility).** On startup the service MUST verify it can interpret + the persisted state (format/version compatibility). Incompatibility MUST be refused + loudly at startup, not discovered as scattered runtime decode errors (INV-43). +- **CN-13 (Maintenance transparency).** Background maintenance — physical reorganization, + deferred deletion, space reclamation — MUST NOT change any committed logical state or + any observable result (INV-17, INV-41). The single sanctioned exception: reader-free + boot-phase maintenance (RS-8), which still MUST NOT change logical state, only physical + representation. One narrow allowance on the read side: the choice of RP-9 *boundary + markers* (header-only emissions of non-matching covered blocks) MAY vary with physical + layout and hence with maintenance; matching content, item sets, coverage, and all + watermarks MUST NOT. + +## 5. Concurrency between subsystems + +Per dataset, three actors touch state concurrently: the writer (single, WP-15), readers +(many), maintenance (background). The matrix of required non-interference: + +| | vs Writer | vs Readers | vs Maintenance | +|---|---|---|---| +| **Writer** | single writer (WP-15) | never blocks admitted readers; readers never see partial writes (CN-2/3) | maintenance never alters what the writer committed (CN-13) | +| **Readers** | — | independent snapshots | reclamation never invalidates a live reader's data (INV-41) | +| **Maintenance** | — | — | idempotent, crash-safe (CN-11) | + +Cross-dataset: logical independence is absolute (INV-35). *Performance* independence is a +liveness/performance requirement (LIV-8, HZ-1), not an isolation invariant — shared +resources exist, but starvation must be bounded. diff --git a/crates/hotblocks/spec/06-invariants.md b/crates/hotblocks/spec/06-invariants.md new file mode 100644 index 00000000..81fad3d4 --- /dev/null +++ b/crates/hotblocks/spec/06-invariants.md @@ -0,0 +1,268 @@ +# 06 — Invariant catalog (safety) + +Every invariant below MUST hold. Scopes: + +- **[state]** — holds in every observable committed state (every snapshot, every + recovered state). +- **[transition]** — relates consecutive committed states of one dataset. +- **[response]** — holds for every read response. +- **[recovery]** — holds across crash/restart boundaries. + +Each entry: statement, why it matters, and how a black-box harness checks it +(test classes CT-* are defined in [12-conformance-tdd.md](12-conformance-tdd.md)). + +--- + +## A. Structural — the stored chain + +**INV-1 — No missing blocks.** [state] +Numbers strictly ascend, and every stored block is the child of the one below it: +`∀ i < n: b_{i+1}.parent_number = b_i.number`. The window holds every *block* of the chain +between its ends — but not every *number*: a slot-numbered chain (Solana) leaves holes where no +block was produced (DEF-1). +*Why:* the window is a contiguous run of the chain. A hole in the numbering is not a hole in the +data, and treating one as the other is how a correct Solana window gets rejected. +*Check:* CT-1 model comparison, dense and sparse scripts; CT-5 scanning reads. + +**INV-2 — Internal linkage.** [state] +`∀ i < n: b_{i+1}.parent_hash = b_i.hash`. +*Why:* the window is one chain, not a set of blocks. +*Check:* CT-1/CT-5 with linkage fields projected. + +**INV-3 — Anchor linkage.** [state] +`anchor.number < first(D)`, and if `anchor.hash ≠ ⊥` then `b_1.parent_hash = anchor.hash`. +(On a densely-numbered chain `anchor.number = first(D) − 1`; on a slot-numbered one the parent +of `b_1` may sit further down.) +*Why:* the window's attachment below is what fork recovery at the window edge relies on. +*Check:* CT-1; CT-2 after restarts (see INV-40). + +**INV-4 — Kind and schema conformance.** [state] +`kind` never changes for a dataset identity (except through DROP+CREATE); every stored +block's payload conforms to the kind's schema; every item's block number equals its +enclosing block's number. +*Check:* CT-1, CT-5 structural validators. + +**INV-5 — Watermark bounds.** [state] +If `seg = ∅` then `fin = ⊥`. Otherwise, if `fin ≠ ⊥` then +`first(D) ≤ fin.number ≤ head(D).number`. +*Why:* a finalized pointer outside the window is meaningless and breaks fork-floor logic. +*Check:* CT-1 (assert on every STATUS read). + +**INV-6 — Finalized-on-chain.** [state] +If `fin ≠ ⊥` then the stored block at height `fin.number` has hash `fin.hash`. +*Why:* finality must describe the chain actually served, else finalized-only reads lie. +*Check:* CT-1; CT-4 with equivocating-finality sources (GAP-4). + +**INV-7 — Provenance fidelity.** [state] +Every stored block was delivered by a configured source, and all queryable field values +equal the source-delivered values for that `⟨number, hash⟩`. The service never invents, +mutates, or transplants data between blocks. +*Check:* CT-1/CT-6 round-trip against the source simulator's ledger. + +--- + +## B. Transition legality + +**INV-10 — Atomic transitions.** [transition] +Every observable state change is exactly one transition from the catalog (03 §2), applied +atomically: no reader or recovery ever observes an intermediate (partial batch, partial +block, truncation-without-replacement, half-trim). +*Check:* CT-3 concurrent readers during heavy write/fork/trim traffic; CT-2 crash points. + +**INV-11 — Append at the tail only.** [transition] +`EXTEND` leaves `seg[‥next]` and the anchor unchanged; new blocks attach only above the +previous head. +*Check:* CT-1 model. + +**INV-12 — Finality monotonicity.** [transition] +`fin.number` never decreases while defined, and at a fixed height the finalized hash never +changes. `fin` may become `⊥` only via RETAIN (window passing above it), RESET, or DROP. +*Why:* clients treat finalized data as irreversible-while-retained. +*Check:* CT-1 (monitor every FINALIZED-HEAD read); CT-4 stale/regressing finality sources. + +**INV-13 — Finalized immutability.** [transition] +No `REPLACE` removes or alters blocks at heights `≤ fin.number`. Finalized blocks leave +the store only via RETAIN / RESET / DROP. +*Check:* CT-1/CT-4 fork storms around the finality boundary. + +**INV-14 — Fork floor.** [transition] +For every `REPLACE(from, B)`: `from > fin.number` (when `fin ≠ ⊥`) **and** +`from ≥ first(D)`. A deeper divergence is representable only as RESET (explicit, alarmed). +*Why:* silent rollback below the window or below finality corrupts continuation clients. +*Check:* CT-4 deep-fork corpus (GAP-3). + +**INV-15 — Retention trims prefix only.** [transition] +`RETAIN` removes only blocks with numbers below its `from`; it never creates gaps, never +touches the suffix, never lowers `first(D)`. +*Check:* CT-1 with retention churn. + +**INV-16 — Frame condition.** [transition] +Without a triggering input (source event, retention trigger/instruction, operator action), +the logical state does not change: `ver` stable ⇒ identical observable state. +*Check:* CT-1 quiescence comparisons; CT-7 soak with idle periods. + +**INV-17 — Maintenance transparency.** [transition] +Background maintenance never changes logical state: for any query, results are identical +with maintenance on or off (metamorphic property) — identical up to the RP-9 +boundary-marker allowance (CN-13): *which* non-matching covered blocks appear as +header-only records may differ; matching content, item sets, and coverage may not. +*Check:* CT-7 A/B soak; CT-3. + +**INV-18 — Anchor carry-over.** [transition] +After a trimming `RETAIN(from)`, the new anchor is `⟨from − 1, hash of from's preceding +block in the pre-trim state⟩` (`hash_at(from − 1)`, DEF-16) — preserved from data, not +recomputed or dropped. +*Check:* CT-1 (STATUS/first-block checks after trims + CONFLICT hints at the window edge). + +--- + +## C. Read semantics + +**INV-20 — Snapshot isolation.** [response] +Every response is computed against exactly one committed state; no response mixes two +states (torn reads are impossible, including during forks and trims). +*Check:* CT-3 — readers hammering during fork/trim storms; any response mixing old and new +fork blocks at linked heights fails INV-2 within the response. + +**INV-21 — Response well-formedness.** [response] +Emitted blocks: strictly ascending, unique numbers, all inside coverage `[from, L]`; +coverage gap-free; every emitted item belongs to its emitting block; blocks truncate whole +(never partial item sets due to budget). +*Check:* CT-5/CT-6 structural validators on every response. + +**INV-22 — Completeness and determinism within coverage.** [response] +For the response's snapshot `S` and coverage `[from, L]`: every block in coverage matching +the selection is emitted (all covered blocks when `include_all`), each with **all** its +matching items projected to the selected fields; the coverage-end block is always emitted +(RP-9 carrier, header-only when unmatched). Matching blocks and their item content are a +deterministic function of `(S, query, coverage)`. Additional covered non-matching blocks +MAY appear as header-only boundary markers and are exempt from the determinism clause +(they may vary with physical layout, CN-13) — they MUST still be true blocks of `S` +inside coverage. +*Check:* CT-1/CT-6 against the reference model evaluated over the same coverage, with the +marker exemption. + +**INV-23 — Anchored ancestry.** [response] +If a query supplies `expected_parent` and succeeds, every emitted block lies on the unique +chain extending that parent (the first covered stored block's +`parent_hash = expected_parent = hash_at(from − 1)`, DEF-16). If the assertion fails +against the snapshot, the response is `CONFLICT` with zero blocks and hints that are true +`⟨p, hash_at(p)⟩` entries of the snapshot's chain ending at `from − 1` (or the nearest +stored position below), ascending, non-empty. +*Check:* CT-4 fork corpus with anchored clients (the client-recovery algorithm of 04 §7 +must never assemble a cross-fork chain). + +**INV-24 — Finalized-only containment.** [response] +A `QUERY-FINALIZED` response's coverage never exceeds `fin(S).number`, and all emitted +blocks lie on the finalized prefix of `S`. While retained and while `fin` is defined, the +content of a once-finalized height never changes across responses (follows from INV-12/13). +*Check:* CT-4: finalized-only pollers must never observe two hashes at one height. + +**INV-25 — Progress.** [response] +A successful query whose effective range is non-empty has coverage of at least one block +(`L ≥ from`), and if the first covered block matches the selection it is emitted even if +it alone exceeds the response budget. +*Why:* rules out client livelock on oversized blocks or zero-progress responses. +*Check:* CT-6 with pathological giant blocks; CT-5. + +**INV-26 — Error soundness.** [response] +Every failure maps to exactly one class of the taxonomy (04 §8) under exactly its +specified trigger; error responses carry no block data; no trigger condition yields +`INTERNAL`. +*Check:* CT-5 error-matrix suite. + +**INV-27 — Range honesty.** [response] +No response emits a block outside `[from, min(to, hi(S))]`; no response emits a block not +present in its snapshot (never fabricated, never resurrected-after-trim). +*Check:* CT-1/CT-3. + +--- + +## D. Watermark reporting + +**INV-30 — Watermark honesty.** [response] +Every HEAD / FINALIZED-HEAD / STATUS value equals the corresponding field of some +committed state of that dataset; `⟨number, hash⟩` pairs are never mixed across states. +*Check:* CT-1 — every reported watermark must appear in the model's committed history. + +**INV-31 — Real-time monotonic reporting.** [response] +Reads reflect versions monotonically w.r.t. real-time order (CN-4): once any read reflects +version `v`, no later-started read reflects `v' < v`. In particular a reported head is +immediately queryable: a query admitted after a HEAD response reflecting version `v` runs +on a snapshot `≥ v`. +*Check:* CT-3 interleaved HEAD+QUERY clients (advertise-then-404 races fail this). + +--- + +## E. Multi-dataset isolation + +**INV-35 — Logical independence.** [state] +The committed state of dataset `D` is a function of `D`'s own inputs (its sources, its +retention instructions, its config) only. No operation on `D'` changes any observable of +`D`. +*Check:* CT-8 noisy-neighbor suites (differential: run D's script with and without D' +active; observables of D identical up to timing). + +**INV-36 — Failure containment.** [state] +A dataset in any failure/alarm/reset state does not corrupt other datasets' state or +render their operations incorrect. (Shared-fate *performance* effects are governed by +LIV-8, not permitted to become correctness effects.) +*Check:* CT-8 with one dataset driven into every FM-* fault while others run CT-1 checks. + +--- + +## F. Durability and recovery + +**INV-40 — Committed-state recovery.** [recovery] +After process crash/kill: recovery restores exactly the last committed state per dataset +(CN-6). After system crash: some committed state within `P-DUR-SYSTEM` (CN-6b). All +fields, including `anchor.hash` and `fin`, equal those of that committed state — recovery +never yields a state that was never committed, a mixture, or a well-formed-but-different +reconstruction. +*Check:* CT-2 kill-point matrix + model comparison after restart (GAP-2). + +**INV-41 — Reclamation invisibility.** [state] +Physical space operations (deferred deletion, reorganization, file-level reclamation) +never change logical state and never invalidate data reachable by any live reader. +Exception: the boot-phase reader-free mode (RS-8), in which no reader can exist by +construction — logical state still MUST be unchanged. +*Check:* CT-3 readers during cleanup churn; CT-7 soak. + +**INV-42 — Recovery idempotence and residue convergence.** [recovery] +Repeated crash/recovery cycles converge to the same committed state; invisible residue +from interrupted writes is bounded and eventually collected (RS-10) — residue never +becomes visible and never permanently blocks reclamation. +*Check:* CT-2 crash-during-recovery; CT-7 residue accounting via OB-6. + +**INV-43 — Boot validation.** [recovery] +On startup, configuration is validated against persisted state: kind mismatch, pinned +anchor mismatch, or format incompatibility (CN-12) MUST cause an explicit refusal +(dataset-level alarm or startup failure with a distinct diagnostic) — never silent +reinterpretation, never silent destruction (see INV-44 for the sanctioned destructive +paths). +*Check:* CT-5 boot matrix (config × pre-state). + +**INV-44 — Destructive operations are explicit.** [transition] +Data leaves the store only through: RETAIN per policy, REPLACE of the volatile suffix, +RESET under its defined triggers (WP-6b / WP-9 / downward retention bound, WP §2.5 / +operator), or DROP of datasets removed from configuration. Each destructive path is deliberate, documented, and observable (OB-9). +There is no other code path by which committed blocks disappear. One non-transition +exception, scoped by CN-6b: a host/power failure may rewind a dataset to an earlier +*committed* state — a bounded suffix of recent commits lost, never a targeted deletion, +never a state that was not committed. A process-level crash loses nothing (CN-6). +*Check:* CT-1 model (any unexplained disappearance fails); CT-2 (a process crash must not +eat committed blocks; a system crash may only rewind to a committed state within +`P-DUR-SYSTEM` — ties INV-40). + +--- + +## Reading the catalog in tests + +A minimal harness assertion set that touches most of the catalog on every step: + +1. After every observed commit (via watermark polling): STATUS + full-window structural + scan ⇒ INV-1..6, 15, 18, 27, 30. +2. Every response through validators ⇒ INV-20..27. +3. Model diff at quiescence ⇒ INV-7, 10, 11, 16, 17, 35, 44. +4. Kill/restart cycles ⇒ INV-40, 42, 43. +5. Fork/finality corpora ⇒ INV-12, 13, 14, 23, 24, 31, 36. diff --git a/crates/hotblocks/spec/07-liveness.md b/crates/hotblocks/spec/07-liveness.md new file mode 100644 index 00000000..a67dcfcb --- /dev/null +++ b/crates/hotblocks/spec/07-liveness.md @@ -0,0 +1,108 @@ +# 07 — Liveness properties + +Safety (06) says nothing bad happens; this document says the system actually makes +progress. Liveness properties are stated with explicit environmental preconditions and +bounds, so they are testable (CT-6/CT-7) and monitorable (11-observability). + +## 0. Environmental definitions + +- **Healthy source set**: at least one configured source is reachable, serves the + dataset's chain from the service's resume position with parent-hash-consistent data, + and its own tip advances (or holds) normally. +- **Adequate resources**: host CPU, memory, and storage I/O below saturation; free disk + above the minimum operating threshold (`P-DISK-FLOOR`); no fault injection active. +- **Quiescent**: no source input, no retention instructions, no client load for a settling + period `P-QUIESCENCE`. + +All time bounds refer to sustained conditions — transient blips inside a bound are +compliant. Each LIV property names the observable(s) that witness it. + +--- + +**LIV-1 — Ingestion progress.** +Given a healthy source set and adequate resources, `head(D)` converges to the source tip: +ingest lag (SLI-1) is eventually bounded by `P-LAG-STEADY`, and from any backlog state the +service catches up at a rate ≥ `P-CATCHUP-RATE` until steady lag is reached. +*Witness:* OB-2 progress heartbeat, SLI-1. *Tests:* CT-6, CT-7. + +**LIV-2 — No unbounded write stall.** +Under the LIV-1 preconditions, the interval during which a dataset makes **zero** commit +progress while its sources offer new data never exceeds `P-STALL-BUDGET`. This bound +covers all internal causes: storage backpressure, maintenance debt, shared-pool +contention, deferred-deletion churn. +*Why this exists:* multi-minute all-dataset freezes have been observed post-deploy; this +property is the formal target the stall harness must enforce (GAP-1). +*Witness:* OB-3 stall detector. *Tests:* CT-7 soak + CT-6 stress. + +**LIV-3 — Query termination.** +Every admitted query terminates (success or defined error) within +`P-QUERY-TIME + P-SCHED-SLACK`, regardless of concurrent write/fork/trim/maintenance +activity. No query hangs indefinitely. +*Witness:* SLI-2 latency distributions, OB-4. *Tests:* CT-3, CT-6. + +**LIV-4 — Waiter release.** +Every head-waiting query (RP-5) is released within `P-HEAD-WAIT + P-SCHED-SLACK` — by +data arrival or `NO_DATA` — including during ingestion stalls and shutdown. +*Witness:* OB-4 waiter gauges. *Tests:* CT-3. + +**LIV-5 — Bounded, decoupled startup.** +After process start: (a) the service begins *accepting* connections within +`P-STARTUP-ACCEPT`, independent of state size; (b) each dataset becomes readable within +`P-STARTUP-READY(state)` — a bound that is a function of that dataset's own state size, +not of other datasets'; (c) readiness is externally observable per dataset (OB-8). +Startup work that scales with total state (recovery scans, boot maintenance) MUST NOT +push (a) beyond its bound. +*Witness:* OB-8 phase timings. *Tests:* CT-2, CT-6 (startup benchmark). + +**LIV-6 — Recovery liveness.** +After any crash, the service reaches LIV-5 readiness and resumes LIV-1 progress without +operator intervention, for every dataset whose persisted state is intact; damaged datasets +alarm (CN-10) without blocking the rest. +*Tests:* CT-2. + +**LIV-7 — Reclamation liveness.** +After logical deletions (retention, forks, drops), physical space converges: within +`P-RECLAIM-LAG`, disk usage returns to within the amplification bound RS-6. Deletion debt +(logically deleted but physically present data) is eventually zero in a quiescent system. +*Witness:* OB-6 space accounting. *Tests:* CT-7 (GAP-6). + +**LIV-8 — No cross-dataset starvation.** +One dataset's workload — ingest volume, query load, fork storms, retention churn, +maintenance debt — MUST NOT stall another dataset's ingestion (beyond `P-STALL-BUDGET`) +or starve its queries indefinitely. Shared resources may slow neighbors proportionally; +they may not halt them. +*Why:* the shared write path makes global halts the observed failure mode of the current +system (GAP-1); this property forbids the class. +*Tests:* CT-8 noisy-neighbor. + +**LIV-9 — Fork convergence and alarmed divergence.** +(a) After an arbitrated fork signal, the dataset commits the corresponding REPLACE and +resumes tip-following within `P-FORK-CONVERGE` (bounds the WP-7 stale-fork visibility +window). (b) Divergences that cannot be applied safely — hints entirely below `fin`, +finality contradictions, un-linkable sources (below-window divergence, WP-6b) — reach a +distinct alarmed state within +`P-ALARM` instead of silent infinite retry (GAP-5). Reads keep being served from the last +good state throughout. +*Tests:* CT-4. + +**LIV-10 — Overload recovery.** +When offered load exceeds capacity, the service sheds with `OVERLOADED` (RP-3) while +continuing to serve admitted work; when load subsides, service returns to normal within +`P-RECOVERY-SETTLE` with no residual degradation (no leaked slots/waiters/buffers, no +hysteresis lockup). +*Tests:* CT-6 load steps. + +**LIV-11 — Retention keeps up.** +Under LIV-1 preconditions, window-trim debt is bounded: `next(D) − first(D)` never exceeds +`k + P-RETENTION-SLACK + (catchup transient)` for `Window(k)` datasets (positions, DEF-9 — +on a slot-numbered chain `span(D)` is smaller still); External +instructions apply within `P-RETENTION-APPLY` (WP-11). +*Tests:* CT-7. + +**LIV-12 — Graceful shutdown.** +On a shutdown request, the service stops admitting work, completes or cleanly truncates +(RP-15) in-flight responses, commits or abandons in-flight batches atomically (INV-10), +and exits within `P-SHUTDOWN` — without panic-class failures and while remaining +crash-equivalent for durability purposes (a shutdown is never *worse* than a crash: +INV-40 still holds). +*Tests:* CT-2 (shutdown as a scheduled kill-point class). diff --git a/crates/hotblocks/spec/08-failure-model.md b/crates/hotblocks/spec/08-failure-model.md new file mode 100644 index 00000000..88ac9391 --- /dev/null +++ b/crates/hotblocks/spec/08-failure-model.md @@ -0,0 +1,86 @@ +# 08 — Failure model + +This document enumerates the faults the system MUST tolerate, and the required response to +each. "Required response" uses four verbs: + +- **mask** — absorb; no externally visible effect beyond timing; +- **degrade** — keep serving with reduced capability, within spec'd bounds; +- **fail-safe** — refuse the affected operation/dataset with a defined error, preserving + all invariants; +- **alarm** — enter/emit a distinct observable state (OB-9) within `P-ALARM`. + +## 0. Global robustness requirements + +- **FM-1 (No externally-triggered termination).** No input from any source or client — + malformed, oversized, adversarial, or pathological-but-valid — may terminate, deadlock, + or permanently wedge the process, nor corrupt any dataset. This includes payload + contents (arbitrary bytes, invalid text encodings, extreme values, huge collections) + flowing through the write path, and any query through the read path. *Tests:* CT-9 fuzz + (GAP-11, GAP-12). +- **FM-2 (Fault classification).** Every failure is classified transient vs integrity + (WP-17); transient → bounded-backoff retry forever; integrity → fail-safe + alarm. A + fault MUST NOT oscillate between classifications for the same cause. +- **FM-3 (Containment).** Fault handling is per-dataset wherever the fault is + per-dataset; per-source wherever per-source. A fault's blast radius (which operations + degrade) MUST be documented by this table and no larger. + +## 1. Source faults (FM-SRC) + +| # | Fault | Required response | +|---|---|---| +| FM-SRC-1 | Source unreachable / times out / disconnects mid-stream | mask: per-source backoff `P-SOURCE-BACKOFF`, fail over to other sources; dataset degrades to "stale but serving" if all sources down; alarm on `P-SOURCE-DOWN-ALARM` of continuous unavailability | +| FM-SRC-2 | Stale data: blocks/watermarks at or below already-stored positions | mask: deduplicate/ignore (WP-16); MUST NOT crash-loop or regress state | +| FM-SRC-3 | Unlinked or out-of-order delivery within a stream (broken parent linkage, non-ascending numbers; number holes on slot-numbered chains are *not* a fault — DEF-1) | fail-safe per run: reject the run (WP-2), reset that source's session, re-request; no state change | +| FM-SRC-4 | Structurally invalid blocks: broken linkage, wrong numbering, schema-invalid payload | fail-safe: reject run; quarantine source after `P-SOURCE-STRIKES` consecutive rejections; alarm | +| FM-SRC-5 | Finality violation: finality hash contradicting stored finalized chain; fork hints entirely below `fin`; finality regression with conflicting hash | fail-safe + alarm (WP-6/WP-8, LIV-9b): do not apply; keep serving last good state; do NOT silently retry forever (GAP-5) | +| FM-SRC-6 | Equivocation between sources (different chains at same heights) | mask via arbitration (WP-4) while a quorum/timeout resolves; if unresolvable, prefer current stored chain, alarm on persistent split | +| FM-SRC-7 | Representation drift (hash casing/format differing across sources of one dataset) | fail-safe: treated as linkage mismatch (exact-match rule DEF-2); alarm on persistent mismatch pattern | +| FM-SRC-8 | Fork storm: rapid repeated reorgs, possibly deeper than the window | degrade: apply per WP-6 within floors (INV-14); deeper-than-window ⇒ RESET + alarm; throughput may drop, correctness never | + +## 2. Storage faults (FM-STOR) + +| # | Fault | Required response | +|---|---|---| +| FM-STOR-1 | Slow storage (I/O saturation, maintenance debt) | degrade: writes throttle; bounded by LIV-2 stall budget; reads keep serving; OB-3 signals pressure | +| FM-STOR-2 | Disk approaching full | alarm at `P-DISK-FLOOR`; degrade per documented policy: reads MUST keep working; writes MAY pause (counts toward stall accounting, exempt from LIV-2 only while below floor); the system MUST NOT corrupt state and MUST provide a documented, bounded recovery path (RS-8 boot maintenance) that does not require scratch space proportional to reclaimable data | +| FM-STOR-3 | Disk full (hard) | fail-safe: no committed-state corruption (INV-40 holds for the pre-full history); recovery path documented; startup after freeing space restores service | +| FM-STOR-4 | Detected corruption of stored state (checksum/decode failure) | fail-safe + alarm per dataset (CN-10): the damaged dataset stops, others serve; corruption MUST be detected (never returned as data), and MUST NOT silently disable global functions (e.g. reclamation for everyone) without alarm (GAP-6 adjacent) | +| FM-STOR-5 | Partial write persisted at crash (torn build) | mask: invisible by CN-2/INV-40; residue collected (RS-10) | + +## 3. Process faults (FM-PROC) + +| # | Fault | Required response | +|---|---|---| +| FM-PROC-1 | Crash / kill -9 at any instant | mask via recovery: INV-40 (exact last committed state), LIV-6 | +| FM-PROC-2 | Crash during recovery/boot maintenance | mask: idempotent recovery (INV-42) | +| FM-PROC-3 | Host/OS/power failure | degrade: bounded suffix loss `P-DUR-SYSTEM` (CN-6b); state still well-formed | +| FM-PROC-4 | Graceful shutdown signal | LIV-12: drain, no panic-class exits, crash-equivalent durability | +| FM-PROC-5 | Internal defect (panic-class error) in one subsystem task | contain: affected dataset/query fails-safe + alarm; process survives where containment is possible; repeated hits → FM-2 integrity classification, not infinite hot-loop | + +## 4. Client faults (FM-CLI) + +| # | Fault | Required response | +|---|---|---| +| FM-CLI-1 | Malformed / oversized / schema-invalid requests | fail-safe: taxonomy errors (RP-1), FM-1 holds | +| FM-CLI-2 | Pathological valid queries (huge ranges, dense filters, giant blocks) | degrade: budgets bound cost (RP-17); progress guaranteed (INV-25); other clients unaffected beyond fair-share slowdown | +| FM-CLI-3 | Slow reader / stalled connection | mask: bounded buffers, cancel on disconnect (RP-18); never pins snapshots/slots beyond `P-QUERY-TIME` | +| FM-CLI-4 | Thundering herd (mass reconnect/poll) | degrade: admission control `OVERLOADED` (RP-3/LIV-10); waiter caps; no collapse; recovery when herd subsides | +| FM-CLI-5 | Disconnect mid-stream | mask: RP-15 truncation semantics; server resources released promptly | + +## 5. Operator faults (FM-OP) + +| # | Fault | Required response | +|---|---|---| +| FM-OP-1 | Config/state contradictions: kind change, pinned-anchor mismatch, format incompatibility | fail-safe at boot: INV-43 refusal with distinct diagnostics; unaffected datasets serve (CN-10) | +| FM-OP-2 | Dataset removed from config | defined destructive path: DROP at boot (INV-44); MUST be logged/observable; (hardening SHOULD: a grace/confirmation mechanism, since this is the highest-blast-radius single config edit) | +| FM-OP-3 | Two service instances over one store | fail-safe: detect divergence, stop the losing writer per dataset, alarm (WP-15); MUST NOT interleave-corrupt | +| FM-OP-4 | Retention mistakes (raise far above head, contradictory instructions) | defined semantics (WP-9/RETAIN cases): destructive outcomes are the documented ones only; observable; idempotent | +| FM-OP-5 | Restart with changed parameters (window size, budgets) | mask: state re-converges to policy (trim or backfill-forward per WP-10/WP-5); no invariant violations during convergence | + +## 6. Fault → property cross-reference + +- Crash matrix: FM-PROC-* ⇢ INV-40/42, LIV-5/6, CT-2. +- Source misbehavior corpus: FM-SRC-* ⇢ INV-7/12/13/14/23, LIV-1/9, CT-4. +- Overload & client abuse: FM-CLI-* ⇢ RP-3/17/18, LIV-3/4/10, CT-6/CT-9. +- Storage pressure: FM-STOR-* ⇢ INV-41, LIV-2/7, RS-6/8, CT-7. +- Operator: FM-OP-* ⇢ INV-43/44, CT-5 boot matrix. diff --git a/crates/hotblocks/spec/09-retention-and-space.md b/crates/hotblocks/spec/09-retention-and-space.md new file mode 100644 index 00000000..e6eb9369 --- /dev/null +++ b/crates/hotblocks/spec/09-retention-and-space.md @@ -0,0 +1,92 @@ +# 09 — Retention and space + +Retention (which blocks the dataset logically keeps) and space (what the store physically +occupies) are deliberately decoupled. This document specifies both and the contract +between them. + +## 1. Retention policies + +- **RS-1 (Policy semantics).** As defined in DEF-9 and WP §2.5: + +| Policy | Guarantee | Trim trigger | +|---|---|---| +| `Window(k)` | availability floor RS-3 + excess bound RS-4 | automatic, after commits that advance `next(D)` | +| `Pinned(from, h?)` | everything ≥ `from` kept; anchor asserted at boot (WP-9 refusal on mismatch) | only when `from` is raised by reconfiguration | +| `External` | everything ≥ last instructed bound kept; unbounded until first instruction; a *downward* instruction is a destructive re-bootstrap (RESET, WP §2.5) | SET-RETENTION (WP-11) | +| `Unbounded` | nothing trimmed | never | + +- **RS-2 (Retention dominates finality).** Trimming ignores `fin`: finalized blocks below + the retention bound are deleted, and `fin` becomes `⊥` when the window passes above it + (03 §2.5). Rationale: this is a bounded hot store, not an archive (NG1). Consequence + for clients: "finalized" means *irreversible while retained*, not *retained forever* + (INV-24 note). +- **RS-3 (Availability floor).** For `Window(k)`: at every committed state, + all blocks in `[next(D) − k, next(D) − 1] ∩ [window start after initial fill, ∞)` are + present and queryable (an interval of *positions* — DEF-9; on a slot-numbered chain it + holds ≤ `k` blocks). Trimming MUST err on the side of keeping more, never less. +- **RS-4 (Excess bound).** For `Window(k)`: eventually (once steady-state is reached and + within `P-RETENTION-APPLY` of each trigger), `first(D) ≥ next(D) − k − P-RETENTION-SLACK`. + Slack exists because trimming may be batch-granular; it is bounded, not best-effort. +- **RS-9 (Dataset removal).** DROP removes all the dataset's data logically at once; + physical reclamation follows RS-5/6. Re-creating the same identity yields a fresh + dataset (WP §2.7). + +## 2. Space model + +Definitions for accounting (all per store unless noted): + +- `live_bytes` — bytes attributable to blocks currently in some dataset's segment + (+ bounded metadata). +- `debt_bytes` — bytes attributable to logically deleted or superseded data not yet + physically reclaimed, plus invisible residue from interrupted writes. +- `disk_bytes` — actual storage footprint. + +Requirements: + +- **RS-5 (Two-phase deletion).** Logical deletion (RETAIN/REPLACE/RESET/DROP commits) is + immediate and cheap; physical reclamation is asynchronous. Between the two, deleted + data is `debt_bytes` — invisible to all reads (INV-41 keeps live readers safe via + versioning, not via keeping data visible). +- **RS-6 (Amplification bound).** In steady state, + `disk_bytes ≤ P-SPACE-AMP × live_bytes + P-SPACE-CONST`. This MUST hold under + continuous churn (window datasets trim continuously — churn IS the steady state). + Unbounded `debt_bytes` growth under any configuration is a defect. (GAP-6 history: + until 2026-07 the system reclaimed physically only in an optional boot mode and default + configurations violated this clause; routine reclaim now runs in default + configurations — deferred point deletes swept every `P-CLEANUP-PERIOD` plus engine + compaction. The bound itself is still unmeasured under churn: CT-7.) +- **RS-7 (Reclamation safety).** = INV-41. Reclamation never affects logical state or + live readers. Corollary: reclamation strategies that cannot honor reader-safety MUST be + confined to RS-8. +- **RS-8 (Boot maintenance mode).** A reader-free maintenance window at startup, before + serving begins, in which the service MAY run reader-unsafe physical operations + (bulk file reclamation, residue purge). Requirements: strictly before any reader can + exist; bounded contribution to LIV-5 budgets; idempotent (INV-42); effective on a + nearly-full disk (its purpose — FM-STOR-2/3 recovery) without scratch space + proportional to the data being reclaimed. +- **RS-10 (Residue convergence).** Invisible residue (from crashes: torn builds; from + operation: superseded internal structures) MUST be collected: residue does not + accumulate without bound across crashes (INV-42) and — critically — residue from one + dataset MUST NOT indefinitely block reclamation for other datasets. (Status: the + global low-watermark coupling now applies only to the boot-phase file unlink (RS-8); + routine compaction-based reclaim is not pinned by it. Interrupted-build residue still + leaks in default configurations because the purge is confined to the same gated boot + mode — GAP-6.) Residue age is observable (OB-6). +- **RS-11 (Deletion cost bounds).** Logical deletion cost is O(deleted-range metadata), + not O(bytes); physical reclamation runs at bounded amortized cost without violating + LIV-2 (deletion-induced maintenance debt counts inside the stall budget). Deleting a + large dataset MUST have bounded peak memory (not proportional to the dataset's size). + +## 3. Interactions + +- **Retention × finality:** RS-2 (dominates). FINALIZE below `first(D)` is ignored (WP + §2.4). +- **Retention × forks:** the fork floor is the window start (INV-14): retention determines + how deep a reorg can be absorbed in place. Operators choosing `k` MUST size it above the + chain's realistic reorg depth; deeper reorgs are RESET events (alarmed). +- **Retention × queries:** trimming during a running query does not affect it (INV-20/41); + the *next* query below the new `first` gets `RANGE_UNAVAILABLE` (RP-4). +- **Retention × recovery:** recovered state reflects committed trims exactly (INV-40); + a trim's anchor carry-over (INV-18) survives restarts. +- **Space × liveness:** reclamation lag is bounded (LIV-7); maintenance debt feeds back + into the write path only within the stall budget (LIV-2, HZ-2/HZ-5). diff --git a/crates/hotblocks/spec/10-performance.md b/crates/hotblocks/spec/10-performance.md new file mode 100644 index 00000000..07d41f43 --- /dev/null +++ b/crates/hotblocks/spec/10-performance.md @@ -0,0 +1,130 @@ +# 10 — Performance model + +This document makes performance testable: a workload vocabulary, precise SLI definitions, +target SLOs (provisional until ratified), resource-bound requirements, and a hazard +register that tells the TDD framework where degradation is expected to hide. + +## 1. Workload model + +A benchmark/soak scenario is described by these parameters (per dataset unless noted): + +| Param | Meaning | Typical envelope | +|---|---|---| +| `W-DATASETS` | concurrently hosted datasets (global) | 1 … ~50 | +| `W-BLOCK-RATE` | source block production rate | 0.1 … 50 blocks/s | +| `W-BLOCK-SIZE` | payload size distribution incl. heavy tails | 1 KB … 10s of MB | +| `W-ITEM-DENSITY` | items per block per collection | sparse … 10⁴ | +| `W-REORG` | fork frequency × depth distribution | rare-shallow … storm (bounded by window) | +| `W-FINALITY-LAG` | source finality distance behind tip | 0 … window size | +| `W-RETENTION` | policy + `k` | minutes … months of blocks | +| `W-QUERY-MIX` | tip-followers (small anchored ranges, high rate) / backfills (long ranges) / watermark pollers | site-specific | +| `W-CLIENTS` | concurrent clients incl. herd events | 1 … 10⁴ | + +Reference scenarios (used by CT-6/CT-7, defined precisely in the harness): +`S1 steady-tip` (all datasets tip-following + poller-heavy query mix), `S2 backfill-storm` +(cold clients scanning whole windows), `S3 reorg-storm`, `S4 churn-soak` (days of S1 + +retention churn), `S5 cold-start` (boot with large existing state under S1 load), +`S6 noisy-neighbor` (one dataset driven to saturation, others on S1). + +## 2. Service-level indicators (definitions) + +All SLIs are measured black-box by the harness (source simulator + client driver + +observability), per dataset unless noted. + +| SLI | Definition | +|---|---| +| SLI-1 **Ingest lag** | time from the source simulator offering block `b` at the service's resume position to `b` being query-visible (commit observed). Steady-state distribution + worst interval. | +| SLI-2 **Query latency** | admission→first byte (TTFB) and admission→completion, per query class (tip / backfill / watermark). | +| SLI-3 **Stream throughput** | decompressed payload bytes/s and blocks/s sustained by one backfill stream; aggregate under `W-CLIENTS`. | +| SLI-4 **Watermark read latency** | HEAD/STATUS round trip. | +| SLI-5 **Startup-accept time** | process start → first accepted connection (LIV-5a). | +| SLI-6 **Startup-ready time** | process start → dataset readable (per dataset; LIV-5b), and → all ready. | +| SLI-7 **Recovery time** | crash → LIV-5 readiness (breakdown by phase via OB-8). | +| SLI-8 **Space amplification** | `disk_bytes / Σ live_bytes` sampled through churn (RS-6). | +| SLI-9 **Stall time** | longest interval with zero commit progress on any dataset whose source offers data (LIV-2); plus count of intervals > 1 s. | +| SLI-10 **Error budget** | rate of `INTERNAL` (must be ~0), rate of `OVERLOADED` under nominal load (must be 0), truncation rate (RP-15). | +| SLI-11 **Head granularity** | distribution of head advancement step size and inter-commit interval at fixed `W-BLOCK-RATE` (batching-induced freshness quantization, WP-3). | +| SLI-12 **Fork convergence time** | fork signal offered → REPLACE committed (LIV-9a). | + +## 3. SLO targets (provisional) + +Targets marked ⚠ are initial engineering proposals to be ratified; "baseline" records the +currently known behavior where incident data exists. A conformance run reports pass/fail +per row under the named scenario. + +| SLI | Scenario | Target ⚠ | Known baseline | +|---|---|---|---| +| SLI-1 p99 | S1 | ≤ 2 s + batch quantum | — | +| SLI-9 max | S1/S4/S6 | ≤ 5 s; zero intervals ≥ 60 s | **351–458 s global freezes observed post-deploy** (GAP-1) | +| SLI-2 TTFB p99 (tip) | S1 | ≤ 300 ms | — | +| SLI-2 TTFB p99 (backfill) | S2 | ≤ 2 s | — | +| SLI-3 per stream | S2 | ≥ 50 MB/s decompressed | — | +| SLI-4 p99 | S1 | ≤ 50 ms | — | +| SLI-5 | S5 | ≤ 3 s, independent of state size | **~35 s refused-connection window observed** (GAP-7) | +| SLI-6 all-ready | S5 | ≤ 60 s for ~50 datasets of nominal size | ~35 s+ (same incident) | +| SLI-7 | S5 | SLI-5/6 bounds + no data loss (INV-40) | — | +| SLI-8 | S4 | ≤ 2.0× steady; bounded monotone convergence after churn bursts | unbounded growth in default config until 2026-07; reclaim path since fixed, bound unmeasured (GAP-6) | +| SLI-10 INTERNAL | all | 0 per 10⁶ requests | crashes on unsupported dialects (GAP-11) | +| SLI-11 | S1 | inter-commit ≤ max(1 s, 1/`W-BLOCK-RATE`); no artificial multi-second quantization at high rates | batch bounds imply stepping (HZ-6) | +| SLI-12 p99 | S3 | ≤ 2 s + one batch time | — | + +## 4. Resource-bound requirements + +- **PF-1 (Memory ceiling).** Total process memory is bounded by a configuration-derivable + ceiling: per-query buffers (≤ `P-RESP-WEIGHT` + `P-RESP-FLUSH` order), per-dataset + ingest accumulation (≤ `P-BATCH-BYTES` order — including for pathological payloads: + GAP-13), caches (explicit sizes), executor concurrency (`P-EXEC-SLOTS`). No workload may + drive memory unboundedly (test: CT-6 with adversarial `W-BLOCK-SIZE`/`W-ITEM-DENSITY`). + Caveat: the read side's "+ one block" allowance (RP-17, INV-25) is configuration-derivable + only if single-block size is bounded at ingest — no such bound exists today + (`P-MAX-BLOCK-BYTES`, GAP-37). +- **PF-2 (End-to-end backpressure).** Every producer→consumer edge in the write pipeline + is bounded-buffer; source intake slows when commits slow. Corollary: backpressure is the + designed response to storage pressure — but it must respect LIV-2/LIV-8 (bounded stall, + no cross-dataset halt). +- **PF-3 (Admission control, read side).** RP-3: refuse beyond capacity with + `OVERLOADED`; admitted work completes within LIV-3. Under saturation the system sheds + load; it does not thrash (LIV-10). +- **PF-4 (Fairness).** Shared execution capacity is apportioned such that no query class + or dataset can starve others indefinitely (HZ-7); a documented fair-share/priority + policy governs contention. +- **PF-5 (Maintenance budget).** Background maintenance (reclamation, reorganization) runs + within an I/O/CPU budget that preserves LIV-2 and SLI-2 targets while still meeting + LIV-7 (a two-sided constraint: too little maintenance breaks space/liveness later, too + much breaks latency now). +- **PF-6 (Startup work scheduling).** Work at boot is split: what must precede accepting + connections (minimal), what must precede per-dataset readiness (that dataset's + recovery), what can run behind serving (boot maintenance where reader-safety allows, + RS-8 constraints permitting). LIV-5 encodes the bounds. + +## 5. Hazard register + +Known mechanisms by which this class of system degrades. Each hazard names the property it +threatens and the probe the TDD framework should build. These are *risk pointers*, not +defect reports — the dated defect list lives in the gap register (12 §6). + +| HZ | Mechanism | Threatens | Probe | +|---|---|---|---| +| HZ-1 | Shared storage write path: engine-level backpressure/stall is global, halting all datasets at once | LIV-2, LIV-8 | S6 + storage-pressure injection; assert per-dataset stall budgets (GAP-1) | +| HZ-2 | Maintenance debt accumulation (deferred deletion → reorganization backlog → write throttling) | LIV-2, SLI-2 | S4 churn soak with OB-3/OB-6 tracking | +| HZ-3 | Boot work scaling with total state (recovery scans, residue purge) | LIV-5, SLI-5/6 | S5 with grown state; regression-track SLI-5 vs state size | +| HZ-4 | Retention/trim bursts (large trims committed at once) | SLI-1/2 latency spikes | step-function retention instructions under load | +| HZ-5 | Space amplification: churn outrunning reclamation; residue pinning (one dataset's artifact blocking global reclamation) | RS-6, LIV-7 | S4 with per-dataset churn skew; residue-age assertions (GAP-6) | +| HZ-6 | Freshness quantization: batch bounds delay head advance at low block rates or large blocks | SLI-1, SLI-11 | sweep `W-BLOCK-RATE` × `W-BLOCK-SIZE`; assert flush-on-tip behavior (WP-3) | +| HZ-7 | Heavy backfill queries starving tip-followers on shared executor capacity | SLI-2 (tip), PF-4 | S2+S1 mixed; per-class latency split | +| HZ-8 | Waiter herds: mass long-poll release on one commit (thundering release), waiter-cap exhaustion | LIV-4, SLI-10 | herd scenario: 10⁴ waiters, single block arrival | +| HZ-9 | Long-lived read snapshots pinning superseded data (MVCC bloat) if response lifetimes are ever unbounded | CN-7, RS-6 | slow-reader clients; assert snapshot lifetime ≤ `P-QUERY-TIME` | +| HZ-10 | Ancestry checks / conflict-hint construction scanning linearly with window size | SLI-2 (anchored tip queries) | anchored-query latency vs window size sweep | +| HZ-11 | Transaction/commit retry storms under contention (optimistic concurrency) | SLI-1, LIV-2 | multi-writer-pattern stress within one process; commit-retry observability | +| HZ-12 | Compression/encoding cost dominating small hot responses | SLI-2 TTFB | encoding on/off comparative bench | + +## 6. Performance testing requirements + +- **PF-7 (Baselining).** Every SLI has a recorded baseline per reference scenario; + CI-grade runs compare against baselines with explicit tolerances; regressions beyond + tolerance fail. +- **PF-8 (Saturation characterization).** For each scenario the harness records the knee + (throughput at which SLOs first break) so capacity planning is data, not folklore. +- **PF-9 (Overload behavior is part of the contract).** Benchmarks MUST include + beyond-capacity phases asserting RP-3/LIV-10 behavior (shed-and-recover), not only + below-capacity happy paths. diff --git a/crates/hotblocks/spec/11-observability.md b/crates/hotblocks/spec/11-observability.md new file mode 100644 index 00000000..2b4ac459 --- /dev/null +++ b/crates/hotblocks/spec/11-observability.md @@ -0,0 +1,77 @@ +# 11 — Observability + +Observability is normative here for two reasons: (1) several liveness properties are only +*testable* through observables; (2) the known failure modes of this system class (global +stalls, silent space growth, silent divergence loops) are exactly the ones that are +invisible without the signals below. "Expose" means: available through the observability +surface of the binding (13 §5) with bounded cardinality. + +## 1. Required signals + +- **OB-1 (Chain state gauges).** Per dataset: `first`, `head.number`, `fin.number`, + commit version (or an equivalent monotone commit counter), last block time (when known), + and the retention policy in force. These are the raw material for lag/progress alerting + and for CT harness quiescence detection. +- **OB-2 (Progress heartbeat).** Per dataset: time since last commit, and time since last + *offered-but-uncommitted* source data (the difference distinguishes "idle chain" from + "stalled service"). LIV-1/LIV-2 are decided from this signal. +- **OB-3 (Write-pressure signals).** Global and per shared resource: whether writes are + currently halted/throttled by the storage layer, current throttle rate, queue depths of + the write pipeline stages, commit-retry counts (HZ-11), maintenance backlog size + (HZ-2). The *existence* of a global write halt MUST be directly observable — the + historical multi-minute freezes were diagnosable only by inference (GAP-1). +- **OB-4 (Query metrics).** Per dataset × query class × outcome (success / each error + class / truncation per RP-15): counts, latency distributions (TTFB, total), emitted + bytes/blocks, coverage sizes; current in-flight and waiting counts against their caps + (`P-EXEC-SLOTS`, `P-WAITERS`). +- **OB-5 (Error budget accounting).** `INTERNAL` occurrences individually traceable + (request-correlatable); `OVERLOADED` rates; source-rejection and quarantine counts per + source (FM-SRC-4). +- **OB-6 (Space accounting).** Per store and attributable per dataset where feasible: + `live_bytes` estimate, `debt_bytes` estimate (logically-deleted-not-reclaimed + + residue), `disk_bytes`, age of oldest unreclaimed debt, and the current reclamation + blocker if any (what pins the watermark — GAP-6 made this a first-class question). +- **OB-7 (Source health).** Per dataset × source: reachability, current backoff state, + last successful delivery, rejected-run counts, arbitration disagreement events + (FM-SRC-6), finality-conflict events (FM-SRC-5). +- **OB-8 (Lifecycle phases).** Startup: timestamped phase transitions (recover → boot + maintenance → per-dataset ready → accepting) so SLI-5/6/7 are measurable in production, + not only in the harness. Liveness/readiness probes: process-accepting vs per-dataset + readable are distinct signals (LIV-5c). Shutdown: drain progress. +- **OB-9 (Alarm states).** Distinct, queryable, per-dataset alarm conditions with reason + codes: integrity conflict (WP-8/FM-SRC-5), RESET occurred (WP §2.6), boot validation + refusal (INV-43), dataset stopped (CN-10), dual-writer detected (WP-15), disk floor + breached (FM-STOR-2). Alarms are edge-triggered events *and* level-readable states. +- **OB-10 (Bounded cardinality).** All label spaces are bounded by configuration (datasets, + sources, classes, outcome enums); unbounded client-derived labels MUST be sanitized + (e.g. allowlisted client identities, "other" bucket). +- **OB-11 (Stall forensics).** When OB-2/OB-3 cross the stall budget, the service SHOULD + capture a self-diagnostic snapshot (thread/task states, pipeline queue depths, storage + engine state) sufficient to attribute the stall post-hoc. Rationale: the observed + production freezes could not be root-caused from standard metrics; capture-on-stall + turns the next occurrence into data (GAP-1). + +## 2. Property → observable mapping + +| Property | Decided by | +|---|---| +| LIV-1 ingest progress | OB-1 head vs simulator tip, OB-2 | +| LIV-2 stall budget | OB-2, OB-3 | +| LIV-3/4 query/waiter termination | OB-4 | +| LIV-5/6 startup/recovery | OB-8 | +| LIV-7 reclamation | OB-6 | +| LIV-8 isolation | OB-1/2/3 per dataset under CT-8 | +| LIV-9 fork convergence / alarmed divergence | OB-1 (version/head), OB-9, OB-7 | +| LIV-10 overload recovery | OB-4, OB-5 | +| LIV-11 retention keep-up | OB-1 (`span` vs `k`) | +| LIV-12 shutdown | OB-8 | +| RS-6 amplification | OB-6 | +| FM alarms | OB-9 | + +## 3. Requirements on the harness (informative) + +The conformance harness treats observables as part of the SUT contract: OB signals are +scraped continuously during every CT run; a liveness verdict derived from harness-side +black-box observation MUST agree with the verdict derived from OB signals — disagreement +means the observability layer itself is defective (lying metrics are treated as failures, +same as lying data). diff --git a/crates/hotblocks/spec/12-conformance-tdd.md b/crates/hotblocks/spec/12-conformance-tdd.md new file mode 100644 index 00000000..097f30c9 --- /dev/null +++ b/crates/hotblocks/spec/12-conformance-tdd.md @@ -0,0 +1,348 @@ +# 12 — Conformance and TDD framework + +This document turns the spec into a test program: the reference model (oracle), the +harness architecture, the test-class taxonomy, the traceability matrix, and the dated gap +register that seeds the hardening backlog. + +Statuses and the gap register reflect the state of knowledge as of **2026-07-12** and are +expected to change; everything else in this document is stable methodology. + +The harness described here exists: [`crates/hotblocks-harness`](../../hotblocks-harness). +Phase 0 of §7 is done — CT-1 runs a happy-path script green against the real binary. + +## 1. Harness architecture + +``` +┌────────────────┐ source protocol ┌─────────────┐ interface binding ┌───────────────┐ +│ Source │────────────────────▶│ │◀──────────────────────│ Client driver │ +│ simulator │ blocks/forks/ │ SUT │ queries/watermarks/ │ + validators │ +│ (scripted │ finality/faults │ (black │ retention │ │ +│ chain gen) │ │ box) │ └───────┬───────┘ +└──────┬─────────┘ └──────┬──────┘ │ + │ │ observability (OB-*) │ + │ ┌─────────────┐ ▼ │ + └───────────▶│ Reference │◀── scraper ─┘ │ + same script │ model │◀───────────────── responses/verdicts ────────────┘ + │ (oracle) │ + └─────────────┘ + + fault injectors: process kill/restart, storage throttle/fill, network faults +``` + +Components: + +1. **Source simulator** — a deterministic, scriptable chain generator implementing the + source side of the binding: block production at `W-BLOCK-RATE`, forks per `W-REORG`, + finality per `W-FINALITY-LAG`, plus the full FM-SRC fault repertoire on demand. It + keeps a **ledger** of everything it served (for INV-7 provenance checks). Payload + generation is kind-parametric with adversarial knobs (sizes, encodings, densities). +2. **Client driver** — implements the client algorithms of 04 §7 (anchored follower, + backfill scanner, watermark poller) and the structural validators (§6 below). +3. **Reference model** — §2. Fed the *same* script the simulator executes plus the + accepted-commit observations, it predicts every observable. +4. **Fault injectors** — process kill at scheduled/random points (CT-2), storage-level + throttling and disk-fill (CT-7), connection faults. +5. **Comparator** — at **quiescence** (no pending input and OB-1/OB-2 stable for + `P-QUIESCENCE`), diff SUT observables against the model: STATUS/HEAD values, full-window + scan results, query results over sampled ranges/selections. + +Determinism rules: the harness controls all inputs and timestamps; SUT-side +nondeterminism (batch boundaries, coverage cuts) is absorbed by comparing at quiescence +and by INV-22's determinism-modulo-coverage. + +## 2. Reference model (normative pseudocode) + +One instance per dataset. This is the oracle for every CT class; ~everything else in the +harness is plumbing around it. + +Lookups and slices below are by *block number*, and numbering may be sparse (DEF-1): +`block_at(n)` is the stored block numbered exactly `n` (`⊥` at a hole); `hash_at(p)` is +the chain hash at a position (DEF-16). + +``` +model Dataset: + kind; ret + seg : list = [] # ascending, parent-linked; numbers may be sparse + anchor : (number, hash|⊥) + fin : Ref|⊥ = ⊥ + ver : int = 0 + + first() = seg[0].number if seg else ⊥ + head() = ref(seg[-1]) if seg else ⊥ + next() = seg[-1].number + 1 if seg else anchor.number + 1 # lowest admissible position + + block_at(n) = the b in seg with b.number == n, else ⊥ + hash_at(p) = b.hash for the highest b in seg with b.number <= p, + else anchor.hash if p >= anchor.number else undefined # DEF-16 + + wf(): # well-formedness — assert after every transition (INV-1..6) + assert ascending_and_linked(seg) # INV-1/2: parent_number + parent_hash + assert anchor.number < first() if seg + assert seg[0].parent_hash == anchor.hash if seg and anchor.hash != ⊥ + assert fin == ⊥ or (seg and first() <= fin.number <= head().number + and block_at(fin.number).hash == fin.hash) + + extend(B, f=⊥): # WP §2.2 + require B and valid_run(B) # WP-2: ascending + linked + require B[0].number >= next() + require B[0].parent_hash == (head().hash if seg else anchor.hash) or anchor.hash == ⊥ and not seg + require B[0].parent_number == head().number if seg + seg += B; ver += 1; if f: finalize_inline(f) + wf() + + replace(from_, B, f=⊥): # WP §2.3 + require seg and B and valid_run(B) and B[0].number >= from_ + require fin == ⊥ or from_ > fin.number # INV-13/14 + require from_ >= first() # INV-14 + require B[0].parent_hash == hash_at(from_ - 1) # DEF-16 (⊥ accepted at the window edge) + seg = [b in seg | b.number < from_] + B; ver += 1; if f: finalize_inline(f) + wf() + + finalize(r): # WP §2.4 + if not seg: return IGNORED + e = min(r.number, head().number) + if e < first(): return IGNORED + if fin != ⊥ and e < fin.number: return IGNORED + if e == r.number and block_at(e) == ⊥: return INTEGRITY_FAULT # hole = chain mismatch + if e == r.number and block_at(e).hash != r.hash: return INTEGRITY_FAULT # WP-8 (alarm) + if fin != ⊥ and e == fin.number and block_at(e).hash != fin.hash: return INTEGRITY_FAULT + fin = (e, block_at(e).hash); ver += 1; wf() # block_at(e) exists on every accepting path + + retain(from_, h=⊥): # WP §2.5 + if from_ == (first() if seg else anchor.number + 1) or (not seg and from_ <= anchor.number + 1): + return validate_anchor(h) # no-op / WP-9 policy + if seg and from_ < first(): + return reset((from_ - 1, h)) # downward bound = destructive + # re-bootstrap (§2.5); alarmed (OB-9) + if seg and from_ <= next(): + if h != ⊥ and h != hash_at(from_ - 1): return reset((from_ - 1, h)) # WP-9 self-heal + new_anchor = (from_ - 1, hash_at(from_ - 1)) # INV-18 (DEF-16) + seg = [b in seg | b.number >= from_]; anchor = new_anchor + if fin != ⊥ and fin.number < from_: fin = ⊥ # RS-2 + ver += 1; wf() + else: + reset((from_ - 1, h)) # from_ > next() + + reset(a): # WP §2.6 — must be alarmed/observable (OB-9) + seg = []; anchor = a; fin = ⊥; ver += 1; wf() + + resolve_fork(hints): # WP-6/WP-6b — REPLACE position, RESET, or fault + require hints ascending, non-empty + if fin != ⊥: + hints = [x for x in hints if x.number >= fin.number] + if not hints: return INTEGRITY_FAULT # below finality + if hints[0].number == fin.number and hints[0].hash != fin.hash: return INTEGRITY_FAULT + if anchor.hash != ⊥ and any(x.number == anchor.number and x.hash != anchor.hash for x in hints): + return RESET((anchor.number, hint hash at that position)) # WP-6b: below-window divergence + m = max({x in hints | stored_or_anchor(x)}, default=⊥) + if m != ⊥: return (m.number + 1, m.hash) + if fin != ⊥: return (fin.number + 1, fin.hash) # volatile suffix only; repeated + # rejection here = FM-SRC-5, never RESET + return (first(), anchor.hash) # full-window replacement *probe*; + # repeated WP-2 rejection at first(D) + # escalates to RESET per WP-6b + + query(q): # 04 — evaluated on a copy = snapshot (INV-20) + if q.dialect != kind: return KIND_MISMATCH + hi = fin.number if q.finalized_only else (head().number if seg else ⊥) + if q.to != ⊥ and q.to < q.from: return MALFORMED_REQUEST + if seg and q.from < first(): return RANGE_UNAVAILABLE + T = fin if q.finalized_only else head() # RP-5b tip anchor + if q.expected_parent != ⊥ and T != ⊥ and q.from == T.number + 1 and q.expected_parent != T.hash: + return CONFLICT(hints = chain_suffix_ending_at(q.from - 1, P_CONFLICT_WINDOW)) + # RP-5b: the assertion references the watermark block's exact position — + # evaluable and definite; MUST precede the NO_DATA / long-poll path + if hi == ⊥ or q.from > hi: return NO_DATA # after RP-5 wait + if q.expected_parent != ⊥ and hash_at(q.from - 1) not in {⊥, q.expected_parent}: + return CONFLICT(hints = chain_suffix_ending_at(q.from - 1, P_CONFLICT_WINDOW)) + # hints are ⟨p, hash_at(p)⟩ entries (RP-11, DEF-16) + if selection_unavailable_at(q, q.from): return ITEM_UNAVAILABLE # RP-8: range *start* only + L = coverage_end_chosen_by_SUT(q, hi) # free variable: from <= L <= min(q.to or hi, hi), + # never crossing an availability boundary (RP-8) + emitted = [render(b, q) for b in seg + if q.from <= b.number <= L and (matches(b, q) or q.include_all)] + end = highest b in seg with b.number <= L # RP-9 carrier + if end != ⊥ and end not in emitted: emitted += [header_only(end)] + # the SUT MAY add further header-only boundary markers of covered blocks; + # layout-dependent, exempt from the INV-22 determinism check + return OK(emitted, L, watermarks = (head(), fin)) +``` + +Two **free variables** are granted to the SUT: `L` (coverage end) and the set of extra +header-only boundary markers (RP-9/INV-22). The comparator checks every response for: a +valid `L`; matching blocks and their item content equal to the model evaluated *at that +`L`*; the coverage-end record present; every extra record being a header-only true block +of the snapshot inside coverage. Everything else must match exactly. + +## 3. Test-class taxonomy + +| CT | Name | Method | Primary properties | +|---|---|---|---| +| CT-1 | **Stateful property tests** | randomized scripts of source events + retention ops + reads; model diff continuously and at quiescence | INV-1..7, 10..18, 20..27, 30/31, 44; WP-*; RP-* | +| CT-2 | **Crash-recovery** | kill-point matrix (during batch write, during fork, during trim, during boot, during shutdown) × restart → model diff; repeated-crash convergence | INV-40, 42, 43; CN-6/9/11; LIV-5/6/12; GAP-2 | +| CT-3 | **Concurrency** | reader swarms hammering during write/fork/trim/maintenance storms; interleaved HEAD+QUERY sequencing checks | INV-20/21/23/31/41; CN-3/4; LIV-3/4 | +| CT-4 | **Source-fault corpus** | scripted FM-SRC-1..8 scenarios incl. fork storms, deep forks, finality conflicts, equivocation | INV-12/13/14/23/24; WP-6/8; LIV-9; FM-SRC-*; GAP-3/4/5 | +| CT-5 | **Interface conformance** | exhaustive request/response matrix against the binding: error taxonomy, watermark headers, encodings, boot config matrix | RP-1..16; INV-26/43; IB-*; GAP-8/9/11 | +| CT-6 | **Performance benchmarks** | reference scenarios S1–S6; SLI capture; SLO gates; saturation knees | SLI-1..12; PF-1..9; LIV-1/3/10; GAP-13 | +| CT-7 | **Soak / endurance** | multi-day S4 churn with fault sprinkling; space, memory, stall, residue tracking | LIV-2/7/11; RS-6/10; INV-16/17; HZ-2/5; GAP-1/6 | +| CT-8 | **Isolation / noisy neighbor** | S6: one dataset saturated/faulted, others measured differentially | INV-35/36; LIV-8; PF-4; GAP-14 | +| CT-9 | **Fuzzing** | source-side payload fuzz (write path) + client-side request fuzz (read path); crash/hang/leak oracles + invariant spot checks | FM-1; WP-18; RP-1/2; GAP-11/12 | + +Every test cites the IDs it verifies; CI reports coverage as "properties exercised", not +lines. + +## 4. Kind-agnostic structural validators (client driver) + +Applicable to any response without understanding kind semantics — the cheap 80% of +INV-21/22/23 (checks in parentheses): + +1. decodable stream, one block object per record (IB-4); +2. strictly ascending, unique block numbers (INV-21); +3. header projection includes number/hash/parent-hash when requested → linkage verifiable + across records and across responses (INV-2/23); +4. every item's block number equals its enclosing block's (INV-21); +5. blocks within `[from, min(to, reported head)]` (INV-27); +6. anchored continuation across responses never breaks parent-hash chains (INV-23); +7. watermark coherence: `first ≤ fin ≤ head` whenever reported together (INV-5/30). + +## 5. Traceability matrix (status @ 2026-07-12) + +Legend: **C** covered, **P** partial (some storage-layer or fixture coverage exists; +service-level black-box coverage absent), **U** untested. Rows that changed with Phase 0 name +the test that moved them; unless a row says otherwise, "covered" means *on the happy path* — +the same property under forks, crashes and retention is the business of CT-2/CT-4. + +| Property | CT class | Status | Note | +|---|---|---|---| +| INV-1..3 structural chain | CT-1 | **C** | `ct1_evm` / `ct1_solana` / `ct1_hyperliquid_fills`: full-window scan, parent linkage + anchor, dense and slot-numbered | +| INV-4 kind/schema | CT-1/5 | **C** for evm/solana/hyperliquid-fills | payload round-trips against the emission oracle per kind; bitcoin, tron and hl-replica-cmds unmodeled | +| INV-5/6 watermark bounds/on-chain | CT-1 | **P — known-violated** | happy path C; finality below the window is accepted after a trim clears `fin` (GAP-27); hash below head unverified (GAP-4) | +| INV-7 provenance | CT-1/6 | **C** | `ct1_happy_path`: what the source served is read back, payload included | +| INV-10 atomic transitions | CT-2/3 | U | | +| INV-11 append | CT-1 | **C** | | +| INV-12/13 finality monotone/immutable | CT-1/4 | **P — known-violated** | monotone advance observed; composed-finality REPLACE below `fin` admitted (GAP-22); regression/conflict paths await CT-4 | +| INV-14 fork floor | CT-4 | **U — known-violated** | GAP-3; composed-finality bypass (GAP-22) | +| INV-15/18 retention trim/anchor | CT-1 | **U — known-violated** | INV-18: trims drop the anchor hash (GAP-23), restart rebuilds it wrong (GAP-2); comparator needs RS-4 slack first (see §7) | +| INV-16 frame | CT-1/7 | U | | +| INV-17 maintenance transparency | CT-7 | P | merge-equivalence tested storage-level | +| INV-20 snapshot isolation | CT-3 | P | single-threaded snapshot test only | +| INV-21/22 response shape/completeness | CT-1/5/6 | P | structural validators + emission diff under `include_all`; coverage cuts and filtered emission await CT-5; comparator must implement the RP-9 marker exemption (spec change 2026-07-12) | +| INV-23 anchored ancestry | CT-1/4 | P | anchored continuation across responses covered; the CONFLICT path awaits CT-4 | +| INV-24 finalized-only | CT-4 | U | | +| INV-25 progress | CT-1/6 | P | a successful response must cover ≥ 1 block — asserted by the scanner | +| INV-26 error soundness | CT-5 | **U — known-violated** | GAP-11, GAP-21, GAP-32, GAP-36 | +| INV-27 range honesty | CT-1 | **C** | validator: no block outside `[from, min(to, head)]` | +| INV-30/31 reporting | CT-1/3 | P | INV-30 asserted at quiescence; INV-31 (real-time monotonicity) untested | +| INV-35/36 isolation | CT-8 | U | all existing tests are single-dataset | +| INV-40 recovery | CT-2 | **U — known-suspect** | GAP-2, GAP-28; restart primitives exist (`Sut::restart`), the kill-point matrix does not | +| INV-41 reclamation invisibility | CT-3/7 | P | snapshot-safety + reader-break tests exist storage-level | +| INV-42 residue convergence | CT-2/7 | P | synthetic orphan purge tested; no crash-driven test | +| INV-43 boot validation | CT-5 | U | | +| INV-44 explicit destruction | CT-1/2 | U | | +| LIV-1/2 progress/stall | CT-6/7 | **U — known-violated** | GAP-1 | +| LIV-3 query termination | CT-3/6 | U | | +| LIV-4 waiter termination | CT-1/3 | P | `ct1_happy_path`: a query above the head answers `NO_DATA` within `P-HEAD-WAIT` | +| LIV-5/6 startup/recovery | CT-2/6 | **U — known-violated** | GAP-7; `Sut::last_startup` records SLI-5 per boot | +| LIV-7 reclamation | CT-7 | U | runtime reclaim on by default since 2026-07 (GAP-6 residual: boot-gated residue purge); convergence unmeasured | +| LIV-8 isolation | CT-8 | **U — known-violated** | GAP-1/14 | +| LIV-9 fork convergence/alarm | CT-4 | **U — known-suspect** | GAP-5 | +| LIV-10 overload recovery | CT-6 | U | | +| LIV-11 retention keep-up | CT-7 | U | | +| LIV-12 shutdown | CT-2 | **U — known-suspect** | GAP-17; `Sut::stop` measures the drain | +| RS-3/4 window floor/excess | CT-1/7 | U | | +| RS-6 amplification | CT-7 | U | reclaim path fixed 2026-07 (GAP-6); bound unmeasured under churn | +| RS-8 boot maintenance | CT-2/7 | P | unlink/orphan-purge behaviors have storage-level tests | +| RS-10/11 residue/deletion cost | CT-7 | P | GAP-6/13 | +| FM-1 robustness | CT-9 | **P — known-violated** | GAP-11/12/26 open; the unterminated-record class is closed (§6.1) and pinned by `ct9_source_faults` | +| FM-SRC-* corpus | CT-4 | U | one stale-pack crash-loop already occurred (GAP-5 class); no strike/quarantine substrate (GAP-30) | +| FM-STOR-2/3 disk pressure | CT-7 | U | incident-derived; no automated test | +| FM-OP-1..5 | CT-5 | U | | +| SLI-1..12 / PF-* | CT-6 | U | no benchmark harness exists | +| OB-1 chain gauges | CT-1 | P | `first_block` / `last_block` / `last_finalized_block` diffed against the model; commit version and retention policy not exported (GAP-34) | +| OB-2..11 | all | P | query metrics exist; stall gauges pending on PR #83 (unmerged); OB-2 heartbeat, OB-6 debt accounting, OB-9 alarms, OB-11 forensics absent | + +## 6. Gap register (dated 2026-07-12, informative) + +Known or strongly suspected divergences between this spec and the current system, from +incident history, code-level review, and coverage analysis. Priorities: P0 = active +production risk, P1 = correctness/robustness hole with plausible trigger, P2 = bounded or +rare, P3 = polish. **First test** names the cheapest failing-test-first entry point. + +| GAP | Statement | Violates | Prio | First test | +|---|---|---|---|---| +| GAP-1 | Whole-service ingest freezes (≈6 min) observed post-deploy; all datasets stall simultaneously; root cause unconfirmed (shared write-path backpressure suspected); no stall observability to attribute it | LIV-2, LIV-8, OB-3/11 | **P0** | CT-7 stall harness: S1 + storage-pressure injection, assert SLI-9 ≤ budget; build OB-11 capture first | +| GAP-2 | Recovered anchor hash is reconstructed from the wrong value after restart — `WriteController::new` takes the first batch's *last*-block hash where the correct value sits in its `parent_block_hash` field; latent until the fork-fallback path consumes it, then ingestion resumes with a wrong expected parent (perpetual source-rejection loop) | INV-40, WP-19, LIV-6 | P1 | CT-2: restart, then force full-window fork fallback; assert resume position equals model | +| GAP-3 | Fork floor at the window start is not enforced (marked-as-known in the system); a deeper-than-window divergence may be mishandled instead of becoming an explicit RESET per WP-6b | INV-14, WP-6/6b | P1 | CT-4 deep-fork case: hints strictly below `first(D)` | +| GAP-4 | Finality reports strictly below the head are applied without verifying the hash against the stored block — in both the standalone FINALIZE path and the batch-composed path; the window floor is not checked either (GAP-27) | INV-6, WP §2.4 | P2 | CT-4: finality with corrupted hash below head; assert INTEGRITY_FAULT not acceptance | +| GAP-5 | Unapplicable divergence (fork below finality, finality conflicts) results in silent bounded-pause retry forever — no alarm state, no distinct observable; one such class already caused a crash-loop incident | LIV-9b, FM-SRC-5, OB-9 | P1 | CT-4: fork-below-finality script; assert alarmed state within `P-ALARM` while reads keep serving | +| GAP-6 | ~~Default deployments never reclaim~~ — routine reclaim fixed 2026-07 (PR #79: 10 s point-delete sweep + deletion-collector + periodic-compaction backstop). REMAINING: the interrupted-build residue purge and the whole-file unlink are confined to the gated boot mode (off by default) — a torn build's residue leaks for good in default config and pins the boot-unlink watermark; SLI-8 under churn still unmeasured | RS-10, RS-8 (RS-6 residual) | P2 (was P0) | CT-7: churn soak in default config; assert SLI-8 bound + residue-age bound | +| GAP-7 | Serving is gated on full initialization: tens of seconds of refused connections after deploy, scaling with state size and dataset count; readiness not observable per dataset | LIV-5, OB-8 | P1 | CT-6 S5: SLI-5 vs state-size regression curve | +| GAP-8 | ~~Zero-emission responses do not convey the coverage end~~ — a de-facto carrier existed all along (the coverage-end block is always emitted, header-only when unmatched) and was adopted as normative RP-9 on 2026-07-12. REMAINING: (a) a zero-emission success now *asserts* full-range coverage, but under time-budget truncation over a blockless range (explicit `to` inside a hole run) the implementation can return an empty 200 having covered only part of it — the client then silently skips the rest; (b) the comparator must implement the INV-22 boundary-marker exemption; (c) in one reachable corner RP-9's clauses are jointly unsatisfiable — an effective range whose covered part contains no stored block and whose coverage cannot legally reach the range end (an availability boundary, RP-8, or a hard `P-QUERY-TIME` stop inside a long hole run): zero emission is legal only at full coverage, and the carrier (highest stored block ≤ `L`) lies below `from`, which INV-27 forbids emitting — here the *spec*, not just the implementation, owes an answer (candidate remedies: an explicit in-band terminal coverage record — a wire change — or forbidding coverage to end inside a hole except at the range end) | RP-9, INV-22, RP-8, INV-27 | P2 | CT-5: hole-range query with explicit `to` + tight budget; assert empty-200 only with full coverage | +| GAP-9 | `NO_DATA` responses carry no watermarks (long-poll clients learn nothing about the head from a timeout). Mechanism: both construction sites of the above-head outcome hardcode "no finalized head", so the header-attaching code is unreachable | RP-5 SHOULD, IB-6 | P3 | CT-5 header assertion | +| GAP-10 | Mid-stream admission failures truncate silently and are not counted; truncation rate invisible | RP-15, OB-4 | P2 | CT-6 overload phase: assert truncation counter ≥ observed truncations | +| GAP-11 | Expressible-but-unsupported dialects (`substrate`, `fuel`) parse and validate, then hit an `unimplemented!()` — the connection dies with no response (task panic; not a process exit as originally filed). More broadly, any panic inside plan execution re-panics the handler through the executor's `expect`, bypassing the whole error taxonomy and every counter (OB-5/SLI-10 blind) | RP-2, FM-1, INV-26, OB-5 | P1 | CT-5: one request per unsupported dialect; assert 4xx `UNSUPPORTED_QUERY`, live process, counted error | +| GAP-12 | At least one payload-content class (invalid text encoding in stats-tracked fields) panics the write path; payload fuzz has never been run | FM-1, WP-18 | P1 | CT-9 source-payload fuzz with crash oracle | +| GAP-13 | Ingest batch accumulation has content-dependent unbounded memory (no hard byte ceiling on some structures) | PF-1 | P2 | CT-6 adversarial `W-BLOCK-SIZE`/`W-ITEM-DENSITY`; RSS ceiling assertion | +| GAP-14 | Read-side capacity (execution slots, waiter slots) is a single global pool: one dataset's query herd can starve all datasets | PF-4, LIV-8 | P2 | CT-8: herd on D′, tip-follower SLOs on D | +| GAP-15 | No explicit store-format compatibility gate at boot; incompatibility surfaces as runtime decode errors | CN-12, INV-43 | P3 | CT-5 boot matrix with future-format fixture | +| GAP-16 | ~~The service layer has essentially zero automated tests~~ — **closed by Phase 0**, see §6.1 | all | — | done | +| GAP-17 | Shutdown can take a panic-class exit path in ingestion cancellation (observed at redeploy) | LIV-12, FM-PROC-4 | P2 | CT-2 shutdown class: SIGTERM under load ×100, zero panic exits | +| GAP-18 | Dual-writer detection exists only on some paths (finality/head updates), not all mutations | WP-15, FM-OP-3 | P3 | CT-5: two harness-driven writers, assert loser stops on every mutation type | +| GAP-20 | `parent_number` linkage is never validated on any layer (the block trait exposes it; nothing reads it): a hash-linked run can claim an arbitrarily higher number for the next block, storing a false hole on a densely-numbered chain — a silent data gap served as if it were a slot gap. (The originally-filed non-monotonic-numbers scenario is unreachable today: the source-position advance forces ascending numbers.) | WP-2, DEF-4, INV-1 | P2 | CT-4/CT-9: hash-linked run with a number jump on a dense chain; the run MUST be rejected with no state change | +| GAP-21 | An anchored query whose `from` sits mid-chunk above a number gap larger than the conflict-check lookback (a hard-coded 100 positions in the plan's base-block check; an empty lookback window degenerates to a plain error) fails `INTERNAL` instead of evaluating the assertion. Real trigger: Solana cluster restarts skip thousands of slots inside the window | RP-11, INV-26 | P2 | CT-5: slot-numbered chain with a >100-position hole inside one chunk; anchored query just above the hole must yield OK/CONFLICT, never 500 | +| GAP-22 | Deep-fork handling can silently replace the finalized prefix: the fork-resolution fallback ignores `fin` (resumes from the window start instead of `⟨fin + 1, fin.hash⟩`), the composed-finality guard admits a REPLACE whose base lies at/below `fin` whenever the pack carries a finality mark ≥ current, and Window trims have dropped the anchor hash (GAP-23) so the replacement attaches unchecked. If the first replacement batch reaches past the old `fin`, the finalized prefix is replaced with no RESET event and no alarm — finalized-only clients observe two hashes at one height (INV-24 broken); otherwise the commit trips the fork-floor check ("can't fork safely") and the epoch parks on the blind 60 s retry loop. Fix: fallback → `fin + 1`; enforce the fork floor at commit unconditionally; carry the anchor hash | WP-6, INV-13/14, INV-24, FM-SRC-5, LIV-9 | **P1** | CT-4: fork with all-mismatching hints on a dataset with `fin` defined; assert REPLACE from `fin + 1` (or alarmed fault) — never a commit whose base ≤ `fin` | +| GAP-23 | Window trims drop the anchor hash: the automatic trim passes no hash and the retained state stores `⊥`, though the correct value sits unused in the first batch's `parent_block_hash`. Disables below-window divergence detection (WP-6b has nothing to contradict) and feeds GAP-22 | INV-18, DEF-7, WP-6b | P1 | CT-1: CONFLICT hints / STATUS at the window edge after a trim; CT-4: below-window fork after a trim must RESET, not absorb silently | +| GAP-24 | One dataset's init failure aborts the whole service: startup propagates the first controller error (kind mismatch, retention bail, corrupt state) instead of alarming that dataset and serving the rest | CN-10, FM-OP-1, INV-36, INV-43 | P1 | CT-5 boot matrix: corrupt one dataset's persisted state; assert the others serve and the broken one alarms | +| GAP-25 | Downward retention (`from < first(D)`) executes as an *implicit, unobservable* RESET (WP §2.5 as amended 2026-07-12 legalizes the destruction, but requires OB-9 observability) — no event, indistinguishable from a trim; and the boot-time `Pinned` equivalent aborts the entire service (via GAP-24) instead of a dataset-level refusal | WP §2.5, OB-9, INV-43 | P2 | CT-1: SET-RETENTION below `first`; assert a RESET observable + serving continuity; CT-5: boot with lowered `Pinned.from` | +| GAP-26 | A block timestamp outside the datetime-conversion range kills the ingest flush — the conversion exists only for a log line — and the dataset enters a permanent crash-loop on redelivery | FM-1, WP-18, CN-8 | P1 | CT-9: serve a block with `time = i64::MAX`; the dataset must ingest it and keep serving | +| GAP-27 | FINALIZE never checks `e ≥ first(D)`: with `fin = ⊥` (e.g. after a trim passed above it) a lagging source's report below the window commits `fin < first(D)` | WP §2.4, INV-5 | P2 | CT-4: finality below the window on a trimmed dataset; assert the report is ignored | +| GAP-28 | The External retention instruction and the empty-dataset anchor are memory-only (the persisted label holds kind/version/fin) — a restart forgets the instructed bound (the dataset re-idles awaiting a new instruction) and resets an empty dataset's anchor | CN-9, INV-40, WP-11 | P2 | CT-2: SET-RETENTION, restart; assert the bound and anchor are recovered | +| GAP-29 | A stalled-but-open connection pins the response's storage snapshot indefinitely (no overall response deadline / server write timeout; only disconnects release it); pinned snapshots block physical reclaim of point-deleted data | CN-7, RP-18, HZ-9, RS-6 | P2 | CT-3/CT-7: zombie client that stops reading; assert snapshot lifetime ≤ `P-QUERY-TIME` and reclaim proceeds | +| GAP-30 | No strike counting exists anywhere: a linkage-mismatch rejection resets the source session and re-requests in a zero-backoff hot loop; sources are never quarantined; cross-source equivocation never alarms. `P-SOURCE-STRIKES` has no substrate, so the WP-6b/FM-SRC-5 escalation rules are currently unimplementable | FM-SRC-3/4/6, WP-2 | P2 | CT-4: source serving a permanently mis-linked run; assert bounded request rate, quarantine after N strikes, alarm | +| GAP-31 | A finality advance landing at/above the resume position while no block arrives is deferred (standalone reports are forwarded only when below the position; suppressed until the position is source-confirmed) — the finalized head stalls although the source declared the stored chain final | WP §2.4 clamping | P3 | CT-4: finality = head during a production stall; assert `fin` advances within a bound | +| GAP-32 | Benign race → 500: a finalized query admitted while `fin` was set, with `fin` cleared by a trim/reset before the snapshot, fails `INTERNAL` ("finalized head is not available yet") instead of `NO_DATA` | INV-26, RP-6 | P3 | CT-3: interleave finalized queries with trims that clear `fin` | +| GAP-33 | The live-stream head-number header is computed from the *current* head, not the snapshot's: a head-lowering fork between snapshot and response start yields a header below blocks actually emitted (IB-6 requires ≥ the snapshot head) | IB-6, CN-5 | P3 | CT-3/CT-4: fork lowering the head during streaming; assert header ≥ snapshot head | +| GAP-34 | OB-1 partially unimplemented: the commit version (present in the persisted label) and the retention policy in force are not exported | OB-1 | P3 | CT-1: scrape assertion once exported | +| GAP-35 | The runtime External instruction `"None"` parks the dataset: the controller maps it to Idle and stops ingestion even on a non-empty dataset (`RetentionStrategy::None → State::Idle`, dataset_controller.rs), while the binding documents `"None"` as Unbounded (13 §6) and WP-5 requires a non-empty dataset to keep ingesting from its window. Whether an External instruction may change the policy *mode* at all is unspecified (WP-11) | WP-5, DEF-9, WP-11, IB §6 | P2 | CT-1/CT-5: SET-RETENTION `"None"` on a non-empty External dataset; assert ingestion continues (or the instruction is refused with a defined error) — the head must keep advancing | +| GAP-36 | `MALFORMED_REQUEST`, `RANGE_UNAVAILABLE`, `ITEM_UNAVAILABLE` and `KIND_MISMATCH` all surface as HTTP 400 with a free-text body (`api.rs error_to_response`); no machine-readable discriminant exists and IB-7 forbids keying on text — clients cannot distinguish "re-anchor upward" from "fix the request", and the CT-5 error matrix cannot verify INV-26 at the binding. 13 §5 marks the discrimination REQUIRED; the structured error body is the missing piece | INV-26, IB-7, 04 §8 | P2 | CT-5: trigger each 400 class; assert a structured field distinguishes them | +| GAP-37 | PF-1's memory ceiling is not configuration-derivable on the read side: INV-25/RP-17 require emitting the first covered block whole even above `P-RESP-WEIGHT`, and nothing bounds a single block at ingest (`P-BATCH-BYTES` is a soft *batch* bound — one oversized block still stores), so per-response memory is bounded only by the largest block a source ever served. No `P-MAX-BLOCK-BYTES` exists | PF-1, RP-17/INV-25, FM-CLI-2 | P2 | CT-6/CT-9: ingest a pathological giant block, query it; assert bounded RSS and whole-block emission (INV-25) | + +### 6.1 Closed + +| GAP | Statement | Closed by | +|---|---|---| +| GAP-16 | No service-level automated tests | [`crates/hotblocks-harness`](../../hotblocks-harness) + `ct1_happy_path` (Phase 0, 2026-07-12) | +| GAP-19 | A source response whose final JSONL record carried no trailing newline panicked the line reader (`LineStream::take_final_line` left its scan position past the emptied buffer). The ingest task died, its buffered batch was lost, and the dataset parked for `P-EPOCH-RETRY` — then crash-looped, since the source served the same body on retry. Violated FM-1, LIV-2 | Found by CT-1 on the harness's first run; fixed in `crates/data-client/src/reqwest/lines.rs`; pinned by a unit test there and by `ct9_source_faults` (2026-07-12) | + +## 7. Build order (recommended) + +- **Phase 0 — skeleton (unblocks everything). ✅ done 2026-07-12.** Source simulator (happy + path), client driver + §4 validators, reference model, quiescence comparator. Exit: CT-1 runs + a happy-path script green end-to-end. (GAP-16) + + Delivered as [`crates/hotblocks-harness`](../../hotblocks-harness), driving the real + binary as a child process over the binding of 13. Its README records the design decisions the + next phases must not undo. Three things the later phases need are built but unexercised, and + three are missing: + + | Built, awaiting scripts | Missing | + |---|---| + | `Sut::crash/stop/restart` (same db, same port) → CT-2 | the CT-2 kill-point matrix | + | `Harness::fork` + `Model::resolve_fork` + the follower's CONFLICT recovery → CT-4 | the CT-4 fork/finality corpus | + | `Model::predict_query` (the full outcome class) → CT-5 | the CT-5 request matrix | + | `SimFaults` injection point → CT-9 | the rest of the FM-SRC repertoire | + + One correctness note for whoever writes the retention tests: the comparator compares the + window's first block *exactly*. That is right today (no test trims) and wrong the moment one + does — the service trims whole chunks, so its window may legally be larger than the model's + (RS-3/RS-4, `P-RETENTION-SLACK`). Give the comparator that tolerance before, not after. + +- **Phase 1 — the P0s.** Stall harness + OB-2/3/11 signals (GAP-1); churn soak + space + accounting via OB-6 (GAP-6). These target the two production incidents. +- **Phase 2 — correctness core.** CT-2 crash matrix (GAP-2), CT-4 fork/finality corpus + (GAP-3/4/5), CT-5 error taxonomy + boot matrix (GAP-8/9/11/15). +- **Phase 3 — robustness.** CT-9 fuzz both surfaces (GAP-12), CT-3 concurrency swarms, + CT-8 isolation (GAP-14), shutdown class (GAP-17). +- **Phase 4 — performance regime.** CT-6 scenarios S1–S6, SLO gates, saturation knees, + baselines (GAP-7/13); wire into CI with tolerances. + +Each phase ends by updating §5 statuses and re-prioritizing §6. diff --git a/crates/hotblocks/spec/13-interface-binding.md b/crates/hotblocks/spec/13-interface-binding.md new file mode 100644 index 00000000..9a1314be --- /dev/null +++ b/crates/hotblocks/spec/13-interface-binding.md @@ -0,0 +1,134 @@ +# 13 — Interface binding (HTTP) + +This document binds the abstract operations of 03/04 to the concrete wire protocol so a +black-box conformance harness can be built. It describes the **external contract** of the +current service generation — endpoints, encodings, status mapping — not internal +implementation. Where the binding today falls short of a spec requirement, the row is +annotated with the relevant GAP. + +## 1. General + +- **IB-1** Transport: HTTP/1.1+; all request/response bodies are JSON unless noted. + Dataset-scoped paths use `/datasets/{id}` where `{id}` is an identifier of bounded + length (≤ 48 chars, `[A-Za-z0-9_-]`); syntactically invalid ids are client errors, + unknown ids map to `UNKNOWN_DATASET` (404). +- **IB-2** Request bodies are plain (uncompressed) JSON, size-limited by `P-BODY-LIMIT`; + responses to query operations are compressed streams — `zstd` if the client's + `Accept-Encoding` includes it, otherwise `gzip` (default even without the header), with + `Content-Encoding` and `Vary: Accept-Encoding` set. +- **IB-3** Correlation: responses carry a request-id header; clients MAY send a client + identity header used for bounded-cardinality attribution (OB-10). + +## 2. Operations → routes + +| Abstract op | Route | Notes | +|---|---|---| +| QUERY | `POST /datasets/{id}/stream` | body = DEF-13 query (dialect-tagged JSON) | +| QUERY-FINALIZED | `POST /datasets/{id}/finalized-stream` | same body; `finalized_only` semantics (RP-6) | +| HEAD | `GET /datasets/{id}/head` | `{"number":N,"hash":"…"}` or `null` | +| FINALIZED-HEAD | `GET /datasets/{id}/finalized-head` | same shape | +| STATUS | `GET /datasets/{id}/status` | kind, retention, first/last block (+hash/time), finalized head | +| METADATA | `GET /datasets/{id}/metadata` | start block, real-time flag, aliases | +| GET-RETENTION | `GET /datasets/{id}/retention` | current policy JSON | +| SET-RETENTION | `POST /datasets/{id}/retention` | policy JSON; only for `External` datasets, else `FORBIDDEN` (403) | +| observability | `GET /metrics` (+ engine-diagnostic routes) | OB surface, text formats | + +Dialects accepted in query bodies: `evm`, `solana`, `bitcoin`, `tron`, +`hyperliquidFills`, `hyperliquidReplicaCmds` — and, expressible in the query schema but +**not served** by this system generation, `substrate`, `fuel` (MUST map to +`UNSUPPORTED_QUERY`; see GAP-11). + +## 3. Query request (DEF-13 binding) + +```jsonc +{ + "type": "evm", // dialect tag = dataset kind + "fromBlock": 123, // required, inclusive + "toBlock": 456, // optional, inclusive + "parentBlockHash": "0x…", // optional = expected_parent: hash of fromBlock's preceding block (DEF-13/16) + "includeAllBlocks": false, // optional + "fields": { … }, // per-table field projections + // item requests (dialect-specific arrays), each ≤ selector rules, total ≤ P-MAX-ITEM-REQ: + "transactions": [ {…} ], "logs": [ {…} ], … +} +``` + +`fromBlock ≥ 0`. For `fromBlock = 0` there is no preceding position on the wire: +`parentBlockHash` MUST be omitted (the genesis anchor `⟨−1, ⊥⟩` carries no hash — DEF-7), +and CONFLICT hints can never reference position `−1` (RP-11's "nearest stored position +below" never applies below genesis). + +## 4. Query response stream + +- **IB-4** Success body: one compressed stream whose decompressed content is + **JSON Lines** — exactly one JSON object per emitted block, in ascending order: + `{"header":{…}, "":[…], …}`. This is the emission contract of 04 §5; the + §4-of-12 validators parse it generically. The `header` object always carries the block + `number` — key fields are emitted regardless of the field projection — so the RP-9 + coverage carrier is recoverable under any selection; `hash`/`parentHash` appear when + projected, and anchored continuation (RP-10) therefore requires projecting them. +- **IB-5** The success status is committed before streaming begins; admission-time errors + therefore arrive as proper error statuses, while post-admission failures surface as + RP-15 truncation (stream ends early; already-sent prefix remains valid JSONL). See + GAP-10 for the observability obligation. +- **IB-6** Response watermark headers (success): finalized head number+hash when defined, + and a head-number header (for the live stream: at least the snapshot head; may be + fresher per CN-5). `NO_DATA` (204) SHOULD carry the same watermark headers — currently + it does not (GAP-9). Coverage reporting per RP-9 is carried by emission itself: the + coverage-end block is always present in the JSONL stream, header-only when it matches + nothing (RP-9 carrier; boundary markers may also appear). Residual: effective ranges + containing no stored block at all (GAP-8). + +## 5. Status mapping + +| Error class (04 §8) | HTTP | +|---|---| +| success | 200 (streamed) | +| `NO_DATA` | 204, empty body | +| `MALFORMED_REQUEST` | 400 (syntactic/semantic request errors, range-below-window message today — see next rows) | +| `RANGE_UNAVAILABLE` | 400 today; distinct machine-readable discrimination from other 400s is REQUIRED for conformance clients (SHOULD: structured error body) — GAP-36 | +| `ITEM_UNAVAILABLE` | 400 (same note — GAP-36) | +| `KIND_MISMATCH` | 400 (same note — GAP-36) | +| `UNSUPPORTED_QUERY` | REQUIRED: 4xx without process damage — currently violated (GAP-11) | +| schema-invalid body | 422 | +| `FORBIDDEN` | 403 | +| `UNKNOWN_DATASET` | 404 | +| `CONFLICT` | 409, body `{"previousBlocks":[{"number":…,"hash":"…"},…]}` = RP-11 hints (ascending; ≥ 1 entry; up to ~`P-CONFLICT-WINDOW`; entries are `⟨position, hash_at(position)⟩` pairs — DEF-16) | +| `OVERLOADED` | 503 | +| `INTERNAL` | 500 | + +- **IB-7** Error bodies MUST NOT leak internals in a way clients must parse; conformance + tests key on status + the structured fields named above only. + +## 6. Retention policy JSON + +`{"FromBlock":{"number":N,"parent_hash":"…"?}}` | `{"Head":N}` | `"None"` — mapping to +DEF-9: `FromBlock` = Pinned / External instruction payload, `Head` = Window, `None` = +Unbounded. (Config-level `Api` marks a dataset as External-mode.) A runtime `"None"` +instruction today *parks* the dataset — ingestion stops — instead of behaving as +Unbounded (GAP-35). + +## 7. Source-side binding (for the simulator) + +The service consumes sources over the same protocol family it serves; the harness's +source simulator implements: + +- `POST /stream` with `{"fromBlock":N, "parentBlockHash":"…"?}` → `200` JSONL block + stream (with finalized-head headers), `204` no-data, or `409` + `previousBlocks` (the + ForkSignal binding — DEF-12). `parentBlockHash` is the hash of `fromBlock`'s preceding + block — the parent of the first block to be served (DEF-16); same semantics as the + client-side query; +- `GET /head`, `GET /finalized-head` for probing; +- finality is conveyed via response headers (`finalized head number/hash`) alongside + block streams. + +Faults for CT-4/CT-9 are injected at this surface: malformed JSONL, broken linkage, +regressive watermarks, conflicting 409 hints, stalls, disconnects. + +## 8. Conformance notes + +- **IB-8** The binding is versioned by this document; any route/shape/status change MUST + update this file and the CT-5 matrix in the same change. +- **IB-9** Anything observable at this surface and not specified here or in 04 is + *unspecified behavior* — conformance tests MUST NOT pin it, and clients MUST NOT rely + on it. diff --git a/crates/hotblocks/spec/14-parameters.md b/crates/hotblocks/spec/14-parameters.md new file mode 100644 index 00000000..447a9116 --- /dev/null +++ b/crates/hotblocks/spec/14-parameters.md @@ -0,0 +1,72 @@ +# 14 — Parameter registry + +Normative documents reference parameters symbolically; this registry records their role +and current values. **"Observed"** = the value the current system generation exhibits +(informative, may drift); **"Target"** = a value this spec requires or proposes where no +setting exists yet (⚠ = to be ratified). A conformance run records the parameter set it +ran against. + +## Read path + +| Parameter | Role (where used) | Observed | Target | +|---|---|---|---| +| `P-QUERY-TIME` | wall-time budget per query response (RP-17, LIV-3, CN-7) | 10 s | keep | +| `P-HEAD-WAIT` | bounded long-poll wait above the head (RP-5, LIV-4) | 5 s | keep | +| `P-RESP-WEIGHT` | per-response emission weight budget (RP-17, INV-25) | 20 MB (weighted) | keep | +| `P-RESP-FLUSH` | streaming buffer flush threshold (RP-17) | 512 KB | keep | +| `P-CONFLICT-WINDOW` | max hints in a CONFLICT payload; also the anchored-check lookback in *positions* (RP-11) | ~100 (varies by detection site: 1…~101); a number gap deeper than the lookback yields INTERNAL (GAP-21) | ≥ 100 uniformly ⚠ | +| `P-MAX-ITEM-REQ` | max item requests per query (RP-1) | 100 | keep | +| `P-BODY-LIMIT` | max request body size (RP-17, IB-2) | ~2 MB (platform default) | make explicit ⚠ | +| `P-EXEC-SLOTS` | global concurrent query work units (RP-3, PF-3) | executor threads × 200 | keep; revisit per-dataset fairness (GAP-14) | +| `P-WAITERS` | global cap on head-waiting queries (RP-5) | 64 000 | keep; same fairness note | +| `P-SCHED-SLACK` | scheduling tolerance added to termination bounds (LIV-3/4) | — | 1 s ⚠ | + +## Write path + +| Parameter | Role | Observed | Target | +|---|---|---|---| +| `P-BATCH-ROWS` | batch flush bound, rows (WP-3, HZ-6) | 200 000 rows | keep | +| `P-BATCH-BYTES` | batch flush bound, bytes (WP-3, PF-1) | ~30 MB (soft) | hard ceiling ⚠ (GAP-13) | +| `P-MAX-BLOCK-BYTES` | ceiling on one block's encoded size, enforced at ingest (WP-2 rejection + source fault); what makes the read-side "+ one block" allowance (RP-17, INV-25) and PF-1's ceiling finite | absent — a single oversized block stores (the batch bound is soft) and must later be emitted whole (GAP-37) | define ⚠ | +| `P-FORK-CONSENSUS` | arbitration timeout before accepting a fork signal (WP-4) | 2 s | keep | +| `P-SOURCE-BACKOFF` | per-source retry backoff schedule (WP-17, FM-SRC-1) | 0→10 s exponential steps | keep | +| `P-EPOCH-RETRY` | pause before restarting a failed ingestion epoch (WP-17) | 60 s | keep; add alarm coupling (GAP-5) | +| `P-SOURCE-STRIKES` | consecutive rejected runs before quarantining a source (FM-SRC-4); also the escalation threshold for unresolvable divergence (WP-6 fallback → FM-SRC-5, WP-6b → RESET) | absent — no rejection counting exists (GAP-30) | define ⚠ | +| `P-SOURCE-DOWN-ALARM` | continuous all-source unavailability before alarm (FM-SRC-1) | — | 5 min ⚠ | +| `P-PROBE-WAIT` | initial tip-probe quorum wait (WP-5) | 5 s | keep | + +## Retention and space + +| Parameter | Role | Observed | Target | +|---|---|---|---| +| `P-RETENTION-SLACK` | allowed window excess beyond `k` (RS-4, WP-10) | one *merged* storage batch (compaction merges old batches up to 200 k rows — the effective trim granularity) | keep, document per deployment | +| `P-RETENTION-APPLY` | External instruction → committed trim (WP-11, LIV-11) | prompt (unbounded formally) | ≤ 60 s ⚠ | +| `P-CLEANUP-PERIOD` | deferred logical-deletion sweep cadence (RS-5) | 10 s | keep | +| `P-CLEANUP-BACKOFF` | sweep retry after failure | 30 s | keep | +| `P-SPACE-AMP` | steady-state disk/live amplification bound (RS-6, SLI-8) | bounded since 2026-07 (PR #79: point-delete sweep + compaction); unmeasured (CT-7, GAP-6) | ≤ 2.0× ⚠ | +| `P-SPACE-CONST` | fixed overhead allowance (RS-6) | — | size per deployment ⚠ | +| `P-RECLAIM-LAG` | logical delete → physical space convergence (LIV-7) | sweep ≤ 10 s + compaction (typically minutes–hours); ≤ 7 d worst case via periodic compaction; interrupted-build residue: ∞ in default config (GAP-6) | ≤ 24 h ⚠ | +| `P-DISK-FLOOR` | free-disk alarm/degrade threshold (FM-STOR-2) | — | define ⚠ | + +## Liveness, durability, lifecycle + +| Parameter | Role | Observed | Target | +|---|---|---|---| +| `P-STALL-BUDGET` | max zero-progress interval under healthy conditions (LIV-2, SLI-9) | violated: 351–458 s freezes observed | ≤ 5 s ⚠ (GAP-1) | +| `P-LAG-STEADY` | steady-state ingest lag bound (LIV-1, SLI-1) | — | ≤ 2 s + batch quantum ⚠ | +| `P-CATCHUP-RATE` | minimum backlog drain rate (LIV-1) | — | per deployment ⚠ | +| `P-FORK-CONVERGE` | fork signal → REPLACE committed (LIV-9a, SLI-12) | — | ≤ 2 s + one batch ⚠ | +| `P-ALARM` | integrity fault → observable alarm (WP-17, LIV-9b, OB-9) | ∞ (no alarm states exist: GAP-5) | ≤ 10 s ⚠ | +| `P-STARTUP-ACCEPT` | process start → accepting connections (LIV-5a, SLI-5) | ~35 s observed (GAP-7) | ≤ 3 s ⚠ | +| `P-STARTUP-READY(state)` | per-dataset readable bound (LIV-5b, SLI-6) | — | budget curve vs state size ⚠ | +| `P-SHUTDOWN` | drain-and-exit bound (LIV-12) | — | ≤ 30 s ⚠ | +| `P-DUR-PROCESS` | commits lost on process crash (CN-6) | 0 | 0 | +| `P-DUR-SYSTEM` | commit-suffix loss window on host/power failure (CN-6b) | bounded, engine-managed (not explicitly configured) | make explicit ⚠ | +| `P-QUIESCENCE` | harness settling period before model comparison (12 §1) | — | 2× `P-CLEANUP-PERIOD` ⚠ | +| `P-RECOVERY-SETTLE` | post-overload return-to-normal bound (LIV-10) | — | ≤ 30 s ⚠ | + +## Encoding + +| Parameter | Role | Observed | Target | +|---|---|---|---| +| `P-ENCODINGS` | supported response codecs (IB-2) | gzip (default, fast level), zstd (level 1) | keep | diff --git a/crates/hotblocks/spec/README.md b/crates/hotblocks/spec/README.md new file mode 100644 index 00000000..c4d974b2 --- /dev/null +++ b/crates/hotblocks/spec/README.md @@ -0,0 +1,73 @@ +# Hotblocks — Behavioral Specification + +This folder contains the abstract behavioral specification of **Hotblocks**: a real-time +block database that maintains a bounded, fork-aware window of one or more blockchains and +serves range queries over it. + +The specification is **implementation-free**. It describes *what* the system must do — its +state model, operations, invariants, liveness properties, failure behavior, and performance +obligations — never *how* any particular implementation does it. It is written so that: + +1. A conformance / TDD framework can be built against it (black-box, via the interface + binding), using the reference model in [12-conformance-tdd.md](12-conformance-tdd.md) + as the oracle. +2. Gaps between intended and actual behavior can be identified, prioritized, and closed. +3. Performance and robustness regressions can be detected against explicit, measurable + properties instead of anecdotes. + +## Document map + +| Doc | Contents | Normative? | +|---|---|---| +| [01-overview.md](01-overview.md) | Purpose, actors, trust model, design goals and non-goals | Yes | +| [02-data-model.md](02-data-model.md) | Definitions: blocks, segments, watermarks, datasets, snapshots, queries | Yes | +| [03-write-path.md](03-write-path.md) | Ingestion, validation, transitions (extend / replace / finalize / retain / reset) | Yes | +| [04-read-path.md](04-read-path.md) | Query contract, coverage, continuation, conflict protocol, error taxonomy | Yes | +| [05-consistency-and-durability.md](05-consistency-and-durability.md) | Atomicity, isolation, visibility, durability, recovery | Yes | +| [06-invariants.md](06-invariants.md) | **The invariant catalog** (safety) — INV-* | Yes | +| [07-liveness.md](07-liveness.md) | Progress properties — LIV-* | Yes | +| [08-failure-model.md](08-failure-model.md) | Fault taxonomy and required responses — FM-* | Yes | +| [09-retention-and-space.md](09-retention-and-space.md) | Retention policies, logical vs physical space, reclamation — RS-* | Yes | +| [10-performance.md](10-performance.md) | Workload model, SLIs/SLOs, resource bounds, hazards — PF-*, SLI-*, HZ-* | Yes (SLO targets provisional) | +| [11-observability.md](11-observability.md) | Required observables so properties are testable/operable — OB-* | Yes | +| [12-conformance-tdd.md](12-conformance-tdd.md) | Reference model, test taxonomy, traceability matrix, **gap register** | Matrix/gaps informative, dated | +| [13-interface-binding.md](13-interface-binding.md) | Mapping of abstract operations to the wire protocol (HTTP) | Yes, for conformance testing | +| [14-parameters.md](14-parameters.md) | Registry of all symbolic parameters `P-*` and their current values | Values informative | + +Dated **audit records** (`audit-YYYY-MM-DD.md`, e.g. [audit-2026-07-12.md](audit-2026-07-12.md)) +capture spec↔implementation reviews: findings with code references, per-GAP verdicts, and the +spec amendments they motivated. They are informative and frozen at their date; living statuses +stay in 12 and 14. + +## Conventions + +- **RFC 2119 keywords.** MUST / MUST NOT / SHOULD / SHOULD NOT / MAY carry their standard + normative meanings. +- **Identifiers** are stable and referenced across documents: + - `DEF-n` — definitions (02). + - `WP-n`, `RP-n`, `CN-n`, `RS-n`, `FM-n`, `PF-n`, `OB-n`, `IB-n` — requirements in the + write-path, read-path, consistency, retention/space, failure-model, performance, + observability, and interface-binding documents. + - `INV-n` — safety invariants (06). `LIV-n` — liveness properties (07). + - `SLI-n` — service-level indicators, `HZ-n` — performance hazards (10). + - `CT-n` — conformance test classes, `GAP-n` — known/suspected gaps (12). + - `P-NAME` — symbolic parameters. Normative text uses only the symbol; concrete values + live in [14-parameters.md](14-parameters.md). +- **Numbering is banded** (gaps in numbering are intentional) so new items can be added to + a category without renumbering. +- **Math notation.** Block numbers are natural numbers; hashes are opaque non-empty + strings compared by exact equality; `⊥` denotes "absent/undefined"; sequences are + written `⟨b_1 … b_n⟩`. +- **Two documents are expected to change often**: 12 (statuses, gap register) and + 14 (parameter values). All others change only when intended behavior changes, and any + such change MUST be reflected in the traceability matrix of 12. + +## How to use this spec for TDD + +1. Build the harness described in 12 against the binding in 13: a scripted **source + simulator** on one side, a **client driver** on the other, the system under test in the + middle as a black box. +2. Implement the reference model (pseudocode in 12) as the oracle. +3. Work through the gap register (12 §6) in priority order; every test cites the `INV` / + `LIV` / `RS` / `FM` / `PF` identifiers it verifies, keeping the traceability matrix + honest. diff --git a/crates/hotblocks/spec/audit-2026-07-12.md b/crates/hotblocks/spec/audit-2026-07-12.md new file mode 100644 index 00000000..90953370 --- /dev/null +++ b/crates/hotblocks/spec/audit-2026-07-12.md @@ -0,0 +1,157 @@ +# Spec ↔ implementation audit — 2026-07-12 + +> **Applied 2026-07-12:** all §4 spec-side fixes (S1–S5) are in the spec; the gap +> register is rewritten (GAP-2/4/6/8/9/11/20/21/22 updated, GAP-23…34 added, GAP-6→P2, +> GAP-22→P1), the traceability matrix and parameter registry corrected. N3 was resolved +> by *legalizing* downward retention as an explicit alarmed RESET (WP §2.5 amended; +> residual impl deltas = GAP-25). N11 resolved in the impl's favor (RP-5b added). +> Implementation bugs stay in the register as the backlog; nothing in the code was +> changed by this audit. + +Full re-read of `crates/hotblocks/spec/` (all 15 docs) against the implementation: +`crates/hotblocks`, `crates/data-source`, `crates/data-client`, `crates/storage`, +`crates/query` (targeted), `crates/primitives`, `crates/data` (targeted). +Branch: `feat/hotblocks-harness`. + +Three sections: **N-*** new findings (not in the gap register), **G-*** corrections to +the existing GAP-1..22 register, **S-*** places where the spec itself is wrong or silent +and should change. Severity uses the register's scale (P0 active risk … P3 polish). + +--- + +## 1. New findings + +### N1 (P1) — Deep fork on a `Window` dataset can silently replace the finalized prefix +**Verdict (2026-07-12): implementation bug, all three links.** The spec is unambiguous on each element (WP-6 fallback = `fin+1`; INV-18 forbids dropping the anchor hash; INV-13/FM-SRC-5 forbid applying a replace below `fin`). The sanctioned outcome for a genuine deeper divergence already exists: explicit, alarmed RESET (WP-6b). Fix in code: enforce the fork floor in `new_chunk` regardless of composed finality, resume the WP-6 fallback from `fin+1`, carry the anchor hash over on trims (N5). +**Violates:** INV-13, INV-24, FM-SRC-5, WP-6b (RESET must be explicit + alarmed). +**Where:** `crates/hotblocks/src/dataset_controller/write_controller.rs:349-357` (`new_chunk` finality-composition guard), reachable via `compute_rollback` fallback (`write_controller.rs:139-142`, = GAP-22) and the dropped anchor hash (N5). +**Chain:** fork signal whose hints all mismatch → GAP-22 fallback resumes from the window start with `parent_block_hash` = `None` (after any Window trim, N5) → source serves its own chain from the window start; its blocks are marked `is_final` (stream-level finalized head ≥ them) → the composed commit hits the guard arm `(Some(new), Some(current)) if new.number >= current.number => Some(new)`, which does **not** check `chunk.first_block() > current.number` → `insert_fork` deletes **all** chunks, including the finalized prefix. No RESET event, no alarm. +**Outcome depends on batch vs window size:** if the first replacement batch reaches past the old `fin`, the replace commits silently (finalized-only pollers observe two hashes at one height — INV-24 broken); if it flushes below `fin`, the commit bails `"can't fork safely"` and the epoch parks on the 60 s retry loop (GAP-22's documented outcome). Neither is the spec'd behavior. +**First test:** CT-4: dataset with `fin` defined, fork signal with all-mismatching hints, replacement chain carrying fresh finality; assert either REPLACE from `fin+1` or RESET+alarm — never a silent full replace. + +### N2 (P1) — One dataset's init failure prevents the whole service from starting +**Violates:** CN-10, FM-OP-1, FM-STOR-4, INV-36 (INV-43 permits *startup failure*, but CN-10/FM-OP-1 explicitly require unaffected datasets to serve). +**Where:** `crates/hotblocks/src/data_service.rs:79` — `controllers.try_next().await?` propagates any single `DatasetController::new` error (kind mismatch, retention gap bail, corrupt label/chunk decode) and aborts `DataService::start`; `main.rs` then exits. +**First test:** CT-5 boot matrix: corrupt one dataset's label; assert the other datasets serve and the broken one is alarmed. + +### N3 (P1) — `RETAIN` below the window start destroys the entire dataset +**Verdict (2026-07-12): the destruction is legalized in the spec** — the impl comment says "by design", and for a hot store drop-and-reingest is the only way to honor a lowered bound (no downward backfill exists). WP §2.5 now specifies `from < first(D)` as an explicit `RESET(⟨from−1, h?⟩)` in self-healing modes, observable per OB-9, with boot-time `Pinned` refusing instead (symmetric with WP-9). Residual implementation deltas → GAP-25: the RESET is currently *unobservable*, and the boot refusal takes down the whole service (GAP-24). +**Violated before the amendment:** WP §2.5 case 1 (`from ≤ first(D)` → no-op), INV-44 (undocumented destructive path), DEF-9 ("history cannot be re-acquired through RETAIN"). +**Where:** `write_controller.rs:170-183` (`_retain` Gap branch, `delete_mismatch=true` deletes every chunk above `from`); intent confirmed by the comment in `dataset_controller.rs:465` ("FromBlock is less than current front, dropping everything by design"). +**Trigger:** runtime `SET-RETENTION`/retention change with `number < first(D)` — every chunk is deleted, `fin` cleared, dataset re-ingests from `number`. At boot the same condition takes the `delete_mismatch=false` path and **bails**, which via N2 takes down the whole service. +The behavior may actually be *desired* (re-acquire deeper history for External datasets) — but then it is a RESET, must be specified as such, alarmed (OB-9), and WP §2.5 amended. Today it is silent data destruction disagreeing with the spec either way. + +### N4 (P2) — FINALIZE accepts marks below the window; no `e ≥ first(D)` check +**Violates:** WP §2.4 ("a report below the window is ignored"), INV-5. +**Where:** `write_controller.rs:286-338` — `finalize()` checks monotonicity vs current `fin` and clamps to the head, but never compares against the window start. Reachable: a Window dataset trims past `fin` (`fin` becomes `⊥`), then a lagging source (finality lag > window — inside the spec's own `W-FINALITY-LAG` envelope) reports finality below `first(D)` → committed `fin < first(D)`; STATUS exposes the violation. +Same hole in the composed path (`new_chunk`): a pack's `finalized_head` is only clamped from above (to the chunk head), never checked against the window floor, and its hash is never verified against stored blocks below the head (that part = GAP-4). + +### N5 (P2) — Anchor hash is dropped on every Window trim +**Violates:** INV-18 ("preserved from data, never recomputed or dropped"), DEF-7; feeds N1 and neuters WP-6b. +**Where:** `dataset_controller.rs:171` passes `None` (`self.retain(self.write.next_block() - n, None)`); `_retain` then stores `self.parent_block_hash = None` (`write_controller.rs:264`). The correct value exists in storage (`first_chunk.parent_block_hash()`) and is simply not used. +Consequence: after any trim, the full-window fork fallback resumes with no expected parent, so a below-window divergence is absorbed silently instead of tripping linkage rejection → WP-6b RESET. (After restart the same field is reconstructed *wrong* rather than `None` — that's GAP-2, `write_controller.rs:40`: uses `first_chunk.last_block_hash()`; fix is the same one line.) + +### N6 (P2) — External retention bound and empty-dataset anchor are not durable +**Violates:** CN-9 ("recovered state … in every field — segment, anchor (number *and* hash), fin, **retention**, kind"), INV-40, RS-1/WP-11. +**Where:** `DatasetLabel::V0` persists only `{kind, version, finalized_head}` (`crates/storage/src/db/data.rs:17-24`); the retention floor of an **empty** dataset lives only in `WriteController` memory (`first_block`/`parent_block_hash`), and the last `SET-RETENTION` instruction lives only in the watch channel (`dataset_controller.rs`). +Consequence: restart of an Api dataset reverts it to `RetentionStrategy::None`; if it is empty it parks in `State::Idle` until the controller re-instructs; an empty dataset's anchor resets to `(0, ⊥)`. + +### N7 (P2) — A stalled-but-connected client pins a RocksDB snapshot indefinitely +**Violates:** CN-7, RP-18, HZ-9. +**Where:** `QueryResponse` owns a `StaticSnapshot` (`crates/hotblocks/src/query/static_snapshot.rs`) for its whole life; `next_data_pack`'s 10 s budget only runs when the body stream is polled. Disconnects are handled (drop), but a zombie peer that stops reading without closing keeps the response — and the RocksDB snapshot — alive for as long as TCP allows (hours). Pinned snapshots block compaction from reclaiming point-deleted data → unbounded `debt_bytes` under GAP-6's *fixed* runtime path. +**Fix shape:** overall response deadline / server write timeout; drop the snapshot when the time budget expires even if bytes are unsent. + +### N8 (P1) — Ingest task dies on an out-of-range block timestamp (permanent per-dataset wedge) +**Violates:** FM-1, WP-18, CN-8 (clock-free correctness), DEF-4 (`time` is informational). +**Where:** `crates/hotblocks/src/dataset_controller/ingest_generic.rs:230-231` — `DateTime::from_timestamp_millis(...).ok_or(...)?` is computed **only for a log line**, but its error kills `flush()` → ingest task dies → 60 s epoch loop re-fetches the same block → crash-loop (same class as closed GAP-19). +**Fix shape:** log the raw i64 (or `unwrap_or` a sentinel); never fail the flush for formatting. + +### N9 (P2) — Linkage-mismatch rejection loops hot: no backoff, no strikes, no quarantine +**Violates:** FM-SRC-3/4 (quarantine after `P-SOURCE-STRIKES`), WP-2 ("offending source penalized"), WP-6/6b escalation (which is *specified in terms of* strike counting), FM-SRC-6 (equivocation alarm). +**Where:** `crates/data-source/src/standard.rs:104-109` — `accept_new_block == false` resets the endpoint to `Ready`, which immediately re-issues the same request (no `on_error`, so no backoff); nothing counts consecutive rejections anywhere in the codebase. `P-SOURCE-STRIKES` is not "implicit" (registry) — it is absent, and with it the WP-6b/FM-SRC-5 escalation logic has no substrate. + +### N10 (P2) — Finality advance is deferred while the chain is stalled +**Violates:** WP §2.4 clamping semantics ("a finality report above the head is clamped to the head" — i.e. applied *now*), LIV-freshness for finalized-stream clients. +**Where:** `ingest_generic.rs:133-137` forwards a standalone `FinalizedHead` only when `head.number < self.first_block`; `standard.rs:156-192` (`on_new_finalized_head`) additionally suppresses reports at/above the position until `position_is_canonical`. Combined effect: a report finalizing the entire stored chain while no new block arrives is not applied until the next block. + +### N11 — Anchored tip query: CONFLICT vs NO_DATA ordering diverges from the reference model +**Verdict (2026-07-12): SPEC BUG — the implementation is correct; fix RP-5 + the §12 model (S2).** +**Where:** `crates/hotblocks/src/query/service.rs:127-139` — for `from == head+1` with mismatched `expected_parent`, the impl returns 409 immediately (hints = `[head]`); the model returns NO_DATA because `from > hi`. +**Why the impl is right, precisely:** at `from == head+1` (resp. `fin+1` for finalized-stream) the client cursor is `(L+1, hash(L))` with `L == head.number` — both sides reference the *same exact position*, so a hash mismatch is a definite fork on any chain (dense or slot-numbered), and the immediate 409 is the only way a tip-follower learns of a reorg without waiting for the next block. **Why the naive general fix is wrong:** for `from > head+1`, `hash_at(from−1)` (DEF-16) equals the head's hash, but the client's `expected_parent` may name a block the service simply hasn't ingested yet (client ahead of a lagging service / other replica) — a CONFLICT there would force a healthy client to roll back. So the correct rule is exactly the impl's special case at the boundary, and NO_DATA above it. +**How it was missed:** RP-5's blanket "`from > hi` → wait → NO_DATA" has no tip exception; the model §12 copied that order; and the CONFLICT path is still `awaits CT-4` in the matrix, so no model-vs-SUT collision had forced the question. CT-4 written against the model as-is would have failed the *correct* implementation. + +### N12 (P3) — 204/NO_DATA loses watermarks on every path (GAP-9, precise mechanism) +**Where:** both construction sites hardcode the absence — `service.rs:155` (`QueryIsAboveTheHead { finalized_head: None }` on wait timeout) and `running.rs:146` (same, when the snapshot has no chunk ≥ `from`). The header-attaching code in `api.rs:289-295` is reachable only with `Some`, i.e. never. Fix is to populate the current finalized head (and add a head-number header, IB-6). + +### N13 (P3) — Finalized query can 500 on a benign race +**Violates:** INV-26 ("no trigger condition yields INTERNAL"). +**Where:** `running.rs:189` — `bail!("Finalized head is not available yet")`: admission saw `fin`, a RETAIN/RESET cleared it before the snapshot → plain anyhow → 500. Should map to NO_DATA. + +### N14 (P2) — Read-side panics bypass the whole error taxonomy +**Violates:** FM-1, INV-26, OB-5/SLI-10 (uncounted failures). +**Where:** (a) `crates/hotblocks/src/types.rs:65` — `DatasetKind::from_query` → `unimplemented!()` for `substrate`/`fuel`, which parse and validate fine (= GAP-11; note: it kills the connection task, not the process — register wording overstates); (b) `crates/hotblocks/src/query/executor.rs:80` — `rx.await.expect("task panicked")`: any panic inside plan execution on the polars pool (e.g. GAP-12-class data) re-panics the handler. Neither increments any metric; SLI-10 is blind to them. + +### N15 (P2) — `parent_number` linkage is never validated (INV-1 residual of GAP-20) +**Violates:** WP-2, DEF-4, INV-1 (`b_{i+1}.parent_number = b_i.number`). +**Where:** `sqd_primitives::Block` exposes `parent_number()` (`crates/primitives/src/types.rs:83`), and **no layer reads it for validation** — not `accept_new_block` (hash only, `standard.rs:133-154`), not `push_block` (hash only, `ingest_generic.rs:192-197`), not `insert_fork` (chunk-boundary hash only). GAP-20's literal scenario (non-monotonic numbers) is now unreachable — `accept_new_block` advances `position.first_block = number + 1`, forcing ascent — but a hash-linked lie remains storable: an EVM source serving block #205 with `parent_hash = hash(#100)` at position 101 creates a silent 104-block false hole on a densely-numbered chain. + +### N16 (P3) — IB-6 head-number header can undercut the snapshot head after a head-lowering fork +**Where:** `api.rs:250` computes the header from the *current* controller head (`max(fin, current head)`), never consulting the snapshot's head. A fork that lowers the head number between snapshot and response start yields a header below blocks actually emitted. IB-6 requires "at least the snapshot head". + +### N17 (P3) — WP-6 resume-point computation is chunk-granular, not block-granular +**Where:** `write_controller.rs:107-137` — hints are matched only against chunk **last** blocks; a hint matching a mid-chunk stored block is skipped and the rollback lands on the next lower chunk boundary (consistently, `insert_fork` can only splice at chunk boundaries). Content-wise benign (the replacement re-serves identical blocks), but it deviates from WP-6's MUST-algorithm, inflates reorg depth observables, re-ingests more than needed, and the reference model will diverge on rollback positions. Spec should either allow boundary-granular rollback (≤ one chunk deeper than necessary) or the impl must refine within-chunk matching. + +### N18 (P3) — OB-1 incomplete: commit version not exported +**Where:** `metrics.rs` exports `first/last/last_finalized/last_block_timestamp` per dataset; `DatasetLabel.version` exists but is not exposed, and the retention policy in force isn't either. OB-1 lists both. (Also: OB-3 stall/pressure gauges are on PR #83, not this branch — the §5 matrix row "stall gauges … exist" is ahead of reality; `get_global_tx_restarts` (HZ-11) is warn-log only.) + +--- + +## 2. Gap-register corrections (spec/12 §6) + +| GAP | Verdict after re-check | Action | +|---|---|---| +| GAP-2 | **Confirmed, sharper**: anchor is reconstructed as `first_chunk.last_block_hash()` — a hash from *inside* the window (`write_controller.rs:40`). Fix = `first_chunk.parent_block_hash()`. | keep; add file:line + one-line fix | +| GAP-3/5 | Confirmed exactly (no anchor-contradiction check, no strike escalation, every integrity fault = generic 60 s epoch loop, no alarm states anywhere). | keep | +| GAP-4 | Confirmed (`finalize`: `new < head_chunk.last_block()` arm accepts the hash unverified); composed path in `new_chunk` has the same hole. | keep; note the second site | +| GAP-6 | **Stale.** PR #79 landed: runtime cleanup point-deletes every 10 s (`main.rs::db_cleanup_task`) + deletion-collector factory + 7-day periodic compaction → default config now reclaims. Remaining: orphan dirty-table purge + file unlink are still behind `--startup-disk-reclaim` (off), so interrupted-build residue still leaks by default and still pins the *startup-unlink* watermark (only). RS-6/LIV-7 plausibly hold now; RS-10 still partially violated. | rewrite the entry; demote from P0 pending CT-7 soak evidence | +| GAP-8 | **Wrong as stated.** A de-facto zero-emission carrier exists: with `include_all=false`, the first and last covered block of every chunk range are force-selected (0-weight entries, `crates/query/src/plan/plan.rs:326-360`) and emitted as header-only records, so a client can always advance on the last emitted header. But it is unspecified (IB-9 forbids relying on it) and contradicts the spec's emission rule (see S3). | rewrite: "carrier exists, unspecified, and violates INV-22 as written" | +| GAP-9 | Confirmed; both sites hardcode `finalized_head: None` (N12). | keep, add mechanism | +| GAP-10 | Confirmed (mid-stream `Busy`/errors → clean truncation via `stream.finish()`, `api.rs:267-286`; nothing counts it). | keep | +| GAP-11 | Confirmed; **wording fix**: `unimplemented!()` in `types.rs:65` kills the connection task, not the process. Add executor `expect` (N14b) to the same entry. | reword | +| GAP-12 | Confirmed plausible; exact panic site `crates/storage/src/table/stats/builder.rs:101-102` (`from_utf8(..).unwrap()` on stats min/max), runs inside `write_new_chunk` → per-dataset crash-loop. | keep, add site | +| GAP-13 | Confirmed: 30 MB bound applies to the builder buffer only (`ingest_generic.rs:210-217`); the processor accumulates until 200 k rows regardless of bytes. | keep | +| GAP-14 | Confirmed (`QueryExecutor.in_flight` and `WaitSlots` are global). | keep | +| GAP-15 | Confirmed (borsh enum decode fails per-op at runtime; no boot gate). | keep | +| GAP-17/18 | Unchanged (no drain of ingest on shutdown beyond HTTP graceful; dual-writer checks only in `finalize`). | keep | +| GAP-20 | **Rework**: literal scenario unreachable (position advance in `accept_new_block` forces ascending numbers); the surviving hole is unvalidated `parent_number` linkage / false holes on dense chains (N15). | rewrite | +| GAP-21 | Confirmed exactly: hard-coded 100-position lookback + empty window → anyhow → 500 (`crates/query/src/plan/plan.rs:142-180`). | keep | +| GAP-22 | Confirmed (`compute_rollback` fallback returns `(first_block, parent_block_hash)` ignoring `fin`, `write_controller.rs:139-142`) — and **upgrade**: combined with the composed-finality guard it escalates to silent finalized-prefix replacement (N1), not just a parked epoch. | merge with N1, raise priority | + +## 3. Parameter-registry corrections (spec/14) + +- `P-SOURCE-STRIKES`: Observed is **absent**, not "(implicit)" — nothing counts rejections (N9). +- `P-RETENTION-SLACK`: Observed "one batch span" is wrong under compaction — trim granularity is one **merged chunk** (up to `MAX_CHUNK_SIZE = 200_000` rows, `crates/storage/src/db/ops/compaction.rs:17`). +- `P-RECLAIM-LAG` / `P-SPACE-AMP`: "∞ / unbounded in default config" are stale post-PR #79; new observed = deletion-collector + `P-CLEANUP-PERIOD` 10 s, backstopped by 7-day periodic compaction (`DEFAULT_PERIODIC_COMPACTION_SECS`). +- `P-HEAD-WAIT` (5 s), `P-BATCH-ROWS` (200 k), `P-WAITERS` (64 000), `P-EXEC-SLOTS` (threads×200), `P-MAX-ITEM-REQ` (100), `P-FORK-CONSENSUS` (2 s), `P-SOURCE-BACKOFF` (0→10 s), `P-EPOCH-RETRY` (60 s), `P-PROBE-WAIT` (5 s), `P-RESP-WEIGHT` (20 MB, `plan.rs:371`), `P-RESP-FLUSH` (512 KB, `response.rs:202`), `P-QUERY-TIME` (10 s), encodings — all verified matching. ✓ + +## 4. Spec-side fixes (the spec is wrong or silent; impl is defensible) + +- **S1 — Empty `External`/`Unbounded` datasets idle.** DEF-9/WP-5 promise a "kind-default position"; none exists. Impl: empty + `None`/`Api` → `State::Idle` until data or an instruction (`dataset_controller.rs:354-360`). Spec should document deferred ingestion instead. +- **S2 — Model ordering for anchored tip queries (N11) — CONFIRMED, spec must change.** Amend RP-5 and the §12 model: let `T` = `head` (live) / `fin` (finalized-only); if `expected_parent ≠ ⊥` and `from == T.number + 1` and `expected_parent ≠ T.hash`, return CONFLICT **before** the head-wait. Do NOT generalize to `from > T.number + 1` — there the assertion references a block the service may simply not have yet, and CONFLICT would force a merely-ahead client to roll back; NO_DATA stays correct above the boundary. +- **S3 — Emission rule vs the de-facto coverage carrier (GAP-8/N-A10).** Either specify "the first and last covered block of the effective range (per response) are always emitted, header-only if unmatched" as the RP-9 carrier and adjust INV-22/the model's `emitted` predicate — or remove the behavior and add an explicit trailer. Today conformance tests would fail the impl for emitting blocks the model forbids. +- **S4 — WP-6 granularity (N17).** Allow boundary-granular rollback: "the resume point MAY be up to one storage-batch below the optimal matching hint, provided it stays ≥ the floors". +- **S5 — Traceability matrix**: OB-row "stall gauges exist" belongs to PR #83, not this branch; INV-15/18 row should note INV-18 is currently *violated* by design of Window trims (N5), not merely untested. + +## 5. Verified-conformant (spot checks worth recording) + +- WP-4 arbitration: majority / all-active / 2 s timeout corroboration, contradiction cancels the timer; longest-hints fork chosen (`standard.rs:282-335`). ✓ +- WP-16 idempotency: below-position blocks skipped silently. ✓ WP-3 flush triggers: rows bound, availability-mask change, `MaybeOnHead`. ✓ (bytes bound is soft = GAP-13) +- WP §2.2/2.3 front attachment at commit: `insert_fork` verifies chunk-boundary hash linkage; run-internal linkage at accept time; batch-internal at push. ✓ (parent_number excepted — N15) +- INV-10/WP-13: two-phase build (dirty tables → atomic index+label tx); readers see committed only; crash residue invisible (FM-STOR-5). ✓ +- DEF-10: every `update_dataset` bumps the persisted label version. ✓ +- DEF-16 in queries: mid-chunk anchored check via `parent_number` window is hole-aware; retention validation (`get_parent_block_hash`, `find_block_row` ≥-semantics) is hole-aware. ✓ +- INV-25: weight-filter `head(1)` fallback emits the first block even when it alone exceeds 20 MB (`plan.rs:375-377`). ✓ +- RP-8: chunk-level availability from `data_availability_mask` (field presence, not emptiness); mask change forces a chunk boundary; mid-range boundary → truncation, next query → 400. ✓ +- RS-3/RS-5/RS-7: trims are chunk-conservative; two-phase deletion; point-delete cleanup is snapshot-safe. ✓ +- CN-6/6b: default RocksDB WAL (no sync) = 0-loss on process crash, bounded on host crash. ✓ (registry: "make explicit" still open) +- FM-OP-3: cross-process double-open prevented by the RocksDB LOCK file; in-process label conflicts via optimistic tx. ✓ (per-path checks = GAP-18) diff --git a/crates/hotblocks/tests/ct1_happy_path.rs b/crates/hotblocks/tests/ct1_happy_path.rs new file mode 100644 index 00000000..a1cd6955 --- /dev/null +++ b/crates/hotblocks/tests/ct1_happy_path.rs @@ -0,0 +1,121 @@ +//! CT-1 — stateful conformance on a happy-path script (spec 12 §3), run against every kind the +//! service serves. +//! +//! Verifies INV-1..3 (structural chain), INV-5/6 (watermark bounds), INV-7 (provenance), +//! INV-11 (append), INV-12 (finality monotone), INV-21/22 (response shape and completeness), +//! INV-23 (anchored ancestry), INV-25 (progress), INV-27 (range honesty), INV-30 (reporting), +//! RP-5 (bounded wait), RP-9/10 (coverage, continuation), OB-1 (chain gauges). + +use std::{sync::Arc, time::Duration}; + +use anyhow::Result; +use sqd_hotblocks_harness::{ + chain::{Chain, Evm, HlFills, Solana}, + driver::FollowStep, + harness::{Harness, HarnessConfig}, + sim::Numbering +}; + +const START: u64 = 1_000; +const PATIENCE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread")] +async fn ct1_evm() -> Result<()> { + ct1(Arc::new(Evm), Numbering::Dense).await +} + +/// Solana numbers blocks by time-based slots, and a slot that produced nothing leaves a hole. +/// The window is still one chain — carried by `parentNumber`, not by the numbering (INV-1/2). +/// The service links batches and chunks by hash and never by number, so it must not care. +#[tokio::test(flavor = "multi_thread")] +async fn ct1_solana() -> Result<()> { + ct1(Arc::new(Solana), Numbering::Sparse).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn ct1_hyperliquid_fills() -> Result<()> { + ct1(Arc::new(HlFills), Numbering::Dense).await +} + +async fn ct1(chain: Arc, numbering: Numbering) -> Result<()> { + let mut cfg = HarnessConfig::from_block(env!("CARGO_BIN_EXE_sqd-hotblocks"), chain, START); + cfg.numbering = numbering; + let mut h = Harness::start(cfg).await?; + + // `Sut::drop` prints the service log on unwind, so a failure comes with its own account. + if let Err(err) = run(&mut h).await { + panic!("CT-1 ({}) failed: {err:?}", h.chain.config_kind()); + } + Ok(()) +} + +async fn run(h: &mut Harness) -> Result<()> { + // --- Act 1: cold ingest of a window, finality trailing the tip ------------------------- + h.produce(50)?; + h.finalize_with_lag(5)?; + + h.settle().await?; + h.assert_conforms().await?; + + assert_eq!( + h.model.first(), + Some(START), + "the window starts where the config pinned it" + ); + assert_eq!(h.model.span(), 50); + assert!(h.model.fin.is_some(), "finality trails the tip"); + + if h.numbering == Numbering::Sparse { + let holes = h.model.seg.windows(2).filter(|w| w[1].number > w[0].number + 1).count(); + assert!(holes > 0, "the sparse script produced a chain with no holes in it"); + } + + // --- Act 2: an anchored client follows the tip ------------------------------------------ + let mut follower = h.follower(); + let tip = h.model.head().expect("a head").number; + follower.follow_to(&h.client, &*h.chain, tip, PATIENCE).await?; + assert_eq!(follower.local.len(), 50, "the follower backfilled the whole window"); + + for _ in 0..4 { + h.produce(5)?; + h.finalize_with_lag(5)?; + let target = h.model.head().expect("a head").number; + follower.follow_to(&h.client, &*h.chain, target, PATIENCE).await?; + } + + h.settle().await?; + h.assert_conforms().await?; + + // The client's chain is the model's window, block for block (INV-23/25). + assert_eq!(follower.local.len(), h.model.span()); + for (got, want) in follower.local.iter().zip(&h.model.seg) { + assert_eq!(got.number, want.number); + assert_eq!( + got.hash, want.hash, + "block {} came back on the wrong branch", + got.number + ); + assert_eq!(got.parent_hash, want.parent_hash); + } + assert_eq!( + follower.rollbacks, 0, + "a happy-path script must never force a client to roll back" + ); + + // Above the head: a bounded wait, then NO_DATA — a poll, not an error (RP-5). + assert_eq!(follower.step(&h.client, &*h.chain).await?, FollowStep::NoData); + + let stats = h.sim.stats(&h.dataset); + assert_eq!( + stats.below_history, 0, + "the service asked the source for blocks below its window" + ); + assert_eq!(stats.fork_signals, 0, "a happy-path script produces no fork signals"); + assert!( + stats.blocks_served >= 70, + "the source served {} blocks", + stats.blocks_served + ); + + Ok(()) +} diff --git a/crates/hotblocks/tests/ct9_source_faults.rs b/crates/hotblocks/tests/ct9_source_faults.rs new file mode 100644 index 00000000..016bc9f8 --- /dev/null +++ b/crates/hotblocks/tests/ct9_source_faults.rs @@ -0,0 +1,56 @@ +//! CT-9 — source-fault corpus (spec 12 §3): a fault at the source surface must not take the +//! write path down with it (FM-1, LIV-2). +//! +//! CT-1 hit this one on the harness's first run: a JSONL body whose last record had no trailing +//! newline panicked `LineStream::take_final_line`, killing the ingest task and parking the +//! dataset for `P-EPOCH-RETRY` — forever, since the source serves the same body on retry. + +use std::{sync::Arc, time::Duration}; + +use anyhow::Result; +use sqd_hotblocks_harness::{ + chain::HlFills, + harness::{Harness, HarnessConfig} +}; + +const START: u64 = 1_000; + +/// An unterminated record is still a record: the service must ingest it and keep ingesting. +#[tokio::test(flavor = "multi_thread")] +async fn ct9_unterminated_final_record_does_not_stall_ingestion() -> Result<()> { + let mut h = Harness::start(HarnessConfig::from_block( + env!("CARGO_BIN_EXE_sqd-hotblocks"), + Arc::new(HlFills), + START + )) + .await?; + + h.sim.inject_fault(&h.dataset, |f| f.unterminated_final_line = true); + + if let Err(err) = run(&mut h).await { + panic!("CT-9 failed: {err:?}"); + } + Ok(()) +} + +async fn run(h: &mut Harness) -> Result<()> { + h.produce(20)?; + h.finalize_with_lag(5)?; + + // The whole assertion: it converges. A panicked ingest task would sit out `P-EPOCH-RETRY` + // and this would time out on an empty window. + h.settle().await?; + h.assert_conforms().await?; + + // And keeps going — not a fault the service survives exactly once. + h.produce(10)?; + h.finalize_with_lag(5)?; + h.settle().await?; + h.assert_conforms().await?; + + assert!( + h.sut.last_startup < Duration::from_secs(60), + "sanity: the service was never restarted by the harness" + ); + Ok(()) +}