From 10347bebd868326f07a58d00f2f96a3a5309a270 Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Tue, 14 Jul 2026 21:20:48 +0300 Subject: [PATCH 1/3] feat(hotblocks): add block hash lookup index Add an opt-in EVM hash-to-block-number index, expose it through the hotblocks HTTP API, and keep entries consistent across ingest, forks, retention, compaction, flag changes, and dataset deletion. Extend the black-box harness with typed hash lookup support and end-to-end coverage for successful, unknown, and replaced-branch hashes. Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 Co-Authored-By: Codex --- crates/hotblocks-harness/src/compare.rs | 22 ++ crates/hotblocks-harness/src/driver.rs | 17 + crates/hotblocks-harness/src/harness.rs | 5 + crates/hotblocks/src/api.rs | 42 +++ crates/hotblocks/src/cli.rs | 11 +- .../dataset_controller/dataset_controller.rs | 14 +- crates/hotblocks/tests/block_hash_index.rs | 80 +++++ crates/storage/src/db/data.rs | 44 +++ crates/storage/src/db/db.rs | 91 +++-- crates/storage/src/db/read/blocks_table.rs | 68 ++++ crates/storage/src/db/read/snapshot.rs | 29 +- crates/storage/src/db/write/dataset_update.rs | 4 +- crates/storage/src/db/write/tx.rs | 116 +++++- crates/storage/tests/block_hash_index.rs | 339 ++++++++++++++++++ 14 files changed, 846 insertions(+), 36 deletions(-) create mode 100644 crates/hotblocks/tests/block_hash_index.rs create mode 100644 crates/storage/tests/block_hash_index.rs diff --git a/crates/hotblocks-harness/src/compare.rs b/crates/hotblocks-harness/src/compare.rs index e06366f6..b2eab639 100644 --- a/crates/hotblocks-harness/src/compare.rs +++ b/crates/hotblocks-harness/src/compare.rs @@ -96,6 +96,28 @@ pub async fn await_quiescence(client: &Client, model: &Model, q: &Quiescence) -> } } +/// Every block on the model's canonical branch must resolve to the same block reference through +/// the public hash-index endpoint. This is opt-in because the production index itself is opt-in. +pub async fn assert_hash_index_conforms(client: &Client, model: &Model) -> Result<()> { + let mut errs = Vec::new(); + for block in &model.seg { + let expected = Some(block.as_ref()); + let observed = client.block_by_hash(&block.hash).await?; + if observed != expected { + errs.push(format!("hash {}: expected {expected:?}, got {observed:?}", block.hash)); + } + } + + if errs.is_empty() { + Ok(()) + } else { + bail!( + "the block-hash index diverged from the model:\n - {}", + errs.join("\n - ") + ) + } +} + /// Diff every observable against the model, reporting *all* violations, not the first. pub async fn assert_conforms(client: &Client, model: &Model, chain: &dyn Chain, dataset: &str) -> Result<()> { let mut errs: Vec = Vec::new(); diff --git a/crates/hotblocks-harness/src/driver.rs b/crates/hotblocks-harness/src/driver.rs index a21f9dfc..d402bd59 100644 --- a/crates/hotblocks-harness/src/driver.rs +++ b/crates/hotblocks-harness/src/driver.rs @@ -148,6 +148,23 @@ impl Client { .await?) } + /// Resolves a block hash through the public hash-index endpoint. A missing hash is a + /// normal `None`; every other non-success response is a binding failure. + pub async fn block_by_hash(&self, hash: &str) -> Result> { + let res = self.http.get(self.url(&format!("hashes/{hash}/block"))).send().await?; + let status = res.status(); + + match status.as_u16() { + 200 => Ok(Some(res.json().await.context("malformed block-hash lookup payload")?)), + 404 => Ok(None), + _ => bail!( + "block-hash lookup failed with {}: {}", + status.as_u16(), + res.text().await.unwrap_or_default() + ) + } + } + /// SET-RETENTION (DEF-9). Returns the status code: `403` unless the dataset is API-controlled. pub async fn set_retention(&self, policy: &Value) -> Result { let res = self.http.post(self.url("retention")).json(policy).send().await?; diff --git a/crates/hotblocks-harness/src/harness.rs b/crates/hotblocks-harness/src/harness.rs index e468d4f8..f079d675 100644 --- a/crates/hotblocks-harness/src/harness.rs +++ b/crates/hotblocks-harness/src/harness.rs @@ -165,6 +165,11 @@ impl Harness { compare::assert_conforms(&self.client, &self.model, &*self.chain, &self.dataset).await } + /// Diff the opt-in block-hash lookup endpoint against every canonical block in the model. + pub async fn assert_hash_index_conforms(&self) -> Result<()> { + compare::assert_hash_index_conforms(&self.client, &self.model).await + } + /// An anchored follower positioned at the bottom of the window (04 §7). pub fn follower(&self) -> Follower { Follower::new( diff --git a/crates/hotblocks/src/api.rs b/crates/hotblocks/src/api.rs index 57cb7d41..7d984c7a 100644 --- a/crates/hotblocks/src/api.rs +++ b/crates/hotblocks/src/api.rs @@ -61,6 +61,7 @@ pub fn build_api(app: App) -> Router { .route("/datasets/{id}/finalized-stream", post(finalized_stream)) .route("/datasets/{id}/head", get(get_head)) .route("/datasets/{id}/finalized-head", get(get_finalized_head)) + .route("/datasets/{id}/hashes/{hash}/block", get(get_block_by_hash)) .route("/datasets/{id}/retention", get(get_retention).post(set_retention)) .route("/datasets/{id}/status", get(get_status)) .route("/datasets/{id}/metadata", get(get_metadata)) @@ -371,6 +372,47 @@ async fn get_head( }) } +async fn get_block_by_hash( + Extension(app): Extension, + Extension(client_id): Extension, + Path((dataset_id, hash)): Path<(DatasetId, String)> +) -> impl IntoResponse { + // Reject absurd lengths before touching the DB (real hashes are 64-88 chars). + if hash.is_empty() || hash.len() > 256 { + return ResponseWithMetadata::new() + .with_client_id(&client_id) + .with_dataset_id(dataset_id) + .with_endpoint("/hashes/{hash}/block") + .with_response(|| text!(StatusCode::BAD_REQUEST, "invalid hash length")); + } + + let dataset = match app.data_service.get_dataset(dataset_id) { + Ok(ds) => ds, + Err(err) => { + return ResponseWithMetadata::new() + .with_client_id(&client_id) + .with_dataset_id(dataset_id) + .with_endpoint("/hashes/{hash}/block") + .with_response(|| text!(StatusCode::NOT_FOUND, "{}", err)); + } + }; + + let response = match dataset.get_block_by_hash(hash).await { + Ok(Some(block_ref)) => json_ok!(block_ref), + Ok(None) => text!(StatusCode::NOT_FOUND, "block not found"), + Err(err) => { + error!(error = ?err, dataset_id = %dataset_id, "get_block_by_hash failed"); + text!(StatusCode::INTERNAL_SERVER_ERROR, "internal error") + } + }; + + ResponseWithMetadata::new() + .with_client_id(&client_id) + .with_dataset_id(dataset_id) + .with_endpoint("/hashes/{hash}/block") + .with_response(|| response) +} + async fn get_retention( Extension(app): Extension, Extension(client_id): Extension, diff --git a/crates/hotblocks/src/cli.rs b/crates/hotblocks/src/cli.rs index 0425adcb..236a7675 100644 --- a/crates/hotblocks/src/cli.rs +++ b/crates/hotblocks/src/cli.rs @@ -86,6 +86,14 @@ pub struct CLI { #[arg(long)] pub startup_disk_reclaim: bool, + /// Index block hashes of newly ingested chunks, enabling + /// `GET /datasets/{id}/hashes/{hash}/block`. EVM datasets only. + /// + /// No backfill: pre-existing chunks stay unresolvable until they roll off + /// via retention. Entries drain as chunks are pruned after switching off. + #[arg(long)] + pub block_hash_index: bool, + /// Known client IDs for metrics labeling. Client IDs not in this list /// will be reported as "unknown" to prevent metrics cardinality abuse. #[arg(long = "known-client", value_name = "ID")] @@ -111,7 +119,8 @@ impl CLI { .with_direct_io(!self.rocksdb_disable_direct_io) .with_max_log_file_size(self.rocksdb_max_log_file_size) .with_keep_log_file_num(self.rocksdb_keep_log_file_num) - .with_periodic_compaction_secs(self.rocksdb_periodic_compaction_secs); + .with_periodic_compaction_secs(self.rocksdb_periodic_compaction_secs) + .with_block_hash_index(self.block_hash_index); if let Some(jobs) = self.rocksdb_max_background_jobs { settings = settings.with_max_background_jobs(jobs); diff --git a/crates/hotblocks/src/dataset_controller/dataset_controller.rs b/crates/hotblocks/src/dataset_controller/dataset_controller.rs index 9fe681b8..be2bdf16 100644 --- a/crates/hotblocks/src/dataset_controller/dataset_controller.rs +++ b/crates/hotblocks/src/dataset_controller/dataset_controller.rs @@ -18,6 +18,7 @@ use crate::{ }; pub struct DatasetController { + db: DBRef, dataset_id: DatasetId, dataset_kind: DatasetKind, retention_sender: tokio::sync::watch::Sender, @@ -71,9 +72,10 @@ impl DatasetController { let task = tokio::spawn(ctl.run(write).in_current_span()); let compaction_task = - tokio::spawn(compaction_loop(db, dataset_id, compaction_enabled_receiver).in_current_span()); + tokio::spawn(compaction_loop(db.clone(), dataset_id, compaction_enabled_receiver).in_current_span()); Ok(Self { + db, dataset_id, dataset_kind, retention_sender, @@ -105,6 +107,16 @@ impl DatasetController { self.head_receiver.borrow().as_ref().map(|h| h.number) } + /// Resolves a block hash to its `BlockRef` via the storage index. + /// `Ok(None)` means the hash is not in the index. + pub async fn get_block_by_hash(&self, hash: String) -> anyhow::Result> { + let db = self.db.clone(); + let dataset_id = self.dataset_id; + tokio::task::spawn_blocking(move || db.snapshot().find_block_by_hash(dataset_id, &hash)) + .await + .context("get_block_by_hash task panicked")? + } + pub fn enable_compaction(&self, yes: bool) { let _ = self.compaction_enabled_sender.send(yes); } diff --git a/crates/hotblocks/tests/block_hash_index.rs b/crates/hotblocks/tests/block_hash_index.rs new file mode 100644 index 00000000..1266f440 --- /dev/null +++ b/crates/hotblocks/tests/block_hash_index.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; + +use anyhow::Result; +use sqd_hotblocks_harness::{ + chain::Evm, + harness::{Harness, HarnessConfig}, + types::block_hash +}; + +const START: u64 = 1_000; + +#[tokio::test(flavor = "multi_thread")] +async fn hash_lookup_resolves_ingested_blocks_and_returns_none_for_unknown_hash() -> Result<()> { + let mut h = start_harness().await?; + + if let Err(err) = assert_ingest_lookup(&mut h).await { + panic!("block-hash ingest lookup failed: {err:?}"); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn hash_lookup_removes_the_replaced_branch_after_a_reorg() -> Result<()> { + let mut h = start_harness().await?; + + if let Err(err) = assert_reorg_lookup(&mut h).await { + panic!("block-hash reorg lookup failed: {err:?}"); + } + Ok(()) +} + +async fn start_harness() -> Result { + let mut cfg = HarnessConfig::from_block(env!("CARGO_BIN_EXE_sqd-hotblocks"), Arc::new(Evm), START); + cfg.sut_args.push("--block-hash-index".into()); + Harness::start(cfg).await +} + +async fn assert_ingest_lookup(h: &mut Harness) -> Result<()> { + h.produce(20)?; + h.settle().await?; + h.assert_hash_index_conforms().await?; + + let unknown_hash = block_hash(START + 1_000, 99); + assert_eq!( + h.client.block_by_hash(&unknown_hash).await?, + None, + "a hash outside the ingested chain must return 404" + ); + + Ok(()) +} + +async fn assert_reorg_lookup(h: &mut Harness) -> Result<()> { + h.produce(20)?; + h.settle().await?; + h.assert_hash_index_conforms().await?; + + let fork_from = START + 10; + let stale_hashes: Vec = h + .model + .blocks_in(fork_from, START + 19) + .iter() + .map(|block| block.hash.clone()) + .collect(); + + h.fork(fork_from, 10)?; + h.settle().await?; + h.assert_conforms().await?; + h.assert_hash_index_conforms().await?; + + for hash in stale_hashes { + assert_eq!( + h.client.block_by_hash(&hash).await?, + None, + "a hash from the replaced branch must return 404" + ); + } + + Ok(()) +} diff --git a/crates/storage/src/db/data.rs b/crates/storage/src/db/data.rs index a4fda754..21c5138e 100644 --- a/crates/storage/src/db/data.rs +++ b/crates/storage/src/db/data.rs @@ -99,6 +99,50 @@ impl Display for ChunkId { } } +/// Key for the `hash -> block number` index in `CF_BLOCK_HASHES`: +/// `dataset_id (48 bytes) || hash UTF-8 bytes`, the hash exactly as it appears +/// in the Arrow `hash` column (no normalization). +pub(crate) struct BlockHashIndexKey { + bytes: Vec +} + +impl BlockHashIndexKey { + pub fn new(dataset_id: DatasetId, hash: &str) -> Self { + let mut bytes = Vec::with_capacity(48 + hash.len()); + bytes.extend_from_slice(dataset_id.as_ref()); + bytes.extend_from_slice(hash.as_bytes()); + Self { bytes } + } + + /// The `[start, end)` key range covering every entry of `dataset_id`. + pub fn dataset_range(dataset_id: DatasetId) -> (Vec, Vec) { + let start = dataset_id.as_ref().to_vec(); + let end = prefix_upper_bound(&start); + (start, end) + } +} + +impl AsRef<[u8]> for BlockHashIndexKey { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +/// Exclusive upper bound of `prefix`'s key range: increments the last +/// non-`0xFF` byte. A `DatasetId` is never all-`0xFF`, so the bound is +/// always non-empty. +fn prefix_upper_bound(prefix: &[u8]) -> Vec { + let mut end = prefix.to_vec(); + while let Some(last) = end.last_mut() { + if *last < u8::MAX { + *last += 1; + return end; + } + end.pop(); + } + end +} + #[derive(BorshSerialize, BorshDeserialize, Debug, Clone, Eq, PartialEq)] pub enum Chunk { V0 { diff --git a/crates/storage/src/db/db.rs b/crates/storage/src/db/db.rs index 5cfd19a6..eb27c8cf 100644 --- a/crates/storage/src/db/db.rs +++ b/crates/storage/src/db/db.rs @@ -6,7 +6,7 @@ use rocksdb::{ColumnFamilyDescriptor, Options as RocksOptions}; use sqd_primitives::Name; use super::{ - data::{Dataset, DatasetId, DatasetKind, DatasetLabel}, + data::{BlockHashIndexKey, Dataset, DatasetId, DatasetKind, DatasetLabel}, read::snapshot::ReadSnapshot }; use crate::db::{ @@ -22,6 +22,7 @@ pub const CF_CHUNKS: Name = "CHUNKS"; pub const CF_TABLES: Name = "TABLES"; pub const CF_DIRTY_TABLES: Name = "DIRTY_TABLES"; pub const CF_DELETED_TABLES: Name = "DELETED_TABLES"; +pub const CF_BLOCK_HASHES: Name = "BLOCK_HASHES"; /// Whole-file rewrite cadence for `CF_TABLES`. RocksDB leaves `periodic_compaction_seconds` /// disabled for leveled compaction without a compaction filter, so the effective baseline @@ -45,7 +46,8 @@ pub struct DatabaseSettings { keep_log_file_num: usize, auto_compactions: bool, max_background_jobs: usize, - periodic_compaction_secs: u64 + periodic_compaction_secs: u64, + block_hash_index: bool } /// RocksDB's default of 2 could not keep up with ingest during the NET-819 incident, but a @@ -69,7 +71,8 @@ impl Default for DatabaseSettings { keep_log_file_num: 10, auto_compactions: true, max_background_jobs: default_max_background_jobs(), - periodic_compaction_secs: DEFAULT_PERIODIC_COMPACTION_SECS + periodic_compaction_secs: DEFAULT_PERIODIC_COMPACTION_SECS, + block_hash_index: false } } } @@ -134,6 +137,14 @@ impl DatabaseSettings { self } + /// Whether newly ingested chunks get their block hashes indexed in + /// `CF_BLOCK_HASHES`. Write-side only: entries are always removed when + /// their chunk is pruned, so the index drains after the flag goes off. + pub fn with_block_hash_index(mut self, yes: bool) -> Self { + self.block_hash_index = yes; + self + } + fn db_options(&self) -> RocksOptions { let mut options = RocksOptions::default(); options.create_if_missing(true); @@ -224,17 +235,23 @@ impl DatabaseSettings { ColumnFamilyDescriptor::new(CF_CHUNKS, self.chunks_cf_options()), ColumnFamilyDescriptor::new(CF_TABLES, self.tables_cf_options()), ColumnFamilyDescriptor::new(CF_DIRTY_TABLES, self.cf_default_options()), - ColumnFamilyDescriptor::new(CF_DELETED_TABLES, self.cf_default_options()) + ColumnFamilyDescriptor::new(CF_DELETED_TABLES, self.cf_default_options()), + ColumnFamilyDescriptor::new(CF_BLOCK_HASHES, self.cf_default_options()) ] )?; - Ok(Database { db, options }) + Ok(Database { + db, + options, + block_hash_index: self.block_hash_index + }) } } pub struct Database { db: RocksDB, - options: RocksOptions + options: RocksOptions, + block_hash_index: bool } impl Database { @@ -293,12 +310,14 @@ impl Database { where F: FnMut(&mut DatasetUpdate<'_>) -> anyhow::Result { - Tx::new(&self.db).run(|tx| { - let mut upd = DatasetUpdate::new(tx, dataset_id)?; - let result = cb(&mut upd)?; - upd.finish()?; - Ok(result) - }) + Tx::new(&self.db) + .with_block_hash_index(self.block_hash_index) + .run(|tx| { + let mut upd = DatasetUpdate::new(tx, dataset_id)?; + let result = cb(&mut upd)?; + upd.finish()?; + Ok(result) + }) } pub fn snapshot(&self) -> ReadSnapshot<'_> { @@ -327,27 +346,57 @@ impl Database { } pub fn delete_dataset(&self, dataset_id: DatasetId) -> anyhow::Result<()> { + self.purge_block_hash_index(dataset_id)?; + + // Metadata is removed atomically in one transaction, so the dataset is + // never observed half-deleted. Tx::new(&self.db).run(|tx| { - let label = tx.find_label_for_update(dataset_id)?; - if label.is_none() { + if tx.find_label_for_update(dataset_id)?.is_none() { return Ok(()); } - - let chunks = tx.list_chunks(dataset_id, 0, None); - for chunk_result in chunks { + for chunk_result in tx.list_chunks(dataset_id, 0, None) { let chunk = chunk_result?; tx.delete_chunk(dataset_id, &chunk)?; } - - tx.delete_label(dataset_id)?; - - Ok(()) + tx.delete_label(dataset_id) })?; self.cleanup()?; Ok(()) } + /// Point-deletes every `CF_BLOCK_HASHES` entry of `dataset_id` in bounded + /// batches (`delete_range` is not supported on `OptimisticTransactionDB`). + /// Runs outside the metadata transaction: a crash in between leaves chunks + /// without index entries (hashes 404), not corruption. + fn purge_block_hash_index(&self, dataset_id: DatasetId) -> anyhow::Result<()> { + const BATCH_SIZE: usize = 10_000; + + let cf = self.db.cf_handle(CF_BLOCK_HASHES).unwrap(); + let (start, end) = BlockHashIndexKey::dataset_range(dataset_id); + + let mut read_opts = rocksdb::ReadOptions::default(); + read_opts.set_iterate_upper_bound(end); + + let mut cursor = self.db.raw_iterator_cf_opt(cf, read_opts); + cursor.seek(&start); + + let mut batch = RocksWriteBatch::default(); + while cursor.valid() { + batch.delete_cf(cf, cursor.key().unwrap()); + if batch.len() >= BATCH_SIZE { + self.db.write(std::mem::take(&mut batch))?; + } + cursor.next(); + } + cursor.status()?; + + if !batch.is_empty() { + self.db.write(batch)?; + } + Ok(()) + } + /// Phase 1 -- logically purge deleted tables (snapshot-safe point deletes). /// Returns the number of tables logically deleted by this call. pub fn cleanup(&self) -> anyhow::Result { diff --git a/crates/storage/src/db/read/blocks_table.rs b/crates/storage/src/db/read/blocks_table.rs index c9a947cd..2c7077dd 100644 --- a/crates/storage/src/db/read/blocks_table.rs +++ b/crates/storage/src/db/read/blocks_table.rs @@ -54,3 +54,71 @@ fn find_block_row(numbers: &[BN], block: BN) -> Option { .min_by_key(|e| e.1) .map(|e| e.0) } + +/// Streams all `(block number, hash)` pairs of a `blocks` table, reading the +/// columns in batches so peak memory stays `O(batch)` even for large compacted +/// chunks. `number` must be `UInt32`/`UInt64` and `hash` must be `Utf8`; +/// anything else is a hard error rather than silently indexed garbage. +pub fn for_each_block_hash( + blocks_table: &TableReader, + mut visit: impl FnMut(BlockNumber, &str) -> anyhow::Result<()> +) -> anyhow::Result<()> { + const BLOCK_HASH_BATCH_SIZE: usize = 4096; + + let schema = blocks_table.schema(); + + let number_idx = schema.index_of("number")?; + let number_type = schema.field(number_idx).data_type().clone(); + match number_type { + DataType::UInt32 | DataType::UInt64 => {} + ref ty => bail!("'number' column has unexpected data type - {}", ty) + } + + let hash_idx = schema.index_of("hash")?; + let hash_type = schema.field(hash_idx).data_type().clone(); + if hash_type != DataType::Utf8 { + bail!("'hash' column has unexpected data type - {}", hash_type) + } + + let num_rows = blocks_table.num_rows(); + let mut number_reader = blocks_table.create_column_reader(number_idx)?; + let mut hash_reader = blocks_table.create_column_reader(hash_idx)?; + + let mut offset = 0; + while offset < num_rows { + let len = std::cmp::min(BLOCK_HASH_BATCH_SIZE, num_rows - offset); + + let numbers = { + let mut builder = AnyBuilder::new(&number_type); + number_reader.read_slice(&mut builder, offset, len)?; + builder.finish() + }; + + let hashes = { + let mut builder = AnyBuilder::new(&hash_type); + hash_reader.read_slice(&mut builder, offset, len)?; + builder.finish() + }; + let hashes = hashes.as_string::(); + + match numbers.data_type() { + DataType::UInt32 => { + let numbers = numbers.as_primitive::().values(); + for i in 0..len { + visit(numbers[i] as BlockNumber, hashes.value(i))?; + } + } + DataType::UInt64 => { + let numbers = numbers.as_primitive::().values(); + for i in 0..len { + visit(numbers[i], hashes.value(i))?; + } + } + _ => unreachable!("'number' column type was validated above") + } + + offset += len; + } + + Ok(()) +} diff --git a/crates/storage/src/db/read/snapshot.rs b/crates/storage/src/db/read/snapshot.rs index 31659392..d44e8ed2 100644 --- a/crates/storage/src/db/read/snapshot.rs +++ b/crates/storage/src/db/read/snapshot.rs @@ -1,14 +1,14 @@ use std::{collections::BTreeMap, ops::Deref, sync::Arc}; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use parking_lot::Mutex; use rocksdb::{ColumnFamily, ReadOptions}; -use sqd_primitives::{BlockNumber, Name}; +use sqd_primitives::{BlockNumber, BlockRef, Name}; use crate::{ db::{ - data::{Chunk, DatasetId}, - db::{RocksDB, RocksIterator, RocksSnapshot, CF_CHUNKS, CF_DATASETS, CF_TABLES}, + data::{BlockHashIndexKey, Chunk, DatasetId}, + db::{RocksDB, RocksIterator, RocksSnapshot, CF_BLOCK_HASHES, CF_CHUNKS, CF_DATASETS, CF_TABLES}, read::chunk::ChunkIterator, table_id::TableId, DatasetLabel @@ -75,6 +75,27 @@ impl<'a> ReadSnapshot<'a> { self.list_chunks(dataset_id, 0, None).into_reversed().next().transpose() } + /// Resolves a block hash via the `CF_BLOCK_HASHES` index. `Ok(None)` means + /// the hash is not indexed (unknown, pre-index chunk, or non-indexed kind). + pub fn find_block_by_hash(&self, dataset_id: DatasetId, hash: &str) -> anyhow::Result> { + let key = BlockHashIndexKey::new(dataset_id, hash); + let Some(bytes) = self + .db + .get_pinned_cf_opt(self.cf_handle(CF_BLOCK_HASHES), &key, &self.new_options())? + else { + return Ok(None); + }; + // A wrong length means corruption; error rather than panic. + let arr: [u8; 8] = bytes + .as_ref() + .try_into() + .context("CF_BLOCK_HASHES value has unexpected length, expected 8 bytes")?; + Ok(Some(BlockRef { + number: BlockNumber::from_be_bytes(arr), + hash: hash.to_string() + })) + } + fn new_options(&self) -> ReadOptions { let mut options = ReadOptions::default(); options.set_snapshot(&self.snapshot); diff --git a/crates/storage/src/db/write/dataset_update.rs b/crates/storage/src/db/write/dataset_update.rs index 34006f88..f2e04f80 100644 --- a/crates/storage/src/db/write/dataset_update.rs +++ b/crates/storage/src/db/write/dataset_update.rs @@ -29,7 +29,8 @@ impl<'a> DatasetUpdate<'a> { pub fn insert_chunk(&self, chunk: &Chunk) -> anyhow::Result<()> { self.tx.validate_chunk_insertion(self.dataset_id, chunk)?; - self.tx.write_chunk(self.dataset_id, chunk) + self.tx.write_chunk(self.dataset_id, chunk)?; + self.tx.index_block_hashes(self.dataset_id, chunk) } pub fn insert_fork(&self, chunk: &Chunk) -> anyhow::Result<()> { @@ -47,6 +48,7 @@ impl<'a> DatasetUpdate<'a> { } pub fn delete_chunk(&self, chunk: &Chunk) -> anyhow::Result<()> { + self.tx.unindex_block_hashes(self.dataset_id, chunk)?; self.tx.delete_chunk(self.dataset_id, chunk) } diff --git a/crates/storage/src/db/write/tx.rs b/crates/storage/src/db/write/tx.rs index 4fca2e80..4e3a3a73 100644 --- a/crates/storage/src/db/write/tx.rs +++ b/crates/storage/src/db/write/tx.rs @@ -9,14 +9,17 @@ use rocksdb::ColumnFamily; use sqd_primitives::BlockNumber; use crate::db::{ - data::ChunkId, + data::{BlockHashIndexKey, ChunkId}, db::{ - RocksDB, RocksIterator, RocksTransaction, RocksTransactionOptions, CF_CHUNKS, CF_DATASETS, CF_DELETED_TABLES, - CF_DIRTY_TABLES + RocksDB, RocksIterator, RocksTransaction, RocksTransactionOptions, CF_BLOCK_HASHES, CF_CHUNKS, CF_DATASETS, + CF_DELETED_TABLES, CF_DIRTY_TABLES + }, + read::{ + blocks_table::{for_each_block_hash, get_parent_block_hash}, + chunk::ChunkIterator }, - read::{blocks_table::get_parent_block_hash, chunk::ChunkIterator}, table_id::TableId, - Chunk, DatasetId, DatasetLabel, ReadSnapshot + Chunk, DatasetId, DatasetKind, DatasetLabel, ReadSnapshot }; static GLOBAL_RESTARTS: AtomicU64 = AtomicU64::new(0); @@ -38,9 +41,17 @@ fn record_restart() { LOCAL_RESTARTS.with_borrow_mut(|val| *val = val.wrapping_add(1)) } +/// Datasets whose block hashes get indexed in `CF_BLOCK_HASHES`. EVM-only for +/// now; hyperliquid must stay out - its `hash` is not a crypto hash and can +/// collide. +fn is_indexed_kind(kind: DatasetKind) -> bool { + kind == DatasetKind::from_str("evm") +} + pub struct Tx<'a> { db: &'a RocksDB, - transaction: RocksTransaction<'a> + transaction: RocksTransaction<'a>, + block_hash_index: bool } impl<'a> Tx<'a> { @@ -50,7 +61,18 @@ impl<'a> Tx<'a> { let transaction = db.transaction_opt(&rocksdb::WriteOptions::default(), &tx_options); - Self { db, transaction } + Self { + db, + transaction, + block_hash_index: false + } + } + + /// Enables block hash indexing for chunks written through this transaction. + /// Set by [`Database::update_dataset`] from the database-level setting. + pub fn with_block_hash_index(mut self, yes: bool) -> Self { + self.block_hash_index = yes; + self } pub fn run(self, mut cb: F) -> anyhow::Result @@ -58,6 +80,7 @@ impl<'a> Tx<'a> { F: FnMut(&Self) -> anyhow::Result { let db = self.db; + let block_hash_index = self.block_hash_index; let mut tx = self; loop { let result = cb(&tx)?; @@ -65,7 +88,7 @@ impl<'a> Tx<'a> { Ok(_) => return Ok(result), Err(err) if err.kind() == rocksdb::ErrorKind::TryAgain || err.kind() == rocksdb::ErrorKind::Busy => { record_restart(); - tx = Self::new(db) + tx = Self::new(db).with_block_hash_index(block_hash_index) } Err(err) => return Err(err.into()) } @@ -133,12 +156,88 @@ impl<'a> Tx<'a> { Ok(()) } + /// Adds every `(hash, block number)` pair of `chunk`'s `blocks` table to + /// `CF_BLOCK_HASHES`. No-op unless indexing is enabled on this transaction + /// and the dataset kind is whitelisted in [`is_indexed_kind`]. + pub fn index_block_hashes(&self, dataset_id: DatasetId, chunk: &Chunk) -> anyhow::Result<()> { + if !self.block_hash_index { + return Ok(()); + } + + let Some(label) = self.find_label_for_update(dataset_id)? else { + return Ok(()); // dataset does not exist - nothing to index + }; + if !is_indexed_kind(label.kind()) { + return Ok(()); + } + + let Some(blocks_table_id) = chunk.tables().get("blocks").copied() else { + return Ok(()); // defensively skip chunks without a blocks table + }; + + let snapshot = ReadSnapshot::new(self.db); + let reader = snapshot.create_table_reader(blocks_table_id)?; + let cf = self.cf_handle(CF_BLOCK_HASHES); + for_each_block_hash(&reader, |number, hash| { + self.transaction + .put_cf(cf, BlockHashIndexKey::new(dataset_id, hash), number.to_be_bytes())?; + Ok(()) + }) + } + + /// Removes every hash of `chunk`'s `blocks` table from `CF_BLOCK_HASHES`. + /// + /// Unlike [`Tx::index_block_hashes`], gated on neither the flag nor the + /// dataset kind, but on whether the dataset has any entries at all - + /// entries written while the flag was on must still be removed when their + /// chunk is pruned, or they would be stranded forever. Idempotent over + /// never-indexed chunks: `delete_cf` on a missing key is a no-op. + pub fn unindex_block_hashes(&self, dataset_id: DatasetId, chunk: &Chunk) -> anyhow::Result<()> { + if !self.has_block_hash_entries(dataset_id)? { + return Ok(()); + } + + let Some(blocks_table_id) = chunk.tables().get("blocks").copied() else { + return Ok(()); + }; + + let snapshot = ReadSnapshot::new(self.db); + let reader = snapshot.create_table_reader(blocks_table_id)?; + let cf = self.cf_handle(CF_BLOCK_HASHES); + for_each_block_hash(&reader, |_number, hash| { + self.transaction + .delete_cf(cf, BlockHashIndexKey::new(dataset_id, hash))?; + Ok(()) + }) + } + + /// Whether `dataset_id` holds at least one `CF_BLOCK_HASHES` entry: a + /// single seek. Iterating the transaction (not the bare DB) keeps the + /// answer accurate part-way through a multi-chunk `insert_fork`. + fn has_block_hash_entries(&self, dataset_id: DatasetId) -> anyhow::Result { + let (start, end) = BlockHashIndexKey::dataset_range(dataset_id); + + let mut read_opts = rocksdb::ReadOptions::default(); + read_opts.set_snapshot(&self.transaction.snapshot()); + read_opts.set_iterate_upper_bound(end); + + let mut cursor = self + .transaction + .raw_iterator_cf_opt(self.cf_handle(CF_BLOCK_HASHES), read_opts); + + cursor.seek(&start); + cursor.status()?; + + Ok(cursor.valid()) + } + pub fn insert_fork(&self, dataset_id: DatasetId, chunk: &Chunk) -> anyhow::Result<()> { let existing = self.list_chunks(dataset_id, 0, None).into_reversed(); for head_result in existing { let head = head_result?; if chunk.first_block() <= head.first_block() { + self.unindex_block_hashes(dataset_id, &head)?; self.delete_chunk(dataset_id, &head)?; } else if head.last_block() + 1 == chunk.first_block() { ensure!( @@ -161,6 +260,7 @@ impl<'a> Tx<'a> { } self.write_chunk(dataset_id, chunk)?; + self.index_block_hashes(dataset_id, chunk)?; Ok(()) } diff --git a/crates/storage/tests/block_hash_index.rs b/crates/storage/tests/block_hash_index.rs new file mode 100644 index 00000000..bba4501f --- /dev/null +++ b/crates/storage/tests/block_hash_index.rs @@ -0,0 +1,339 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use arrow::{ + array::{ArrayRef, RecordBatch, StringArray, UInt64Array}, + datatypes::{DataType, Field, Schema} +}; +use sqd_primitives::BlockRef; +use sqd_storage::{ + db::{Chunk, CompactionStatus, Database, DatabaseSettings, DatasetId, DatasetKind}, + table::write::use_small_buffers +}; +use tempfile::TempDir; + +fn open_db_with(kind: &str, block_hash_index: bool) -> (TempDir, Database, DatasetId) { + // The TempDir guard is returned and kept alive for the whole test so the + // on-disk database isn't removed out from under RocksDB. + let db_dir = tempfile::tempdir().unwrap(); + let db = reopen(&db_dir, block_hash_index); + let dataset_id = DatasetId::from_str("test-dataset"); + db.create_dataset(dataset_id, DatasetKind::from_str(kind)).unwrap(); + (db_dir, db, dataset_id) +} + +/// Opens the database at `dir` again, e.g. to simulate a restart with a +/// different `block_hash_index` setting. Any previous `Database` over the same +/// directory must be dropped first - RocksDB holds an exclusive lock on it. +fn reopen(dir: &TempDir, block_hash_index: bool) -> Database { + DatabaseSettings::default() + .with_block_hash_index(block_hash_index) + .open(dir.path()) + .unwrap() +} + +fn open_db(kind: &str) -> (TempDir, Database, DatasetId) { + open_db_with(kind, true) +} + +fn setup_evm_db() -> (TempDir, Database, DatasetId) { + open_db("evm") +} + +/// Canonical hash for a block number. +fn block_hash(n: u64) -> String { + format!("0x{:064x}", n) +} + +/// Builds an EVM-shaped chunk: a `blocks` table with `number` (UInt64) and +/// `hash` (Utf8) columns covering `first..=last`, hashes derived via `hash_fn`. +fn make_evm_chunk_with( + db: &Database, + first: u64, + last: u64, + parent_hash: &str, + hash_fn: impl Fn(u64) -> String +) -> Chunk { + let schema = Arc::new(Schema::new(vec![ + Field::new("number", DataType::UInt64, false), + Field::new("hash", DataType::Utf8, false), + ])); + + let numbers: Vec = (first..=last).collect(); + let hashes: Vec = numbers.iter().map(|n| hash_fn(*n)).collect(); + + let number_arr = Arc::new(UInt64Array::from(numbers)) as ArrayRef; + let hash_arr = Arc::new(StringArray::from( + hashes.iter().map(String::as_str).collect::>() + )) as ArrayRef; + + let mut builder = db.new_table_builder(schema.clone()); + let batch = RecordBatch::try_new(schema, vec![number_arr, hash_arr]).unwrap(); + builder.write_record_batch(&batch).unwrap(); + + let mut tables = BTreeMap::new(); + tables.insert("blocks".to_owned(), builder.finish().unwrap()); + + Chunk::V1 { + first_block: first, + last_block: last, + last_block_hash: hash_fn(last), + parent_block_hash: parent_hash.to_owned(), + first_block_time: None, + last_block_time: None, + tables + } +} + +fn make_evm_chunk(db: &Database, first: u64, last: u64, parent_hash: &str) -> Chunk { + make_evm_chunk_with(db, first, last, parent_hash, block_hash) +} + +fn lookup(db: &Database, dataset_id: DatasetId, hash: &str) -> Option { + db.snapshot().find_block_by_hash(dataset_id, hash).unwrap() +} + +fn assert_resolves(db: &Database, dataset_id: DatasetId, n: u64) { + assert_eq!( + lookup(db, dataset_id, &block_hash(n)), + Some(BlockRef { + number: n, + hash: block_hash(n) + }), + "block {} should resolve via its canonical hash", + n + ); +} + +fn assert_absent(db: &Database, dataset_id: DatasetId, hash: &str) { + assert_eq!(lookup(db, dataset_id, hash), None, "hash {} should not resolve", hash); +} + +#[test] +fn index_ingest_and_lookup() { + let (_dir, db, dataset_id) = setup_evm_db(); + + let chunk = make_evm_chunk(&db, 0, 9, "base"); + db.insert_chunk(dataset_id, &chunk).unwrap(); + + for n in 0..=9 { + assert_resolves(&db, dataset_id, n); + } + assert_absent(&db, dataset_id, "0xdeadbeef"); + assert_absent(&db, dataset_id, &block_hash(10)); +} + +#[test] +fn index_large_chunk_spans_multiple_read_batches() { + // > 4096 rows forces `for_each_block_hash` through more than one batch, + // exercising the offset advancement across the batch boundary. + let (_dir, db, dataset_id) = setup_evm_db(); + + let last = 5000; + let chunk = make_evm_chunk(&db, 0, last, "base"); + db.insert_chunk(dataset_id, &chunk).unwrap(); + + for n in [0, 1, 4095, 4096, 4097, last] { + assert_resolves(&db, dataset_id, n); + } +} + +#[test] +fn index_fork_replaces_hashes() { + let (_dir, db, dataset_id) = setup_evm_db(); + + let chunk1 = make_evm_chunk(&db, 0, 9, "base"); + let chunk2 = make_evm_chunk(&db, 10, 19, &block_hash(9)); + db.insert_chunk(dataset_id, &chunk1).unwrap(); + db.insert_chunk(dataset_id, &chunk2).unwrap(); + + // Fork rewrites blocks 10..=19 with different hashes. + let fork = make_evm_chunk_with(&db, 10, 19, &block_hash(9), |n| format!("fork_{}", n)); + db.insert_fork(dataset_id, &fork).unwrap(); + + // chunk1's hashes are untouched. + for n in 0..=9 { + assert_resolves(&db, dataset_id, n); + } + // old canonical hashes of the forked range are gone, forked ones resolve. + for n in 10..=19 { + assert_absent(&db, dataset_id, &block_hash(n)); + assert_eq!( + lookup(&db, dataset_id, &format!("fork_{}", n)), + Some(BlockRef { + number: n, + hash: format!("fork_{}", n) + }) + ); + } +} + +#[test] +fn index_delete_chunk_removes_hashes() { + // Models the retention path (DatasetUpdate::delete_chunk). + let (_dir, db, dataset_id) = setup_evm_db(); + + let chunk1 = make_evm_chunk(&db, 0, 9, "base"); + let chunk2 = make_evm_chunk(&db, 10, 19, &block_hash(9)); + db.insert_chunk(dataset_id, &chunk1).unwrap(); + db.insert_chunk(dataset_id, &chunk2).unwrap(); + + db.update_dataset(dataset_id, |tx| tx.delete_chunk(&chunk1)).unwrap(); + + for n in 0..=9 { + assert_absent(&db, dataset_id, &block_hash(n)); + } + for n in 10..=19 { + assert_resolves(&db, dataset_id, n); + } +} + +#[test] +fn index_delete_dataset_removes_all_hashes() { + let (_dir, db, dataset_id) = setup_evm_db(); + + let chunk1 = make_evm_chunk(&db, 0, 9, "base"); + let chunk2 = make_evm_chunk(&db, 10, 19, &block_hash(9)); + db.insert_chunk(dataset_id, &chunk1).unwrap(); + db.insert_chunk(dataset_id, &chunk2).unwrap(); + + db.delete_dataset(dataset_id).unwrap(); + + assert!(db.get_all_datasets().unwrap().is_empty()); + for n in 0..=19 { + assert_absent(&db, dataset_id, &block_hash(n)); + } +} + +#[test] +fn index_survives_compaction() { + // Regression guard for the "compaction must not touch the index" decision. + // Many small chunks (>= 50) ensure real merging is triggered. + let (_dir, db, dataset_id) = setup_evm_db(); + let _sb = use_small_buffers(); + + let n_chunks = 60u64; + let blocks_per_chunk = 4u64; + let mut parent = "base".to_owned(); + for c in 0..n_chunks { + let first = c * blocks_per_chunk; + let last = first + blocks_per_chunk - 1; + let chunk = make_evm_chunk(&db, first, last, &parent); + db.insert_chunk(dataset_id, &chunk).unwrap(); + parent = block_hash(last); + } + let total_blocks = n_chunks * blocks_per_chunk; + + for n in 0..total_blocks { + assert_resolves(&db, dataset_id, n); + } + + let mut merged = false; + loop { + match db + .perform_dataset_compaction(dataset_id, Some(100), Some(1.25), None) + .unwrap() + { + CompactionStatus::Ok(_) => merged = true, + _ => break + } + } + assert!(merged, "expected compaction to merge at least once"); + + // Sanity: chunks really were merged (fewer than we inserted). + let chunk_count = db.snapshot().list_chunks(dataset_id, 0, None).count(); + assert!(chunk_count < n_chunks as usize, "compaction should reduce chunk count"); + + // The index is untouched: every hash still resolves to the same number. + for n in 0..total_blocks { + assert_resolves(&db, dataset_id, n); + } +} + +#[test] +fn non_evm_dataset_is_not_indexed() { + // Same EVM-shaped blocks table, but a solana dataset -> nothing is indexed. + let (_dir, db, dataset_id) = open_db("solana"); + + let chunk = make_evm_chunk(&db, 0, 9, "base"); + db.insert_chunk(dataset_id, &chunk).unwrap(); + + for n in 0..=9 { + assert_absent(&db, dataset_id, &block_hash(n)); + } +} + +#[test] +fn index_disabled_writes_nothing() { + // An EVM dataset still isn't indexed while the flag is off. + let (_dir, db, dataset_id) = open_db_with("evm", false); + + let chunk = make_evm_chunk(&db, 0, 9, "base"); + db.insert_chunk(dataset_id, &chunk).unwrap(); + + for n in 0..=9 { + assert_absent(&db, dataset_id, &block_hash(n)); + } + + // Pruning a never-indexed chunk short-circuits on the prefix probe. + db.update_dataset(dataset_id, |tx| tx.delete_chunk(&chunk)).unwrap(); +} + +#[test] +fn index_entries_drain_after_flag_is_turned_off() { + // Guards the asymmetric gating: `index_block_hashes` honours the flag, + // `unindex_block_hashes` does not. Entries written while the flag was on must + // still be reclaimed by retention once it goes off, or they would be stranded. + let (dir, db, dataset_id) = setup_evm_db(); + + let chunk1 = make_evm_chunk(&db, 0, 9, "base"); + let chunk2 = make_evm_chunk(&db, 10, 19, &block_hash(9)); + db.insert_chunk(dataset_id, &chunk1).unwrap(); + db.insert_chunk(dataset_id, &chunk2).unwrap(); + drop(db); + + // Restart with indexing disabled. + let db = reopen(&dir, false); + + // New chunks are no longer indexed... + let chunk3 = make_evm_chunk(&db, 20, 29, &block_hash(19)); + db.insert_chunk(dataset_id, &chunk3).unwrap(); + for n in 20..=29 { + assert_absent(&db, dataset_id, &block_hash(n)); + } + + // ...but pruning still reclaims what the previous run wrote. + db.update_dataset(dataset_id, |tx| tx.delete_chunk(&chunk1)).unwrap(); + for n in 0..=9 { + assert_absent(&db, dataset_id, &block_hash(n)); + } + for n in 10..=19 { + assert_resolves(&db, dataset_id, n); + } + + // Down to the last entry, after which the probe short-circuits. + db.update_dataset(dataset_id, |tx| tx.delete_chunk(&chunk2)).unwrap(); + for n in 10..=19 { + assert_absent(&db, dataset_id, &block_hash(n)); + } + db.update_dataset(dataset_id, |tx| tx.delete_chunk(&chunk3)).unwrap(); +} + +#[test] +fn probe_sees_pending_writes_within_the_same_transaction() { + // Verifies the claim in `has_block_hash_entries`: iterating the transaction + // merges its own uncommitted puts, so a chunk indexed and pruned inside one + // `update_dataset` closure leaves nothing behind, even though the dataset had + // zero committed entries when the probe ran. + let (_dir, db, dataset_id) = setup_evm_db(); + + let chunk = make_evm_chunk(&db, 0, 9, "base"); + db.update_dataset(dataset_id, |tx| { + tx.insert_chunk(&chunk)?; + tx.delete_chunk(&chunk) + }) + .unwrap(); + + for n in 0..=9 { + assert_absent(&db, dataset_id, &block_hash(n)); + } +} From df4f7889767ae3a4154f035ddfd5a97e8958d886 Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Tue, 14 Jul 2026 21:40:45 +0300 Subject: [PATCH 2/3] docs(hotblocks): spec the hash indexes, block and transaction IB-8 requires a route change to land with its binding and CT-5 update, and the block hash index shipped without either. The spec also had no vocabulary for what it introduced: a derived index that is sound but deliberately incomplete, where a hit is authoritative and a miss proves nothing (DEF-17, RP-19, NG7). Without that stated, a lookup miss reads as "not in the window", which it is not. Transaction-by-hash is specified ahead of its implementation (GAP-38) because its fork behaviour is a trap that block hashes hide: a block hash belongs to one branch forever, but a reorged-out transaction is routinely re-included at a new position, so a fork that unindexes after indexing erases the fresh entry (INV-47). It also costs one entry per transaction rather than per block, which is why RS-12 sizes and switches the two indexes independently. Co-Authored-By: Claude Opus 4.8 --- crates/hotblocks/spec/01-overview.md | 13 +++++ crates/hotblocks/spec/02-data-model.md | 22 +++++++++ crates/hotblocks/spec/03-write-path.md | 10 ++++ crates/hotblocks/spec/04-read-path.md | 28 +++++++++++ crates/hotblocks/spec/06-invariants.md | 47 ++++++++++++++++++- .../hotblocks/spec/09-retention-and-space.md | 20 ++++++++ crates/hotblocks/spec/11-observability.md | 7 +++ crates/hotblocks/spec/12-conformance-tdd.md | 33 ++++++++++++- crates/hotblocks/spec/13-interface-binding.md | 15 ++++++ crates/hotblocks/spec/14-parameters.md | 3 ++ 10 files changed, 196 insertions(+), 2 deletions(-) diff --git a/crates/hotblocks/spec/01-overview.md b/crates/hotblocks/spec/01-overview.md index 9d4d5d06..c89d4860 100644 --- a/crates/hotblocks/spec/01-overview.md +++ b/crates/hotblocks/spec/01-overview.md @@ -11,6 +11,12 @@ It is the "hot" complement to an archival store: archives hold the deep, immutab Hotblocks holds the volatile tip where blocks arrive continuously, forks and rollbacks happen, and freshness is measured in fractions of a second. +Alongside range queries it offers **point lookups by hash** — resolving a block hash, or a +transaction hash, to a position in the window — for consumers that hold a bare hash from a +log, an event, or a third party and need a block number to query with. These lookups are +served from optional derived indexes ([DEF-17](02-data-model.md)) and come with a sharp +caveat spelled out in NG7 below. + The system is a single service instance hosting many datasets concurrently (tens of datasets is the normal operating point). Datasets are independent chains: different networks, different chain families ("kinds"), different retention policies. @@ -59,6 +65,13 @@ networks, different chain families ("kinds"), different retention policies. the server ([RP-10](04-read-path.md)). - **NG6 — No exactly-once source consumption.** Source delivery is at-least-once; ingestion is idempotent with respect to redelivery ([WP-16](03-write-path.md)). +- **NG7 — Hash lookups are a convenience, not an oracle.** The hash indexes are optional, + derived, and deliberately incomplete: never backfilled, enabled per deployment, defined + only for kinds that expose the hash. A lookup **hit** is authoritative; a lookup **miss** + is not evidence that the block or transaction is absent from the window + ([RP-19](04-read-path.md)). Making misses meaningful would require backfilling history + the hot store has already accepted without an index — work whose cost scales with the + window, for a guarantee a range query already provides. ## 5. Trust model diff --git a/crates/hotblocks/spec/02-data-model.md b/crates/hotblocks/spec/02-data-model.md index 1bf87f9b..a65a8652 100644 --- a/crates/hotblocks/spec/02-data-model.md +++ b/crates/hotblocks/spec/02-data-model.md @@ -173,6 +173,27 @@ progress ([RP-9](04-read-path.md)). **DEF-15 (Watermark reads).** Point reads returning the current committed `head`, `fin`, retention policy, and dataset status. Defined in [04-read-path.md §6](04-read-path.md). +**DEF-17 (Hash indexes).** Two optional per-dataset **partial** maps over the current state: + +``` +bidx(D) : Hash ⇀ ℕ — block hash → block number +tidx(D) : Hash ⇀ ℕ × ℕ — transaction hash → ⟨block number, transaction index⟩ +``` + +They are **derived**, never primary: everything they name already lives in `seg`, no +transition reads them, and dropping them loses no information. They are **partial by +design** — a hash may be absent from an index while its block sits in `seg`. Absence +therefore says nothing about `seg`; only presence does ([RP-19](04-read-path.md)). The +sources of partiality are structural, not incidental: an index is enabled per deployment +(`P-BLOCK-INDEX` / `P-TX-INDEX`), it is never backfilled, so blocks ingested while it was +off stay unnamed for as long as they remain in the window, and it is defined only for +kinds whose schema exposes the hash (EVM today). + +Within one dataset each index is a function: a block hash names at most one block, a +transaction hash at most one transaction *of the current chain*. Across branches the two +differ — a block hash belongs to exactly one branch forever, while a transaction hash is +routinely re-included at a different position after a reorg ([INV-47](06-invariants.md)). + ## 6. Transitions (summary) The complete write-side vocabulary; semantics in [03-write-path.md](03-write-path.md): @@ -203,4 +224,5 @@ advance arriving together); invariants are evaluated at commit points only. | preceding block / `hash_at(p)` | DEF-16 | | coverage | DEF-14 | | fork hints | `ForkSignal.hints` (DEF-12), also the payload of the CONFLICT error (RP-11) | +| hash index / `bidx`, `tidx` | DEF-17 | | batch | the unit of one `EXTEND`/`REPLACE` commit (bounded by P-BATCH-ROWS / P-BATCH-BYTES) | diff --git a/crates/hotblocks/spec/03-write-path.md b/crates/hotblocks/spec/03-write-path.md index 65cbe4b5..89580957 100644 --- a/crates/hotblocks/spec/03-write-path.md +++ b/crates/hotblocks/spec/03-write-path.md @@ -250,6 +250,16 @@ history cannot be re-acquired through RETAIN. concurrent controller of the same dataset is detected (state changed under the writer's feet), the writer MUST stop mutating that dataset and raise an alarm (FM-OP-3) — both writers continuing is forbidden. +- **WP-20 (Derived index maintenance).** A derived index (DEF-17) is maintained *inside* + the transition that changes what it names, never by a follow-up pass: entries appear and + disappear in the same commit as their blocks (INV-46). A reader therefore can never + observe an index that disagrees with the window it was read from, and a crash can never + land between a block and its entry. Enabling or disabling an index is a deployment + change, not a transition: it decides whether *newly* ingested blocks are named and never + rewrites committed history — which is precisely why the indexes are partial (DEF-17). + Removal is unconditional: entries are dropped with their blocks whether or not the index + is currently enabled, so a disabled index drains within one retention period rather than + rotting into stale entries. ## 4. Ingestion error handling diff --git a/crates/hotblocks/spec/04-read-path.md b/crates/hotblocks/spec/04-read-path.md index 7f24bda6..5485a581 100644 --- a/crates/hotblocks/spec/04-read-path.md +++ b/crates/hotblocks/spec/04-read-path.md @@ -13,6 +13,8 @@ mapping lives in [13-interface-binding.md](13-interface-binding.md). | `HEAD` | dataset | `head(D)` Ref or "none" | | `FINALIZED-HEAD` | dataset | `fin(D)` Ref or "none" | | `STATUS` | dataset | kind, retention policy, `first`, `head` (+hash, time), `fin` | +| `BLOCK-BY-HASH` | dataset, hash | the Ref of the block with that hash, or "none" (§6.1) | +| `TX-BY-HASH` | dataset, hash | `⟨block number, transaction index⟩` + the hash, or "none" (§6.1) | | `GET-RETENTION` / `SET-RETENTION` | dataset (+policy) | current policy / acceptance | ## 2. Query admission @@ -130,6 +132,31 @@ For coverage `[from, L]` against snapshot `S`: - **RP-14 (Status).** STATUS reports a consistent view: kind, retention policy, `first`, `head` (number, hash, time when known), `fin` — all from one snapshot. +### 6.1 Hash lookups + +`BLOCK-BY-HASH` and `TX-BY-HASH` answer from `bidx`/`tidx` (DEF-17) of one snapshot `S`. + +- **RP-19 (Sound, not complete).** The two directions of a lookup carry very different + weight, and clients MUST treat them differently: + - a **hit** is true of `S` and as trustworthy as a range query: the named block is in + `seg(S)` carrying exactly that hash, and the named transaction sits in that block at + that index (INV-45). A hit is never stale, never orphaned, never from a replaced + branch; + - a **miss** proves *nothing*. The block or transaction MAY be sitting in the window + unindexed — the index was disabled when it was ingested, the dataset predates it, or + the kind is not covered (DEF-17). Clients MUST NOT read "none" as "not in the window"; + the authoritative absence test remains a range query. + + A lookup against a dataset whose index is disabled behaves as a lookup against an empty + index — "none", not an error. This is what makes the miss direction uninformative *by + construction* rather than by accident, and it is the whole price of not backfilling + (NG7). +- **RP-20 (Bounded and non-competing).** A hash argument longer than `P-HASH-MAXLEN` (or + empty) MUST be rejected as `MALFORMED_REQUEST` before the store is touched. Lookups are + point reads: their cost MUST NOT scale with the window, and they MUST NOT consume the + query execution or waiter budgets (`P-EXEC-SLOTS`, `P-WAITERS`) — a lookup flood may not + starve range queries, and a query flood may not stall lookups (PF-4). + ## 7. Fork handling — the CONFLICT protocol - **RP-11 (Anchored queries).** When `expected_parent ≠ ⊥`, it asserts the hash of @@ -184,6 +211,7 @@ stable API surface. An error response carries no block data. | `RANGE_UNAVAILABLE` | `from < first` (window moved past it) | no — re-anchor upward | | `ITEM_UNAVAILABLE` | selection touches an absent item collection (RP-8) | no | | `NO_DATA` | `from` above watermark after bounded wait (RP-5/6) | yes — poll | +| `NOT_FOUND` | hash lookup with no index entry (RP-19 — **not** proof of absence) | no | | `CONFLICT` | anchored-ancestry mismatch (RP-11) | yes — after re-anchoring | | `OVERLOADED` | admission control (RP-3) / waiter cap | yes — backoff | | `FORBIDDEN` | SET-RETENTION on a non-External dataset | no | diff --git a/crates/hotblocks/spec/06-invariants.md b/crates/hotblocks/spec/06-invariants.md index 81fad3d4..32ff3ed7 100644 --- a/crates/hotblocks/spec/06-invariants.md +++ b/crates/hotblocks/spec/06-invariants.md @@ -256,6 +256,50 @@ eat committed blocks; a system crash may only rewind to a committed state within --- +## G. Derived indexes + +The hash indexes (DEF-17) are optional and deliberately incomplete, so there is exactly one +thing they must never do: lie. The catalog below is what "never lies" decomposes into. + +**INV-45 — Index soundness.** [state] +Every entry of `bidx(D)` / `tidx(D)` is true of the state it is read from: `bidx(h) = n` +implies `seg` holds a block numbered `n` whose hash is `h`; `tidx(h) = ⟨n, i⟩` implies that +block holds a transaction at index `i` with hash `h`. No entry names a block outside the +window, a replaced branch, or nothing at all. +*Why:* an index that may lie is worse than no index — a client cannot cheaply re-verify a +hit, so a false positive propagates silently. Partiality is a documented cost (RP-19); +unsoundness is a defect. +*Check:* CT-1 — after every commit, every hash of every block (and transaction) removed by +a fork or a trim MUST resolve to "none", and every hash still in the window MUST resolve to +its true position or to "none", never to a wrong one. CT-2 repeats it across restarts. + +**INV-46 — Index maintenance is part of the transition.** [transition] +Every transition that removes a block from `seg` (REPLACE, RETAIN, RESET, DROP) removes +that block's entry — and the entries of its transactions — in the *same* commit (WP-20). +Transitions that preserve `seg` while reorganizing storage (INV-17) preserve the indexes +exactly. No committed state, and no recovered state, contains an entry for a block outside +its window. +*Why:* this is what makes INV-45 survive churn, and it is the cheapest thing in the system +to get wrong — one delete path that forgets its entries leaves an index that lies forever, +and the lie is invisible until a client trusts it. +*Check:* CT-1 fork + trim churn; CT-7 soak (entry count tracks window size with no upward +drift); CT-2 crash during a trim. + +**INV-47 — Fork re-inclusion names the new position.** [transition] +If a REPLACE removes a transaction and the new branch re-includes it at a different block +or index, then after the commit `tidx` resolves its hash to the **new** position — never to +the old one, and never to "none". +*Why:* block hashes and transaction hashes behave differently across a fork, and the +difference is a trap. A block hash belongs to one branch forever, so a fork can remove and +add entries in any order. A transaction hash is routinely re-included on the winning branch +— so a fork that removes the old entries *after* adding the new ones erases exactly the +entry a client is about to ask for, and the resulting "none" is indistinguishable from the +legitimate partiality of RP-19. Removal must precede insertion within the commit. +*Check:* CT-4 — reorg script that re-includes a transaction at a new block/index; assert +the lookup names the new position (not "none", not the old position). + +--- + ## Reading the catalog in tests A minimal harness assertion set that touches most of the catalog on every step: @@ -265,4 +309,5 @@ A minimal harness assertion set that touches most of the catalog on every step: 2. Every response through validators ⇒ INV-20..27. 3. Model diff at quiescence ⇒ INV-7, 10, 11, 16, 17, 35, 44. 4. Kill/restart cycles ⇒ INV-40, 42, 43. -5. Fork/finality corpora ⇒ INV-12, 13, 14, 23, 24, 31, 36. +5. Fork/finality corpora ⇒ INV-12, 13, 14, 23, 24, 31, 36, 47. +6. Hash lookups of everything the window gained and lost, after every commit ⇒ INV-45, 46. diff --git a/crates/hotblocks/spec/09-retention-and-space.md b/crates/hotblocks/spec/09-retention-and-space.md index e6eb9369..fc581a1e 100644 --- a/crates/hotblocks/spec/09-retention-and-space.md +++ b/crates/hotblocks/spec/09-retention-and-space.md @@ -76,6 +76,21 @@ Requirements: not O(bytes); physical reclamation runs at bounded amortized cost without violating LIV-2 (deletion-induced maintenance debt counts inside the stall budget). Deleting a large dataset MUST have bounded peak memory (not proportional to the dataset's size). +- **RS-12 (Derived index space).** Index bytes (DEF-17) count toward `live_bytes` and fall + under RS-6 like any other bytes: enabling an index raises the denominator, so the + amplification bound neither loosens nor tightens. Entries are removed in the same commit + as their blocks (INV-46), so their space becomes ordinary `debt_bytes` and converges per + RS-5/LIV-7 — **an index is never a leak path**, in either flag direction: enabling one + does not backfill existing blocks, disabling one does not eagerly erase entries, and both + states converge within one retention period as the window turns over. + + Sizing is where the two indexes part company, and operators MUST budget them separately. + `bidx` costs one entry per *block*. `tidx` costs one entry per *transaction* — on a busy + EVM chain roughly two orders of magnitude more, so its footprint tracks the transaction + rate × retention, not the block rate. A retention window that makes `bidx` a rounding + error can make `tidx` the largest single consumer in the store. This asymmetry is why the + two are independently enabled (`P-BLOCK-INDEX`, `P-TX-INDEX`) rather than sharing a + switch. ## 3. Interactions @@ -88,5 +103,10 @@ Requirements: the *next* query below the new `first` gets `RANGE_UNAVAILABLE` (RP-4). - **Retention × recovery:** recovered state reflects committed trims exactly (INV-40); a trim's anchor carry-over (INV-18) survives restarts. +- **Retention × hash indexes:** retention is what bounds an index (RS-12) *and* what makes + it forget — a hash resolvable today stops resolving once its block leaves the window, and + a client cannot distinguish that from a hash that was never indexed (RP-19). Retention is + also the only mechanism that repairs an index: history missed while the flag was off + drains out on its own. - **Space × liveness:** reclamation lag is bounded (LIV-7); maintenance debt feeds back into the write path only within the stall budget (LIV-2, HZ-2/HZ-5). diff --git a/crates/hotblocks/spec/11-observability.md b/crates/hotblocks/spec/11-observability.md index 2b4ac459..0596c2c4 100644 --- a/crates/hotblocks/spec/11-observability.md +++ b/crates/hotblocks/spec/11-observability.md @@ -50,6 +50,13 @@ surface of the binding (13 §5) with bounded cardinality. engine state) sufficient to attribute the stall post-hoc. Rationale: the observed production freezes could not be root-caused from standard metrics; capture-on-stall turns the next occurrence into data (GAP-1). +- **OB-12 (Hash index state).** Per dataset × index (DEF-17): whether it is enabled, its + entry count and estimated bytes (feeding OB-6's live/debt accounting — RS-12), and lookup + counts split by outcome (hit / miss) with latency. Rationale: because a miss is + indistinguishable from a genuine absence by design (RP-19), the miss *rate* is the only + signal an operator has. A rate near 1 is the signature of an index that is empty for a + structural reason — flag switched on after the window had filled, unsupported kind — and + without this signal it looks exactly like a chain on which nobody queries real hashes. ## 2. Property → observable mapping diff --git a/crates/hotblocks/spec/12-conformance-tdd.md b/crates/hotblocks/spec/12-conformance-tdd.md index 097f30c9..810e0e98 100644 --- a/crates/hotblocks/spec/12-conformance-tdd.md +++ b/crates/hotblocks/spec/12-conformance-tdd.md @@ -74,6 +74,10 @@ model Dataset: hash_at(p) = b.hash for the highest b in seg with b.number <= p, else anchor.hash if p >= anchor.number else undefined # DEF-16 + bidx(h) = b.number for the b in seg with b.hash == h, else ⊥ # DEF-17 + tidx(h) = (b.number, i) for the i-th tx of the b in seg + with tx.hash == h, else ⊥ + wf(): # well-formedness — assert after every transition (INV-1..6) assert ascending_and_linked(seg) # INV-1/2: parent_number + parent_hash assert anchor.number < first() if seg @@ -173,6 +177,25 @@ valid `L`; matching blocks and their item content equal to the model evaluated * `L`*; the coverage-end record present; every extra record being a header-only true block of the snapshot inside coverage. Everything else must match exactly. +`bidx`/`tidx` are the one place the model is **stronger than the contract**, and the +comparator must not confuse the two. The model is always complete over `seg`; the SUT is +only required to be *sound* (RP-19), because an index is not backfilled and may be disabled. +So the comparator asserts, unconditionally: + +- every SUT **hit** matches the model exactly (a hit for a hash the model does not hold, or + at a position the model disagrees with, is a hard failure — INV-45); +- every hash the model has **dropped** (forked away, trimmed, dropped) resolves to "none" + (INV-46) — this is the assertion that catches a stale index, and it is the one that + matters; +- a transaction re-included by a fork resolves to its **new** position (INV-47). + +Full completeness — *every* block of `seg` resolves — may be asserted only by runs that +guarantee the preconditions themselves: a fresh store, the index enabled for the whole run, +and an indexed kind. Every script today satisfies all three, so +`Harness::assert_hash_index_conforms` asserts completeness legitimately; the moment a script +restarts the SUT with the flag flipped, or seeds a pre-existing store, that assertion must +weaken to soundness or it will fail on correct behavior. + ## 3. Test-class taxonomy | CT | Name | Method | Primary properties | @@ -181,7 +204,7 @@ of the snapshot inside coverage. Everything else must match exactly. | CT-2 | **Crash-recovery** | kill-point matrix (during batch write, during fork, during trim, during boot, during shutdown) × restart → model diff; repeated-crash convergence | INV-40, 42, 43; CN-6/9/11; LIV-5/6/12; GAP-2 | | CT-3 | **Concurrency** | reader swarms hammering during write/fork/trim/maintenance storms; interleaved HEAD+QUERY sequencing checks | INV-20/21/23/31/41; CN-3/4; LIV-3/4 | | CT-4 | **Source-fault corpus** | scripted FM-SRC-1..8 scenarios incl. fork storms, deep forks, finality conflicts, equivocation | INV-12/13/14/23/24; WP-6/8; LIV-9; FM-SRC-*; GAP-3/4/5 | -| CT-5 | **Interface conformance** | exhaustive request/response matrix against the binding: error taxonomy, watermark headers, encodings, boot config matrix | RP-1..16; INV-26/43; IB-*; GAP-8/9/11 | +| CT-5 | **Interface conformance** | exhaustive request/response matrix against the binding: error taxonomy, watermark headers, encodings, hash-lookup matrix, boot config matrix | RP-1..16, 19/20; INV-26/43; IB-*; GAP-8/9/11/38/39 | | CT-6 | **Performance benchmarks** | reference scenarios S1–S6; SLI capture; SLO gates; saturation knees | SLI-1..12; PF-1..9; LIV-1/3/10; GAP-13 | | CT-7 | **Soak / endurance** | multi-day S4 churn with fault sprinkling; space, memory, stall, residue tracking | LIV-2/7/11; RS-6/10; INV-16/17; HZ-2/5; GAP-1/6 | | CT-8 | **Isolation / noisy neighbor** | S6: one dataset saturated/faulted, others measured differentially | INV-35/36; LIV-8; PF-4; GAP-14 | @@ -238,6 +261,10 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | INV-42 residue convergence | CT-2/7 | P | synthetic orphan purge tested; no crash-driven test | | INV-43 boot validation | CT-5 | U | | | INV-44 explicit destruction | CT-1/2 | U | | +| INV-45 index soundness | CT-1/2 | **P** | `block_hash_index.rs`: hits match the model on ingest, unknown hashes miss, the replaced branch stops resolving. No crash or trim coverage; `tidx` unbuilt (GAP-38) | +| INV-46 index maintenance | CT-1/4/7 | **P** | the fork path is covered black-box; trim / DROP / compaction paths have storage-level tests only (`crates/storage/tests/block_hash_index.rs`), never through the binding | +| INV-47 fork re-inclusion | CT-4 | **U** | untestable until `tidx` exists (GAP-38); the hazard is invisible on block hashes | +| RP-19/20 lookup contract | CT-5 | **P** | hit/miss shapes pinned; the length cap, the disabled-index case and the NOT_FOUND vs UNKNOWN_DATASET split (GAP-39) are unasserted | | LIV-1/2 progress/stall | CT-6/7 | **U — known-violated** | GAP-1 | | LIV-3 query termination | CT-3/6 | U | | | LIV-4 waiter termination | CT-1/3 | P | `ct1_happy_path`: a query above the head answers `NO_DATA` within `P-HEAD-WAIT` | @@ -259,6 +286,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | SLI-1..12 / PF-* | CT-6 | U | no benchmark harness exists | | OB-1 chain gauges | CT-1 | P | `first_block` / `last_block` / `last_finalized_block` diffed against the model; commit version and retention policy not exported (GAP-34) | | OB-2..11 | all | P | query metrics exist; stall gauges pending on PR #83 (unmerged); OB-2 heartbeat, OB-6 debt accounting, OB-9 alarms, OB-11 forensics absent | +| OB-12 index state | CT-1 | **U** | nothing exported: no entry count, no bytes, no hit/miss (GAP-40) | ## 6. Gap register (dated 2026-07-12, informative) @@ -305,6 +333,9 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-35 | The runtime External instruction `"None"` parks the dataset: the controller maps it to Idle and stops ingestion even on a non-empty dataset (`RetentionStrategy::None → State::Idle`, dataset_controller.rs), while the binding documents `"None"` as Unbounded (13 §6) and WP-5 requires a non-empty dataset to keep ingesting from its window. Whether an External instruction may change the policy *mode* at all is unspecified (WP-11) | WP-5, DEF-9, WP-11, IB §6 | P2 | CT-1/CT-5: SET-RETENTION `"None"` on a non-empty External dataset; assert ingestion continues (or the instruction is refused with a defined error) — the head must keep advancing | | GAP-36 | `MALFORMED_REQUEST`, `RANGE_UNAVAILABLE`, `ITEM_UNAVAILABLE` and `KIND_MISMATCH` all surface as HTTP 400 with a free-text body (`api.rs error_to_response`); no machine-readable discriminant exists and IB-7 forbids keying on text — clients cannot distinguish "re-anchor upward" from "fix the request", and the CT-5 error matrix cannot verify INV-26 at the binding. 13 §5 marks the discrimination REQUIRED; the structured error body is the missing piece | INV-26, IB-7, 04 §8 | P2 | CT-5: trigger each 400 class; assert a structured field distinguishes them | | GAP-37 | PF-1's memory ceiling is not configuration-derivable on the read side: INV-25/RP-17 require emitting the first covered block whole even above `P-RESP-WEIGHT`, and nothing bounds a single block at ingest (`P-BATCH-BYTES` is a soft *batch* bound — one oversized block still stores), so per-response memory is bounded only by the largest block a source ever served. No `P-MAX-BLOCK-BYTES` exists | PF-1, RP-17/INV-25, FM-CLI-2 | P2 | CT-6/CT-9: ingest a pathological giant block, query it; assert bounded RSS and whole-block emission (INV-25) | +| GAP-38 | `TX-BY-HASH` is specified (DEF-17, RP-19, IB §2) but not implemented — `tidx` does not exist. A consumer holding a bare transaction hash, which is the common case for anything reading logs or third-party feeds, has no way to reach a block number. Building it is not a variant of `bidx`: it costs one entry per *transaction* (RS-12 sizing) and it is the only index where the fork-ordering hazard of INV-47 can fire, since transaction hashes recur across branches and block hashes never do | DEF-17, RP-19, INV-47, IB §2 | P2 | CT-5: GET the transaction route (404 route-not-found today); then CT-4 re-inclusion per INV-47 | +| GAP-39 | A hash-lookup miss and an unknown dataset are both 404 with only a free-text body between them, and IB-7 forbids keying on text. This is worse than the GAP-36 family it belongs to: RP-19 makes "this hash is not indexed" a *deliberately uninformative* answer, so a client that cannot separate it from "this dataset does not exist" cannot tell a misconfiguration from a legitimate miss at all | INV-26, IB-7, RP-19 | P3 | CT-5: unknown dataset vs unknown hash; assert a structured discriminant | +| GAP-40 | The hash indexes export nothing (OB-12): no entry count, no bytes toward OB-6/RS-12, no hit/miss rate. Because a miss is uninformative by design, an index that is empty for a structural reason — enabled after the window had filled, wrong kind — is indistinguishable in production from one nobody queries. The `--block-hash-index` flag can be on, the endpoint can 404 every request, and no signal says so | OB-12, OB-6 | P3 | CT-1: scrape assertion once exported | ### 6.1 Closed diff --git a/crates/hotblocks/spec/13-interface-binding.md b/crates/hotblocks/spec/13-interface-binding.md index 9a1314be..f45de0f1 100644 --- a/crates/hotblocks/spec/13-interface-binding.md +++ b/crates/hotblocks/spec/13-interface-binding.md @@ -28,6 +28,8 @@ annotated with the relevant GAP. | HEAD | `GET /datasets/{id}/head` | `{"number":N,"hash":"…"}` or `null` | | FINALIZED-HEAD | `GET /datasets/{id}/finalized-head` | same shape | | STATUS | `GET /datasets/{id}/status` | kind, retention, first/last block (+hash/time), finalized head | +| BLOCK-BY-HASH | `GET /datasets/{id}/hashes/{hash}/block` | `{"number":N,"hash":"…"}`; miss = `NOT_FOUND`, **not** proof of absence (RP-19) | +| TX-BY-HASH | `GET /datasets/{id}/hashes/{hash}/transaction` | `{"blockNumber":N,"transactionIndex":i,"hash":"…"}` — **not implemented** (GAP-38) | | METADATA | `GET /datasets/{id}/metadata` | start block, real-time flag, aliases | | GET-RETENTION | `GET /datasets/{id}/retention` | current policy JSON | | SET-RETENTION | `POST /datasets/{id}/retention` | policy JSON; only for `External` datasets, else `FORBIDDEN` (403) | @@ -93,6 +95,7 @@ below" never applies below genesis). | schema-invalid body | 422 | | `FORBIDDEN` | 403 | | `UNKNOWN_DATASET` | 404 | +| `NOT_FOUND` (hash lookup miss) | 404 — indistinguishable from `UNKNOWN_DATASET` except by free-text body (GAP-39) | | `CONFLICT` | 409, body `{"previousBlocks":[{"number":…,"hash":"…"},…]}` = RP-11 hints (ascending; ≥ 1 entry; up to ~`P-CONFLICT-WINDOW`; entries are `⟨position, hash_at(position)⟩` pairs — DEF-16) | | `OVERLOADED` | 503 | | `INTERNAL` | 500 | @@ -100,6 +103,18 @@ below" never applies below genesis). - **IB-7** Error bodies MUST NOT leak internals in a way clients must parse; conformance tests key on status + the structured fields named above only. +## 5a. Hash lookup binding + +- **IB-10** `{hash}` is a path segment carrying the hash exactly as it appears in the + dataset's own data — no normalization, no case folding, no `0x` handling: the service + compares the string it is given against the string it stored. A client that hashes + differently than its chain does simply misses. Length is bounded by `P-HASH-MAXLEN`; + empty or longer is `MALFORMED_REQUEST` (400) decided before any store access (RP-20). +- **IB-11** A miss and a disabled index are the same response (404 `NOT_FOUND`). The + binding deliberately exposes **no** way to ask "is this index enabled / complete?" — + RP-19 forbids a client from acting on that difference anyway; operators read it from + OB-12 instead. + ## 6. Retention policy JSON `{"FromBlock":{"number":N,"parent_hash":"…"?}}` | `{"Head":N}` | `"None"` — mapping to diff --git a/crates/hotblocks/spec/14-parameters.md b/crates/hotblocks/spec/14-parameters.md index 447a9116..cd0335b3 100644 --- a/crates/hotblocks/spec/14-parameters.md +++ b/crates/hotblocks/spec/14-parameters.md @@ -20,6 +20,7 @@ ran against. | `P-EXEC-SLOTS` | global concurrent query work units (RP-3, PF-3) | executor threads × 200 | keep; revisit per-dataset fairness (GAP-14) | | `P-WAITERS` | global cap on head-waiting queries (RP-5) | 64 000 | keep; same fairness note | | `P-SCHED-SLACK` | scheduling tolerance added to termination bounds (LIV-3/4) | — | 1 s ⚠ | +| `P-HASH-MAXLEN` | max accepted hash length on a lookup, rejected before store access (RP-20) | 256 chars | keep | ## Write path @@ -47,6 +48,8 @@ ran against. | `P-SPACE-CONST` | fixed overhead allowance (RS-6) | — | size per deployment ⚠ | | `P-RECLAIM-LAG` | logical delete → physical space convergence (LIV-7) | sweep ≤ 10 s + compaction (typically minutes–hours); ≤ 7 d worst case via periodic compaction; interrupted-build residue: ∞ in default config (GAP-6) | ≤ 24 h ⚠ | | `P-DISK-FLOOR` | free-disk alarm/degrade threshold (FM-STOR-2) | — | define ⚠ | +| `P-BLOCK-INDEX` | block hash index enabled (DEF-17, RS-12) | off by default (`--block-hash-index`); EVM only | keep | +| `P-TX-INDEX` | transaction hash index enabled (DEF-17, RS-12) | absent — not implemented (GAP-38) | define ⚠; must stay independent of `P-BLOCK-INDEX` (RS-12 sizing asymmetry) | ## Liveness, durability, lifecycle From 3fd246c88a297885a2f5157f4a3bd98c1d994250 Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Tue, 14 Jul 2026 23:20:45 +0300 Subject: [PATCH 3/3] ci: cut debuginfo, unthrottle jobs, drop the redundant LFS fetch Cargo's dev/test default is debug = 2 -- full DWARF is most of a ~280MB test binary and makes linking the slowest step, while CI only ever reads file:line out of a backtrace. --jobs 2 capped rustc and, through NUM_JOBS, the RocksDB C++ build as well, on a host with 8 cores. The LFS step re-fetched what checkout's `lfs: true` had already pulled, and `--all` pulls objects for every ref, so .git/lfs grew without bound on a runner whose workspace persists. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/tests.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2b76294e..d82dad88 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,6 +11,12 @@ on: env: CARGO_TERM_COLOR: always + # Cargo's default for dev/test is debug = 2 -- full DWARF, which is most of a ~280MB test + # binary and makes linking the slowest part of the job. Line tables keep file:line in panic + # backtraces, which is all CI reads. Set here rather than in [profile.*] so local debuggers + # keep full info. + CARGO_PROFILE_DEV_DEBUG: line-tables-only + CARGO_PROFILE_TEST_DEBUG: line-tables-only concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.ref }} @@ -65,7 +71,7 @@ jobs: with: shared-key: rust-ci-test cache-on-failure: true - - name: Fetch LFS files - run: git lfs fetch --all && git lfs checkout + # 8 cores, shared with the second runner -- which picks up this workflow's clippy job. + # cc(1) inherits NUM_JOBS, so this caps the RocksDB C++ build too. - name: Run tests - run: cargo test --all-features --jobs 2 + run: cargo test --all-features --jobs 4