Integrate signature capture and batch evaluation.

Makefile.am

@@ -54,6 +54,7 @@ src_libbitcoin_node_la_SOURCES = \
     src/chasers/chaser_template.cpp \
     src/chasers/chaser_transaction.cpp \
     src/chasers/chaser_validate.cpp \
+    src/chasers/chaser_validate_batch.cpp \
     src/messages/block.cpp \
     src/messages/transaction.cpp \
     src/protocols/protocol.cpp \

builds/msvc/vs2022/libbitcoin-node/libbitcoin-node.vcxproj

@@ -135,6 +135,7 @@
     <ClCompile Include="..\..\..\..\src\chasers\chaser_template.cpp" />
     <ClCompile Include="..\..\..\..\src\chasers\chaser_transaction.cpp" />
     <ClCompile Include="..\..\..\..\src\chasers\chaser_validate.cpp" />
+    <ClCompile Include="..\..\..\..\src\chasers\chaser_validate_batch.cpp" />
     <ClCompile Include="..\..\..\..\src\configuration.cpp" />
     <ClCompile Include="..\..\..\..\src\error.cpp" />
     <ClCompile Include="..\..\..\..\src\estimator.cpp" />

builds/msvc/vs2022/libbitcoin-node/libbitcoin-node.vcxproj.filters

@@ -105,6 +105,9 @@
     <ClCompile Include="..\..\..\..\src\chasers\chaser_validate.cpp">
       <Filter>src\chasers</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\chasers\chaser_validate_batch.cpp">
+      <Filter>src\chasers</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\configuration.cpp">
       <Filter>src</Filter>
     </ClCompile>

builds/msvc/vs2026/libbitcoin-node/libbitcoin-node.vcxproj

@@ -135,6 +135,7 @@
     <ClCompile Include="..\..\..\..\src\chasers\chaser_template.cpp" />
     <ClCompile Include="..\..\..\..\src\chasers\chaser_transaction.cpp" />
     <ClCompile Include="..\..\..\..\src\chasers\chaser_validate.cpp" />
+    <ClCompile Include="..\..\..\..\src\chasers\chaser_validate_batch.cpp" />
     <ClCompile Include="..\..\..\..\src\configuration.cpp" />
     <ClCompile Include="..\..\..\..\src\error.cpp" />
     <ClCompile Include="..\..\..\..\src\estimator.cpp" />

builds/msvc/vs2026/libbitcoin-node/libbitcoin-node.vcxproj.filters

@@ -105,6 +105,9 @@
     <ClCompile Include="..\..\..\..\src\chasers\chaser_validate.cpp">
       <Filter>src\chasers</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\chasers\chaser_validate_batch.cpp">
+      <Filter>src\chasers</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\configuration.cpp">
       <Filter>src</Filter>
     </ClCompile>

include/bitcoin/node/chase.hpp

@@ -96,6 +96,10 @@ enum class chase
     /// Issued by 'organize' and handled by 'check', 'validate', 'confirm'.
     disorganized,
 
+    /// Download concurrency window completed, advancing to next (height_t).
+    /// Issued by 'check' and handled by 'validate'.
+    advanced,
+
     /// Check/Identify.
     /// -----------------------------------------------------------------------
 

include/bitcoin/node/chasers/chaser_confirm.hpp

@@ -58,7 +58,9 @@ class BCN_API chaser_confirm
     virtual bool confirm_block(const header_link& link,
         size_t height, const header_links& popped, size_t fork_point) NOEXCEPT;
     virtual bool complete_block(const code& ec, const header_link& link,
-        size_t height, bool bypassed) NOEXCEPT;
+        size_t height, bool bypass) NOEXCEPT;
+    virtual bool notify_block(const code& ec, size_t height,
+        const database::header_link& link, bool bypass) NOEXCEPT;
 
 private:
     bool set_reorganized(const header_link& link,
@@ -68,9 +70,6 @@ class BCN_API chaser_confirm
     bool roll_back(const header_links& popped, size_t fork_point,
         size_t top) NOEXCEPT;
     void announce(const header_link& link, height_t height) NOEXCEPT;
-
-    // This is thread safe.
-    const bool defer_;
 };
 
 } // namespace node

include/bitcoin/node/chasers/chaser_validate.hpp

@@ -20,6 +20,7 @@
 #define LIBBITCOIN_NODE_CHASERS_CHASER_VALIDATE_HPP
 
 #include <atomic>
+#include <mutex>
 #include <bitcoin/node/chasers/chaser.hpp>
 #include <bitcoin/node/define.hpp>
 
@@ -42,6 +43,9 @@ class BCN_API chaser_validate
     void stop() NOEXCEPT override;
 
 protected:
+    using signatures = system::chain::signatures;
+    using race = network::race_unity<const code&, const database::tx_link&>;
+
     /// Post a method in base or derived class in parallel (use PARALLEL).
     template <class Derived, typename Method, typename... Args>
     inline auto parallel(Method&& method, Args&&... args) NOEXCEPT
@@ -50,45 +54,54 @@ class BCN_API chaser_validate
             BIND_TO(method, args));
     }
 
-    typedef network::race_unity<const code&, const database::tx_link&> race;
-
     virtual bool handle_chase(const code& ec, chase event_,
         event_value value) NOEXCEPT;
 
     virtual void do_regressed(height_t branch_point) NOEXCEPT;
+    virtual void do_advanced(height_t height) NOEXCEPT;
     virtual void do_checked(height_t height) NOEXCEPT;
     virtual void do_bumped(height_t height) NOEXCEPT;
     virtual void do_bump(height_t height) NOEXCEPT;
 
+    /// Validation.
     virtual void post_block(const database::header_link& link,
         bool bypass) NOEXCEPT;
     virtual void validate_block(const database::header_link& link,
         bool bypass) NOEXCEPT;
-    virtual code validate(bool bypass, const system::chain::block& block,
-        const database::header_link& link,
+    virtual code validate(bool& batched, bool& faulted, bool bypass,
+        const system::chain::block& block, const database::header_link& link,
         const system::chain::context& ctx) NOEXCEPT;
     virtual code populate(bool bypass, const system::chain::block& block,
         const system::chain::context& ctx) NOEXCEPT;
     virtual void complete_block(const code& ec,
-        const database::header_link& link, size_t height,
-        bool bypassed) NOEXCEPT;
+        const database::header_link& link, size_t height, bool bypass,
+        bool batched=false, bool faulted=false) NOEXCEPT;
+    virtual void notify_block(const code& ec, size_t height,
+        const database::header_link& link, bool bypass) NOEXCEPT;
+
+    /// Batching.
+    virtual code start_batch() NOEXCEPT;
+    virtual void process_batch() NOEXCEPT;
+    virtual void push_batch(const database::header_link& link) NOEXCEPT;
+    virtual signatures get_capture(const database::header_link& link) NOEXCEPT;
 
     // Override base class strand because it sits on the network thread pool.
     network::asio::strand& strand() NOEXCEPT override;
     bool stranded() const NOEXCEPT override;
 
 private:
+    static constexpr auto relaxed = std::memory_order_relaxed;
+    using shared_lock = const std::shared_lock<std::shared_mutex>;
+    using shared_lock_cptr = std::shared_ptr<shared_lock>;
     using atomic_counter = std::atomic<size_t>;
     using atomic_counter_ptr = std::shared_ptr<atomic_counter>;
-    using signatures = system::chain::signatures;
     using threshold_group = signatures::threshold_group;
     using missed = signatures::miss;
 
-    signatures get_capture(const database::header_link& link) NOEXCEPT;
-
-    // Handlers.
+    // Capture handlers.
     void do_log(const system::chain::script& missed) NOEXCEPT;
-    void do_fire(missed miss, size_t count) NOEXCEPT;
+    void do_fire(missed miss, size_t count,
+        const shared_lock_cptr& lock) NOEXCEPT;
     bool do_ecdsa(const system::hash_digest& digest,
         const system::ec_compressed& point, const system::ec_signature& sign,
         const database::header_link& link) NOEXCEPT;
@@ -98,20 +111,25 @@ class BCN_API chaser_validate
     bool do_multisig(const system::hash_digest& digest,
         const system::ec_compresseds& points,
         const system::ec_signatures& signs, const database::header_link& link,
-        const atomic_counter_ptr& id) NOEXCEPT;
+        const atomic_counter_ptr& sequence) NOEXCEPT;
     bool do_threshold(const threshold_group& group,
         const database::header_link& link,
-        const atomic_counter_ptr& id) NOEXCEPT;
+        const atomic_counter_ptr& sequence) NOEXCEPT;
 
+    // Capture helpers.
     void log_capture(const std::string_view& name,
         size_t captured, size_t missed) const NOEXCEPT;
     void log_captures() const NOEXCEPT;
 
-    // This is protected by strand.
+    // These are protected by strand.
+    database::header_links batched_{};
     network::threadpool validation_threadpool_;
 
     // These are thread safe.
 
+    // This prevents table updates during batch verify.
+    std::shared_mutex mutex_{};
+
     std::atomic<size_t> ecdsa_{};
     std::atomic<size_t> schnorr_{};
     std::atomic<size_t> multisig_{};
@@ -129,7 +147,6 @@ class BCN_API chaser_validate
     const size_t maximum_backlog_;
     const bool batch_signatures_;
     const bool node_witness_;
-    const bool defer_;
     const bool filter_;
 };
 

include/bitcoin/node/error.hpp

@@ -108,7 +108,15 @@ enum error_t : uint8_t
     estimates_push2,
     estimates_pop1,
     estimates_pop2,
-    capture_fault
+    batch1,
+    batch2,
+    batch3,
+    batch4,
+    batch5,
+    batch6,
+    batch7,
+    batch8,
+    batch9
 };
 
 // No current need for error_code equivalence mapping.

include/bitcoin/node/impl/chasers/chaser_organize.ipp

@@ -90,8 +90,7 @@ bool CLASS::handle_chase(const code&, chase event_, event_value value) NOEXCEPT
         case chase::unvalid:
         case chase::unconfirmable:
         {
-            // !mark_unconfirmable allows node to stall, preserving log.
-            if (!node_settings().mark_unconfirmable)
+            if (!database_settings().mark_unconfirmable)
                 break;
 
             // Roll back the candidate chain to confirmed top (via fork point).

include/bitcoin/node/settings.hpp

@@ -41,9 +41,6 @@ class BCN_API settings
     bool memory_priority;
     bool allow_overlapped;
     bool batch_signatures;
-    bool mark_unconfirmable;
-    bool defer_validation;
-    bool defer_confirmation;
     float allowed_deviation;
     float minimum_fee_rate;
     float minimum_bump_rate;

src/chasers/chaser_check.cpp

@@ -206,7 +206,7 @@ void chaser_check::do_handle_purged(const code&) NOEXCEPT
     BC_ASSERT(stranded());
 
     start_tracking();
-    do_bump(height_t{});
+    do_bump({});
 }
 
 // starved
@@ -354,16 +354,20 @@ void chaser_check::do_regressed(height_t branch_point) NOEXCEPT
 // track downloaded in order (to move download window)
 // ----------------------------------------------------------------------------
 
-void chaser_check::do_advanced(height_t height) NOEXCEPT
+void chaser_check::do_advanced(height_t) NOEXCEPT
 {
     BC_ASSERT(stranded());
 
     // Validations are not ordered, so accumulate vs. compare height.
+    // Advancement through window is a quantity, not dedicated to a branch.
     ++advanced_;
 
-    // The full set of requested hashes has been validated.
+    // The full count of requested hashes has been validated.
     if (advanced_ == requested_)
-        do_headers(height);
+    {
+        notify(error::success, chase::advanced, advanced_);
+        do_headers({});
+    }
 }
 
 void chaser_check::do_checked(height_t height) NOEXCEPT
@@ -372,7 +376,7 @@ void chaser_check::do_checked(height_t height) NOEXCEPT
 
     // Candidate block was checked at the given height, advance.
     if (height == add1(position()))
-        do_bump(height);
+        do_bump({});
 }
 
 void chaser_check::do_bump(height_t) NOEXCEPT
@@ -386,13 +390,10 @@ void chaser_check::do_bump(height_t) NOEXCEPT
 
     // TODO: query.is_associated() is expensive (hashmap search).
     // Skip checked blocks starting immediately after last checked.
-    while (!closed() && query.is_associated(
-        query.to_candidate((height = add1(height)))))
-    {
+    while (!closed() && query.is_associated(query.to_candidate(++height)))
         set_position(height);
-    }
 
-    do_headers(sub1(height));
+    do_headers({});
 }
 
 // add headers
@@ -402,11 +403,8 @@ void chaser_check::do_headers(height_t) NOEXCEPT
 {
     BC_ASSERT(stranded());
 
-    const auto added = set_unassociated();
-    if (is_zero(added))
-        return;
-
-    notify(error::success, chase::download, added);
+    if (const auto added = set_unassociated(); is_nonzero(added))        
+        notify(error::success, chase::download, added);
 }
 
 // get/put hashes

src/chasers/chaser_confirm.cpp

@@ -34,8 +34,7 @@ using namespace std::placeholders;
 BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT)
 
 chaser_confirm::chaser_confirm(full_node& node) NOEXCEPT
-  : chaser(node),
-    defer_(node.node_settings().defer_confirmation)
+  : chaser(node)
 {
 }
 
@@ -49,11 +48,7 @@ code chaser_confirm::start() NOEXCEPT
         LOGN("Node is current at startup block [" << position() << "].");
     }
 
-    if (!defer_)
-    {
-        SUBSCRIBE_CHASE(handle_chase, _1, _2, _3);
-    }
-
+    SUBSCRIBE_CHASE(handle_chase, _1, _2, _3);
     return error::success;
 }
 
@@ -286,10 +281,7 @@ bool chaser_confirm::confirm_block(const header_link& link, size_t height,
 
     if (const auto ec = query.block_confirmable(link))
     {
-        // !mark_unconfirmable allows node to stall, preserving log.
-        // Will continue to validate this block and fail to confirm here.
-        if (node_settings().mark_unconfirmable &&
-            !query.set_block_unconfirmable(link))
+        if (!query.set_block_unconfirmable(link))
         {
             fault(error::confirm9);
             return false;
@@ -326,16 +318,22 @@ bool chaser_confirm::complete_block(const code& ec, const header_link& link,
 {
     BC_ASSERT(stranded());
 
-    if (ec)
+    // Database errors are fatal (or disk full recoverable).
+    if (ec && database::error::error_category::contains(ec))
     {
-        // Database errors are fatal.
-        if (database::error::error_category::contains(ec))
-        {
-            LOGF("Fault confirming [" << height << "] " << ec.message());
-            fault(ec);
-            return false;
-        }
+        LOGF("Fault confirming [" << height << "] " << ec.message());
+        fault(ec);
+        return false;
+    }
+
+    return notify_block(ec, height, link, bypass);
+}
 
+bool chaser_confirm::notify_block(const code& ec, size_t height,
+    const header_link& link, bool bypass) NOEXCEPT
+{
+    if (ec)
+    {
         // UNCONFIRMABLE BLOCK (not a fault but discontinue)
         notify(ec, chase::unconfirmable, link);
         fire(events::block_unconfirmable, height);