Enforce configurable maximum ledger transaction size

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 
+- The `ledger.max_transaction_size` configuration option now limits the total serialised size of a transaction when writing it to the ledger. The limit covers the whole ledger entry: the fixed 8-byte ledger entry header, the ledger encryption header, public domain size field, public domain and encrypted private domain. It is enforced only when serialising new transactions; deserialising existing entries (including during recovery) and snapshots are not affected. The default value is `100MB`. (#7992)
 - JWT/JWK auto-refresh outbound HTTP fetches (OpenID metadata and JWKS) now use the curl multi singleton client introduced in #7102, replacing the previous `RPCSessions::create_client()` path. Connection and TLS failures are now counted in refresh failure metrics via `send_refresh_jwt_keys_error()`, improving observability of network-level refresh errors (#7989).
 - JWT/JWK auto-refresh now supports configuring the maximum response body size for fetched OpenID metadata and JWKS via the `jwt.key_refresh_max_response_size` node startup config setting (#7989).
 - Fatal task worker stack traces now use libbacktrace for improved function and source-location resolution. Building CCF now requires the libbacktrace development package, and the RPM development package depends on `libbacktrace-static` (#7721).

include/ccf/node/startup_config.h

@@ -48,6 +48,7 @@ namespace ccf
       std::string directory = "ledger";
       std::vector<std::string> read_only_directories;
       ccf::ds::SizeString chunk_size = {"5MB"};
+      ccf::ds::SizeString max_transaction_size = {"100MB"};
 
       bool operator==(const Ledger&) const = default;
     };

src/common/configuration.h

@@ -65,7 +65,11 @@ namespace ccf
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(CCFConfig::Ledger);
   DECLARE_JSON_REQUIRED_FIELDS(CCFConfig::Ledger);
   DECLARE_JSON_OPTIONAL_FIELDS(
-    CCFConfig::Ledger, directory, read_only_directories, chunk_size);
+    CCFConfig::Ledger,
+    directory,
+    read_only_directories,
+    chunk_size,
+    max_transaction_size);
 
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(CCFConfig::LedgerSignatures);
   DECLARE_JSON_REQUIRED_FIELDS(CCFConfig::LedgerSignatures);

src/consensus/ledger_enclave.h

@@ -8,6 +8,8 @@
 #include "kv/kv_types.h"
 #include "kv/serialised_entry_format.h"
 
+#include <fmt/format.h>
+
 namespace consensus
 {
   class LedgerEnclave
@@ -25,7 +27,22 @@ namespace consensus
     {
       auto header =
         serialized::peek<ccf::kv::SerialisedEntryHeader>(data, size);
-      size_t entry_size = ccf::kv::serialised_entry_header_size + header.size;
+      const size_t body_size = header.size;
+      // This is a buffer-bounds safety check, not the configurable transaction
+      // size limit: that limit applies only when serialising new transactions.
+      // Deserialisation (including recovery replay) is exempt, so entries
+      // written under a larger or unset limit can always be read back.
+      if (body_size + ccf::kv::serialised_entry_header_size > size)
+      {
+        throw std::logic_error(fmt::format(
+          "Cannot read transaction with serialised body size {} bytes from "
+          "buffer containing {} bytes after the fixed {}-byte ledger entry "
+          "header",
+          body_size,
+          size - ccf::kv::serialised_entry_header_size,
+          ccf::kv::serialised_entry_header_size));
+      }
+      size_t entry_size = ccf::kv::serialised_entry_header_size + body_size;
       std::vector<uint8_t> entry(data, data + entry_size);
       serialized::skip(data, size, entry_size);
       return entry;

src/enclave/enclave.h

@@ -83,6 +83,7 @@ namespace ccf
       size_t sig_tx_interval,
       size_t sig_ms_interval,
       size_t chunk_threshold,
+      size_t max_transaction_size,
       const ccf::consensus::Configuration& consensus_config,
       const ccf::crypto::CurveID& curve_id,
       ccf::ds::WorkBeaconPtr work_beacon_,
@@ -101,6 +102,7 @@ namespace ccf
 
       network.tables->set_chunker(
         std::make_shared<ccf::kv::LedgerChunker>(chunk_threshold));
+      network.tables->set_max_transaction_size(max_transaction_size);
 
       LOG_TRACE_FMT("Creating node");
       node = std::make_unique<ccf::NodeState>(

src/enclave/main.cpp

@@ -106,6 +106,7 @@ namespace ccf
         ccf_config.ledger_signatures.tx_count,
         ccf_config.ledger_signatures.delay.count_ms(),
         ccf_config.ledger.chunk_size,
+        ccf_config.ledger.max_transaction_size,
         ccf_config.consensus,
         ccf_config.node_certificate.curve_id,
         work_beacon,

src/kv/committable_tx.h

@@ -91,7 +91,9 @@ namespace ccf::kv
         entry_type,
         entry_flags,
         tx_commit_evidence_digest,
-        claims_digest_);
+        claims_digest_,
+        false /* historical_hint */,
+        pimpl->store->get_max_transaction_size());
 
       // Process in security domain order
       for (auto domain : {SecurityDomain::PUBLIC, SecurityDomain::PRIVATE})
@@ -161,6 +163,11 @@ namespace ccf::kv
       ccf::kv::ConsensusHookPtrs hooks;
 
       std::optional<Version> new_maps_conflict_version = std::nullopt;
+      // If serialisation later rejects this transaction because it exceeds the
+      // configured size limit, roll the store back to the state from before
+      // apply_changes mutates maps and advances the version.
+      const auto [rollback_txid, rollback_term] =
+        pimpl->store->current_txid_and_commit_term();
 
       bool track_deletes_on_missing_keys = false;
       auto c = apply_changes(
@@ -246,6 +253,12 @@ namespace ccf::kv
             std::move(hooks)),
           false);
       }
+      catch (const MaxTransactionSizeExceeded&)
+      {
+        pimpl->store->rollback(rollback_txid, rollback_term);
+        committed = false;
+        throw;
+      }
       catch (const std::exception& e)
       {
         committed = false;

src/kv/generic_serialise_wrapper.h

@@ -23,6 +23,7 @@ namespace ccf::kv
     TxID tx_id;
     EntryType entry_type;
     SerialisedEntryFlags header_flags;
+    size_t max_transaction_size;
 
     std::shared_ptr<AbstractTxEncryptor> crypto_util;
 
@@ -67,10 +68,13 @@ namespace ccf::kv
       // in regular transactions, but absent in snapshots.
       const ccf::crypto::Sha256Hash& commit_evidence_digest_ = {},
       const ccf::ClaimsDigest& claims_digest_ = ccf::no_claims(),
-      bool historical_hint_ = false) :
+      bool historical_hint_ = false,
+      size_t max_transaction_size_ =
+        SerialisedEntryHeader::max_serialised_entry_body_size) :
       tx_id(tx_id_),
       entry_type(entry_type_),
       header_flags(header_flags_),
+      max_transaction_size(max_transaction_size_),
       crypto_util(std::move(e)),
       historical_hint(historical_hint_)
     {
@@ -174,8 +178,16 @@ namespace ccf::kv
       }
       entry_header.set_size(size_);
 
+      // The configured limit applies to the whole serialised ledger entry,
+      // including the fixed-size entry header.
       size_ += sizeof(SerialisedEntryHeader);
 
+      if (size_ > max_transaction_size)
+      {
+        throw MaxTransactionSizeExceeded(
+          describe_serialised_entry_size_error(size_, max_transaction_size));
+      }
+
       std::vector<uint8_t> entry(size_);
       auto* data_ = entry.data();
 
@@ -302,7 +314,7 @@ namespace ccf::kv
       size_t size,
       ccf::kv::Term& term,
       EntryFlags& flags,
-      bool historical_hint) override
+      bool historical_hint = false) override
     {
       current_reader = &public_reader;
       const auto* data_ = data;
@@ -320,7 +332,6 @@ namespace ccf::kv
           tx_header.size,
           size_));
       }
-
       const auto* gcm_hdr_data = data_;
 
       switch (tx_header.version)

src/kv/kv_types.h

@@ -30,6 +30,7 @@
 #include <memory>
 #include <optional>
 #include <set>
+#include <stdexcept>
 #include <string>
 #include <tuple>
 #include <unordered_set>
@@ -383,6 +384,13 @@ namespace ccf::kv
     }
   };
 
+  class MaxTransactionSizeExceeded : public std::logic_error
+  {
+  public:
+    MaxTransactionSizeExceeded(const std::string& msg) : std::logic_error(msg)
+    {}
+  };
+
   class TxHistory
   {
   public:
@@ -722,6 +730,7 @@ namespace ccf::kv
     virtual std::shared_ptr<TxHistory> get_history() = 0;
     virtual std::shared_ptr<ILedgerChunker> get_chunker() = 0;
     virtual EncryptorPtr get_encryptor() = 0;
+    [[nodiscard]] virtual size_t get_max_transaction_size() const = 0;
     virtual std::unique_ptr<AbstractExecutionWrapper> deserialize(
       const std::vector<uint8_t>& data,
       bool public_only = false,

src/kv/serialised_entry_format.h

@@ -4,7 +4,10 @@
 
 #include "ds/ccf_assert.h"
 
+#include <climits>
+#include <fmt/format.h>
 #include <stdint.h>
+#include <string>
 
 namespace ccf::kv
 {
@@ -25,22 +28,36 @@ namespace ccf::kv
     static constexpr auto BITS_FOR_SIZE =
       (sizeof(uint64_t) - sizeof(uint8_t) - sizeof(SerialisedEntryFlags)) *
       CHAR_BIT;
+    static constexpr uint64_t max_serialised_entry_body_size =
+      (uint64_t{1} << BITS_FOR_SIZE) - 1;
     uint64_t size : BITS_FOR_SIZE = 0;
 
     void set_size(uint64_t size_)
     {
-      [[maybe_unused]] static constexpr size_t max_entry_size = 1UL
-        << BITS_FOR_SIZE;
       CCF_ASSERT_FMT(
-        size_ < max_entry_size,
+        size_ <= max_serialised_entry_body_size,
         "Cannot serialise entry of size {} (max allowed size is {})",
         size_,
-        max_entry_size);
+        max_serialised_entry_body_size);
       size = size_;
     }
   };
   static_assert(sizeof(SerialisedEntryHeader) == sizeof(uint64_t));
 
   static constexpr size_t serialised_entry_header_size =
     sizeof(SerialisedEntryHeader);
-}
+
+  static inline std::string describe_serialised_entry_size_error(
+    size_t entry_size, size_t max_entry_size)
+  {
+    return fmt::format(
+      "Cannot serialise transaction with total serialised size {} bytes. The "
+      "configured maximum is {} bytes. The size compared to this limit is the "
+      "whole ledger entry: the fixed {}-byte ledger entry header plus the "
+      "serialised transaction body (any ledger encryption header, public "
+      "domain size field, public domain and encrypted private domain).",
+      entry_size,
+      max_entry_size,
+      serialised_entry_header_size);
+  }
+}

src/kv/store.h

@@ -15,6 +15,7 @@
 #include "kv_types.h"
 
 #define FMT_HEADER_ONLY
+#include <algorithm>
 #include <atomic>
 #include <fmt/format.h>
 #include <memory>
@@ -97,6 +98,8 @@ namespace ccf::kv
     std::shared_ptr<ILedgerChunker> chunker = nullptr;
     EncryptorPtr encryptor = nullptr;
     SnapshotterPtr snapshotter = nullptr;
+    size_t max_transaction_size =
+      SerialisedEntryHeader::max_serialised_entry_body_size;
 
     // Generally we will only accept deserialised views if they are contiguous -
     // at Version N we reject everything but N+1. The exception is when a Store
@@ -223,6 +226,30 @@ namespace ccf::kv
       return encryptor;
     }
 
+    void set_max_transaction_size(size_t max_transaction_size_)
+    {
+      static const auto max_allocatable_body =
+        std::vector<uint8_t>().max_size() - serialised_entry_header_size;
+      const auto effective_max = std::min(
+        static_cast<size_t>(
+          SerialisedEntryHeader::max_serialised_entry_body_size),
+        max_allocatable_body);
+      if (max_transaction_size_ > effective_max)
+      {
+        throw std::logic_error(fmt::format(
+          "Configured maximum transaction size {} exceeds the largest "
+          "supported serialised transaction body size {}",
+          max_transaction_size_,
+          effective_max));
+      }
+      max_transaction_size = max_transaction_size_;
+    }
+
+    [[nodiscard]] size_t get_max_transaction_size() const override
+    {
+      return max_transaction_size;
+    }
+
     void set_snapshotter(const SnapshotterPtr& snapshotter_)
     {
       snapshotter = snapshotter_;
@@ -387,6 +414,10 @@ namespace ccf::kv
       std::unique_ptr<AbstractSnapshot> snapshot) override
     {
       auto e = get_encryptor();
+      // Snapshots capture the entire committed state and are legitimately much
+      // larger than any single transaction, so the configured
+      // max_transaction_size (a per-transaction write limit) is deliberately
+      // not applied here.
       return snapshot->serialise(e);
     }
 
@@ -405,6 +436,9 @@ namespace ccf::kv
 
       ccf::kv::Term term = 0;
       ccf::kv::EntryFlags entry_flags = {};
+      // Snapshots are not subject to the per-transaction max_transaction_size
+      // limit (see serialise_snapshot), so it is not applied when reading one
+      // back either.
       auto v_ = d.init(data, size, term, entry_flags, is_historical);
       if (!v_.has_value())
       {