diff --git a/conanfile.py b/conanfile.py index ced6dea4a..3490170fa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "2.7.0" + version = "2.6.0" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" diff --git a/src/lib/homestore_backend/CMakeLists.txt b/src/lib/homestore_backend/CMakeLists.txt index 78f4611f3..e360e851c 100644 --- a/src/lib/homestore_backend/CMakeLists.txt +++ b/src/lib/homestore_backend/CMakeLists.txt @@ -78,7 +78,6 @@ target_sources(homestore_test_misc PRIVATE $(blob_header_buf().bytes()); } sisl::io_blob_safe& blob_header_buf() { return data_bufs_[blob_header_idx_]; } }; @@ -123,14 +127,8 @@ BlobManager::AsyncResult< blob_id_t > HSHomeObject::_put_blob(ShardInfo const& s return folly::makeUnexpected(BlobError(BlobErrorCode::RETRY_REQUEST)); } - // check user key size - if (blob.user_key.size() > BlobHeader::max_user_key_length) { - BLOGE(tid, shard.id, new_blob_id, "input user key length > max_user_key_length {}", blob.user_key.size(), - BlobHeader::max_user_key_length); - return folly::makeUnexpected(BlobError(BlobErrorCode::INVALID_ARG)); - } // Create a put_blob request which allocates for header, key and blob_header, user_key. Data sgs are added later - auto req = put_blob_req_ctx::make(sisl::round_up(sizeof(BlobHeader), repl_dev->get_blk_size())); + auto req = put_blob_req_ctx::make(sizeof(BlobHeader) + blob.user_key.size()); req->header()->msg_type = ReplicationMessageType::PUT_BLOB_MSG; req->header()->payload_size = 0; req->header()->payload_crc = 0; @@ -159,18 +157,11 @@ BlobManager::AsyncResult< blob_id_t > HSHomeObject::_put_blob(ShardInfo const& s req->blob_header()->object_offset = blob.object_off; // Append the user key information if present. - if (!blob.user_key.empty()) { - std::memcpy(req->blob_header()->user_key, blob.user_key.data(), blob.user_key.size()); - } + if (!blob.user_key.empty()) { req->copy_user_key(blob.user_key); } // Set offset of actual data after the blob header and user key (rounded off) req->blob_header()->data_offset = req->blob_header_buf().size(); - if (req->blob_header()->data_offset != _data_block_size) { - BLOGE(tid, shard.id, new_blob_id, "data offset {}, req->blob_header_buf().size() {}", - req->blob_header()->data_offset, req->blob_header_buf().size()); - RELEASE_ASSERT(req->blob_header()->data_offset == _data_block_size, - "blob header should be one block after padding"); - } + // In case blob body is not aligned, create a new aligned buffer and copy the blob body. if (((r_cast< uintptr_t >(blob.body.cbytes()) % io_align) != 0) || ((blob_size % io_align) != 0)) { // If address or size is not aligned, create a separate aligned buffer and do expensive memcpy. @@ -357,7 +348,10 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob_data(const shared< home return folly::makeUnexpected(BlobError(BlobErrorCode::READ_FAILED)); } - std::string user_key = std::string((const char*)header->user_key, (size_t)header->user_key_size); + // Metadata start offset is just after blob header + std::string user_key = header->user_key_size + ? std::string((const char*)(read_buf.bytes() + sizeof(BlobHeader)), (size_t)header->user_key_size) + : std::string{}; uint8_t const* blob_bytes = read_buf.bytes() + header->data_offset; uint8_t computed_hash[BlobHeader::blob_max_hash_len]{}; diff --git a/src/lib/homestore_backend/hs_homeobject.hpp b/src/lib/homestore_backend/hs_homeobject.hpp index 789670918..56125662d 100644 --- a/src/lib/homestore_backend/hs_homeobject.hpp +++ b/src/lib/homestore_backend/hs_homeobject.hpp @@ -371,8 +371,6 @@ class HSHomeObject : public HomeObjectImpl { // Padding of zeroes is added to make sure the whole payload be aligned to device block size. struct BlobHeader : DataHeader { static constexpr uint64_t blob_max_hash_len = 32; - static constexpr uint64_t max_blocks = 64; - static constexpr uint64_t max_user_key_length = 1024 + 1; enum class HashAlgorithm : uint8_t { NONE = 0, @@ -384,14 +382,12 @@ class HSHomeObject : public HomeObjectImpl { HashAlgorithm hash_algorithm; mutable uint8_t header_hash[blob_max_hash_len]{}; uint8_t hash[blob_max_hash_len]{}; - uint8_t block_hashes[max_blocks][blob_max_hash_len]{}; shard_id_t shard_id; blob_id_t blob_id; uint32_t blob_size; uint64_t object_offset; // Offset of this blob in the object. Provided by GW. uint32_t data_offset; // Offset of actual data blob stored after the metadata. uint32_t user_key_size; // Actual size of the user key. - uint8_t user_key[max_user_key_length]{}; std::string to_string() const { return fmt::format("magic={:#x} version={} shard={:#x} blob_size={} user_size={} algo={} hash={:np}\n", @@ -440,8 +436,7 @@ class HSHomeObject : public HomeObjectImpl { } }; #pragma pack() - // size of BlobHeader should be smaller than _data_block_size - static_assert(sizeof(BlobHeader) < _data_block_size); + struct BlobInfo { shard_id_t shard_id; blob_id_t blob_id; diff --git a/src/lib/homestore_backend/pg_blob_iterator.cpp b/src/lib/homestore_backend/pg_blob_iterator.cpp index 0881bf18b..6e72a6b28 100644 --- a/src/lib/homestore_backend/pg_blob_iterator.cpp +++ b/src/lib/homestore_backend/pg_blob_iterator.cpp @@ -230,7 +230,9 @@ BlobManager::AsyncResult< blob_read_result > HSHomeObject::PGBlobIterator::load_ return blob_read_result(blob_id, std::move(read_buf), ResyncBlobState::CORRUPTED); } - std::string user_key = std::string((const char*)header->user_key, (size_t)header->user_key_size); + std::string user_key = header->user_key_size + ? std::string((const char*)(read_buf.bytes() + sizeof(BlobHeader)), (size_t)header->user_key_size) + : std::string{}; uint8_t const* blob_bytes = read_buf.bytes() + header->data_offset; uint8_t computed_hash[BlobHeader::blob_max_hash_len]{}; diff --git a/src/lib/homestore_backend/snapshot_receive_handler.cpp b/src/lib/homestore_backend/snapshot_receive_handler.cpp index 705d31c93..c60e97504 100644 --- a/src/lib/homestore_backend/snapshot_receive_handler.cpp +++ b/src/lib/homestore_backend/snapshot_receive_handler.cpp @@ -225,7 +225,9 @@ int HSHomeObject::SnapshotReceiveHandler::process_blobs_snapshot_data(ResyncBlob LOGE("Invalid header found for blob_id={}: [header={}]", blob->blob_id(), header->to_string()); return INVALID_BLOB_HEADER; } - std::string user_key = std::string((const char*)header->user_key, (size_t)header->user_key_size); + std::string user_key = header->user_key_size + ? std::string(r_cast< const char* >(blob_data + sizeof(BlobHeader)), header->user_key_size) + : std::string{}; uint8_t computed_hash[BlobHeader::blob_max_hash_len]{}; home_obj_.compute_blob_payload_hash(header->hash_algorithm, blob_data + header->data_offset, diff --git a/src/lib/homestore_backend/tests/homeobj_fixture.hpp b/src/lib/homestore_backend/tests/homeobj_fixture.hpp index 599386caa..4174682f2 100644 --- a/src/lib/homestore_backend/tests/homeobj_fixture.hpp +++ b/src/lib/homestore_backend/tests/homeobj_fixture.hpp @@ -759,14 +759,17 @@ class HomeObjectFixture : public ::testing::Test { auto blob = build_blob(blob_id); auto blob_size = blob.body.size(); - uint64_t header_size{sisl::round_up(sizeof(HSHomeObject::BlobHeader), io_align)}; - header_size = sisl::round_up(header_size, HSHomeObject::_data_block_size); + uint64_t actual_written_size{ + uint32_cast(sisl::round_up(sizeof(HSHomeObject::BlobHeader) + blob.user_key.size(), io_align))}; - blob_size = sisl::round_up(blob_size, io_align); - blob_size = sisl::round_up(blob_size, HSHomeObject::_data_block_size); + if (((r_cast< uintptr_t >(blob.body.cbytes()) % io_align) != 0) || ((blob_size % io_align) != 0)) { + blob_size = sisl::round_up(blob_size, io_align); + } + + actual_written_size += blob_size; - auto actual_written_size = header_size + blob_size; - EXPECT_EQ(actual_written_size % HSHomeObject::_data_block_size, 0); + auto pad_len = sisl::round_up(actual_written_size, HSHomeObject::_data_block_size) - actual_written_size; + if (pad_len) { actual_written_size += pad_len; } return actual_written_size / HSHomeObject::_data_block_size; } diff --git a/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp b/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp index 91267d766..7471d3b55 100644 --- a/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp +++ b/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp @@ -232,9 +232,8 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandler) { for (uint64_t i = 1; i <= num_shards_per_pg; i++) { shard_ids.push_back(i); } - auto pg_entry = - CreateResyncPGMetaDataDirect(builder, pg_id, &uuid, pg->pg_info_.size, pg->pg_info_.expected_member_num, - pg->pg_info_.chunk_size, blob_seq_num, num_shards_per_pg, &members, &shard_ids); + auto pg_entry = CreateResyncPGMetaDataDirect(builder, pg_id, &uuid, pg->pg_info_.size, pg->pg_info_.expected_member_num, pg->pg_info_.chunk_size, + blob_seq_num, num_shards_per_pg, &members, &shard_ids); builder.Finish(pg_entry); auto pg_meta = GetResyncPGMetaData(builder.GetBufferPointer()); auto ret = handler->process_pg_snapshot_data(*pg_meta); @@ -305,7 +304,8 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandler) { // Construct raw blob buffer auto blob = build_blob(cur_blob_id); - const auto aligned_hdr_size = sisl::round_up(sizeof(HSHomeObject::BlobHeader), _obj_inst->_data_block_size); + const auto aligned_hdr_size = + sisl::round_up(sizeof(HSHomeObject::BlobHeader) + blob.user_key.size(), io_align); sisl::io_blob_safe blob_raw(aligned_hdr_size + blob.body.size(), io_align); HSHomeObject::BlobHeader hdr; hdr.type = HSHomeObject::DataHeader::data_type_t::BLOB_INFO; @@ -316,14 +316,18 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandler) { hdr.user_key_size = blob.user_key.size(); hdr.object_offset = blob.object_off; hdr.data_offset = aligned_hdr_size; - if (!blob.user_key.empty()) { std::memcpy(hdr.user_key, blob.user_key.data(), blob.user_key.size()); } _obj_inst->compute_blob_payload_hash(hdr.hash_algorithm, blob.body.cbytes(), blob.body.size(), - hdr.user_key, hdr.user_key_size, hdr.hash, + reinterpret_cast< uint8_t* >(blob.user_key.data()), + blob.user_key.size(), hdr.hash, HSHomeObject::BlobHeader::blob_max_hash_len); hdr.seal(); std::memcpy(blob_raw.bytes(), &hdr, sizeof(HSHomeObject::BlobHeader)); - std::memcpy(blob_raw.bytes() + hdr.data_offset, blob.body.cbytes(), blob.body.size()); + if (!blob.user_key.empty()) { + std::memcpy((blob_raw.bytes() + sizeof(HSHomeObject::BlobHeader)), blob.user_key.data(), + blob.user_key.size()); + } + std::memcpy(blob_raw.bytes() + aligned_hdr_size, blob.body.cbytes(), blob.body.size()); // Simulate blob data corruption - tamper with random bytes if (is_corrupted_batch || blob_state == ResyncBlobState::CORRUPTED) {