diff options
Diffstat (limited to 'src/mongo/s/chunk_map_test.cpp')
| -rw-r--r-- | src/mongo/s/chunk_map_test.cpp | 455 |
1 files changed, 434 insertions, 21 deletions
diff --git a/src/mongo/s/chunk_map_test.cpp b/src/mongo/s/chunk_map_test.cpp index 6514fc00745..96980c009f7 100644 --- a/src/mongo/s/chunk_map_test.cpp +++ b/src/mongo/s/chunk_map_test.cpp @@ -27,18 +27,66 @@ * it in the license file. */ +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest + #include "mongo/platform/basic.h" +#include "mongo/logv2/log.h" +#include "mongo/platform/random.h" #include "mongo/s/chunk_manager.h" +#include "mongo/s/chunk_writes_tracker.h" +#include "mongo/s/chunks_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { +using chunks_test_util::assertEqualChunkInfo; +using chunks_test_util::calculateCollVersion; +using chunks_test_util::calculateIntermediateShardKey; +using chunks_test_util::genChunkVector; +using chunks_test_util::genRandomSplitPoints; +using chunks_test_util::performRandomChunkOperations; + namespace { -const NamespaceString kNss("TestDB", "TestColl"); +PseudoRandom _random{SecureRandom().nextInt64()}; + const ShardId kThisShard("testShard"); +ShardVersionMap getShardVersionMap(const ChunkMap& chunkMap) { + return chunkMap.getShardVersionsMap(); +} + +std::map<ShardId, ChunkVersion> calculateShardVersions( + const std::vector<std::shared_ptr<ChunkInfo>>& chunkVector) { + std::map<ShardId, ChunkVersion> svMap; + for (const auto& chunk : chunkVector) { + auto mapIt = svMap.find(chunk->getShardId()); + if (mapIt == svMap.end()) { + svMap.emplace(chunk->getShardId(), chunk->getLastmod()); + continue; + } + if (mapIt->second.isOlderThan(chunk->getLastmod())) { + mapIt->second = chunk->getLastmod(); + } + } + return svMap; +} + +std::vector<std::shared_ptr<ChunkInfo>> toChunkInfoPtrVector( + const std::vector<ChunkType>& chunkTypes, bool initializeWriteTrackerRandom = true) { + std::vector<std::shared_ptr<ChunkInfo>> chunkPtrs; + chunkPtrs.reserve(chunkTypes.size()); + for (const auto& chunkType : chunkTypes) { + auto chunkInfoPtr = std::make_shared<ChunkInfo>(chunkType); + if (initializeWriteTrackerRandom) { + chunkInfoPtr->getWritesTracker()->addBytesWritten(_random.nextInt64(30)); + } + chunkPtrs.push_back(std::move(chunkInfoPtr)); + } + return chunkPtrs; +} + class ChunkMapTest : public unittest::Test { public: const KeyPattern& getShardKeyPattern() const { @@ -49,16 +97,36 @@ public: return _uuid; } + const OID& collEpoch() const { + return _epoch; + } + + const Timestamp& collTimestamp() const { + return _collTimestamp; + } + + ChunkMap makeChunkMap(const std::vector<std::shared_ptr<ChunkInfo>>& chunks) const { + const auto chunkBucketSize = + static_cast<size_t>(_random.nextInt64(chunks.size() * 1.2) + 1); + LOGV2(7162701, "Creating new chunk map", "chunkBucketSize"_attr = chunkBucketSize); + return ChunkMap{collEpoch(), collTimestamp(), chunkBucketSize}.createMerged(chunks); + } + + std::vector<ChunkType> genRandomChunkVector(size_t maxNumChunks = 30, + size_t minNumChunks = 1) const { + return chunks_test_util::genRandomChunkVector( + _uuid, _epoch, _collTimestamp, maxNumChunks, minNumChunks); + } + private: - KeyPattern _shardKeyPattern{BSON("a" << 1)}; + KeyPattern _shardKeyPattern{chunks_test_util::kShardKeyPattern}; const UUID _uuid = UUID::gen(); + const OID _epoch{OID::gen()}; + const Timestamp _collTimestamp{1, 1}; }; -} // namespace - TEST_F(ChunkMapTest, TestAddChunk) { - const OID epoch = OID::gen(); - ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; auto chunk = std::make_shared<ChunkInfo>( ChunkType{uuid(), @@ -66,18 +134,258 @@ TEST_F(ChunkMapTest, TestAddChunk) { version, kThisShard}); - ChunkMap chunkMap{epoch, Timestamp(1, 1)}; - auto newChunkMap = chunkMap.createMerged({chunk}); + auto newChunkMap = makeChunkMap({chunk}); ASSERT_EQ(newChunkMap.size(), 1); } +TEST_F(ChunkMapTest, ConstructChunkMapRandom) { + auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); + + const auto expectedShardVersions = calculateShardVersions(chunkVector); + const auto expectedCollVersion = calculateCollVersion(expectedShardVersions); + + const auto chunkMap = makeChunkMap(chunkVector); + + // Check that it contains all the chunks + ASSERT_EQ(chunkVector.size(), chunkMap.size()); + // Check collection version + ASSERT_EQ(expectedCollVersion, chunkMap.getVersion()); + + size_t i = 0; + chunkMap.forEach([&](const auto& chunkPtr) { + const auto& expectedChunkPtr = chunkVector[i++]; + assertEqualChunkInfo(*expectedChunkPtr, *chunkPtr); + return true; + }); + + // Validate all shard versions + const auto shardVersions = getShardVersionMap(chunkMap); + ASSERT_EQ(expectedShardVersions.size(), shardVersions.size()); + for (const auto& mapIt : shardVersions) { + ASSERT_EQ(expectedShardVersions.at(mapIt.first), mapIt.second.shardVersion); + } + + // Check that vectors are balanced in size + auto maxVectorSize = static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() * 1.5)); + auto minVectorSize = std::min( + chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2))); + + for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) { + ASSERT_GTE(chunkVectorPtr->size(), minVectorSize); + ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize); + } +} + +TEST_F(ChunkMapTest, ConstructChunkMapRandomAllChunksSameVersion) { + auto chunkVector = genRandomChunkVector(); + auto commonVersion = chunkVector.front().getVersion(); + + // Set same version on all chunks + for (auto& chunk : chunkVector) { + chunk.setVersion(commonVersion); + } + + auto chunkInfoVector = toChunkInfoPtrVector(chunkVector); + const auto expectedShardVersions = calculateShardVersions(chunkInfoVector); + const auto expectedCollVersion = calculateCollVersion(expectedShardVersions); + + ASSERT_EQ(commonVersion, expectedCollVersion); + + const auto chunkMap = makeChunkMap(chunkInfoVector); + + // Check that it contains all the chunks + ASSERT_EQ(chunkInfoVector.size(), chunkMap.size()); + // Check collection version + ASSERT_EQ(expectedCollVersion, chunkMap.getVersion()); + + size_t i = 0; + chunkMap.forEach([&](const auto& chunkPtr) { + const auto& expectedChunkPtr = chunkInfoVector[i++]; + assertEqualChunkInfo(*expectedChunkPtr, *chunkPtr); + return true; + }); + + // Validate all shard versions + const auto shardVersions = getShardVersionMap(chunkMap); + ASSERT_EQ(expectedShardVersions.size(), shardVersions.size()); + for (const auto& mapIt : shardVersions) { + ASSERT_EQ(expectedShardVersions.at(mapIt.first), mapIt.second.shardVersion); + } +} + +/* + * Check that constucting a ChunkMap with chunks that have mismatching timestamp fails. + */ +TEST_F(ChunkMapTest, ConstructChunkMapMismatchingTimestamp) { + auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); + + // Set a different epoch in one of the chunks + const Timestamp wrongTimestamp{Date_t::now()}; + ASSERT_NE(wrongTimestamp, collTimestamp()); + const auto wrongChunkIdx = _random.nextInt32(chunkVector.size()); + const auto oldChunk = chunkVector.at(wrongChunkIdx); + const auto oldVersion = oldChunk->getLastmod(); + const ChunkVersion wrongVersion{ + oldVersion.majorVersion(), oldVersion.minorVersion(), collEpoch(), wrongTimestamp}; + chunkVector[wrongChunkIdx] = std::make_shared<ChunkInfo>( + ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()}); + + ASSERT_THROWS_CODE( + makeChunkMap(chunkVector), AssertionException, ErrorCodes::ConflictingOperationInProgress); +} + +TEST_F(ChunkMapTest, UpdateMapNotLeaveSmallVectors) { + const ChunkVersion initialVersion{1, 0, collEpoch(), collTimestamp()}; + auto chunkVector = toChunkInfoPtrVector( + genChunkVector(uuid(), genRandomSplitPoints(8), initialVersion, 1 /*numShards*/)); + + const auto chunkBucketSize = 4; + LOGV2(7162703, "Constructing new chunk map", "chunkBucketSize"_attr = chunkBucketSize); + const auto initialChunkMap = + ChunkMap(collEpoch(), collTimestamp(), chunkBucketSize).createMerged(chunkVector); + + // Check that it contains all the chunks + ASSERT_EQ(chunkVector.size(), initialChunkMap.size()); + + auto mergedVersion = initialChunkMap.getVersion(); + mergedVersion.incMinor(); + + auto mergedChunk = std::make_shared<ChunkInfo>(ChunkType{ + uuid(), + ChunkRange{chunkVector[4]->getRange().getMin(), chunkVector.back()->getRange().getMax()}, + mergedVersion, + kThisShard}); + const auto chunkMap = initialChunkMap.createMerged({mergedChunk}); + + // Check that vectors are balanced in size + auto maxVectorSize = std::lround(chunkMap.getMaxChunkVectorSize() * 1.5); + auto minVectorSize = std::min( + chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2))); + + for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) { + ASSERT_GTE(chunkVectorPtr->size(), minVectorSize); + ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize); + } +} + +/* + * Check that updating a ChunkMap with chunks that have mismatching timestamp fails. + */ +TEST_F(ChunkMapTest, UpdateChunkMapMismatchingTimestamp) { + auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); + + auto chunkMap = makeChunkMap(chunkVector); + auto collVersion = chunkMap.getVersion(); + + // Set a different epoch in one of the chunks + const Timestamp wrongTimestamp{Date_t::now()}; + const auto wrongChunkIdx = _random.nextInt32(chunkVector.size()); + const auto oldChunk = chunkVector.at(wrongChunkIdx); + const ChunkVersion wrongVersion{ + collVersion.majorVersion(), collVersion.minorVersion(), collEpoch(), wrongTimestamp}; + auto updateChunk = std::make_shared<ChunkInfo>( + ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()}); + + ASSERT_THROWS_CODE(chunkMap.createMerged({updateChunk}), + AssertionException, + ErrorCodes::ConflictingOperationInProgress); +} + +/* + * Check that updating a ChunkMap with chunks that have lower version fails. + */ +TEST_F(ChunkMapTest, UpdateChunkMapLowerVersion) { + auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); + + auto chunkMap = makeChunkMap(chunkVector); + + const auto wrongChunkIdx = _random.nextInt32(chunkVector.size()); + const auto oldChunk = chunkVector.at(wrongChunkIdx); + const ChunkVersion wrongVersion{0, 1, collEpoch(), collTimestamp()}; + auto updateChunk = std::make_shared<ChunkInfo>( + ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()}); + + ASSERT_THROWS_CODE(chunkMap.createMerged({updateChunk}), AssertionException, 626840); +} +/* + * Test update of ChunkMap with random chunk manipulation (splits/merges/moves); + */ +TEST_F(ChunkMapTest, UpdateChunkMapRandom) { + auto initialChunks = genRandomChunkVector(); + auto initialChunksInfo = toChunkInfoPtrVector(initialChunks); + + const auto initialChunkMap = makeChunkMap(initialChunksInfo); + + const auto initialShardVersions = calculateShardVersions(initialChunksInfo); + const auto initialCollVersion = calculateCollVersion(initialShardVersions); + + auto chunks = initialChunks; + + const auto maxNumChunkOps = 2 * initialChunks.size(); + const auto numChunkOps = _random.nextInt32(maxNumChunkOps); + performRandomChunkOperations(&chunks, numChunkOps); + + auto chunksInfo = toChunkInfoPtrVector(initialChunks, false /* initializeWriteTrackerRandom */); + + std::vector<std::shared_ptr<ChunkInfo>> updatedChunksInfo; + for (auto& chunkPtr : chunksInfo) { + // First overlapping chunk in the initial vector + const auto& overlapInitChunk = + **std::lower_bound(initialChunksInfo.begin(), + initialChunksInfo.end(), + ShardKeyPattern::toKeyString(chunkPtr->getRange().getMin()), + [](const auto& chunkInfo, const std::string& shardKeyString) { + return chunkInfo->getMaxKeyString() <= shardKeyString; + }); + // The new chunks inherits the written bytes from the first overlapping old chunk + chunkPtr->getWritesTracker()->addBytesWritten( + overlapInitChunk.getWritesTracker()->getBytesWritten()); + + if (!chunkPtr->getLastmod().isOlderOrEqualThan(initialCollVersion)) { + updatedChunksInfo.push_back(std::make_shared<ChunkInfo>(ChunkType{ + uuid(), chunkPtr->getRange(), chunkPtr->getLastmod(), chunkPtr->getShardId()})); + } + } + + const auto expectedShardVersions = calculateShardVersions(chunksInfo); + const auto expectedCollVersion = calculateCollVersion(expectedShardVersions); + auto chunkMap = initialChunkMap.createMerged(updatedChunksInfo); + + // Check that it contains all the chunks + ASSERT_EQ(chunksInfo.size(), chunkMap.size()); + // Check collection version + ASSERT_EQ(expectedCollVersion, chunkMap.getVersion()); + + size_t i = 0; + chunkMap.forEach([&](const auto& chunkPtr) { + const auto& expectedChunkPtr = chunksInfo[i++]; + assertEqualChunkInfo(*expectedChunkPtr, *chunkPtr); + return true; + }); + + // Validate all shard versions + const auto shardVersions = getShardVersionMap(chunkMap); + ASSERT_EQ(expectedShardVersions.size(), shardVersions.size()); + for (const auto& mapIt : shardVersions) { + ASSERT_EQ(expectedShardVersions.at(mapIt.first), mapIt.second.shardVersion); + } + + // Check that vectors are balanced in size + auto maxVectorSize = static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() * 1.5)); + auto minVectorSize = std::min( + chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2))); + + for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) { + ASSERT_GTE(chunkVectorPtr->size(), minVectorSize); + ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize); + } +} + TEST_F(ChunkMapTest, TestEnumerateAllChunks) { - const OID epoch = OID::gen(); - ChunkMap chunkMap{epoch, Timestamp(1, 1)}; - ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; - auto newChunkMap = chunkMap.createMerged( + auto newChunkMap = makeChunkMap( {std::make_shared<ChunkInfo>( ChunkType{uuid(), ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)}, @@ -107,12 +415,11 @@ TEST_F(ChunkMapTest, TestEnumerateAllChunks) { ASSERT_EQ(count, newChunkMap.size()); } + TEST_F(ChunkMapTest, TestIntersectingChunk) { - const OID epoch = OID::gen(); - ChunkMap chunkMap{epoch, Timestamp(1, 1)}; - ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; - auto newChunkMap = chunkMap.createMerged( + auto newChunkMap = makeChunkMap( {std::make_shared<ChunkInfo>( ChunkType{uuid(), ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)}, @@ -135,14 +442,33 @@ TEST_F(ChunkMapTest, TestIntersectingChunk) { SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMin() == BSON("a" << 0))); ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMax() == BSON("a" << 100))); + + // findIntersectingChunks returns last chunk if invoked with MaxKey + intersectingChunk = + newChunkMap.findIntersectingChunk(BSON("a" << getShardKeyPattern().globalMax())); + ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMin() == + BSON("a" << 100))); + ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMax() == + getShardKeyPattern().globalMax())); +} + +TEST_F(ChunkMapTest, TestIntersectingChunkRandom) { + auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); + + const auto chunkMap = makeChunkMap(chunks); + + auto targetChunkIt = chunks.begin() + _random.nextInt64(chunks.size()); + auto intermediateKey = calculateIntermediateShardKey( + (*targetChunkIt)->getMin(), (*targetChunkIt)->getMax(), 0.2 /* minKeyProb */); + + auto intersectingChunkPtr = chunkMap.findIntersectingChunk(intermediateKey); + assertEqualChunkInfo(**(targetChunkIt), *intersectingChunkPtr); } TEST_F(ChunkMapTest, TestEnumerateOverlappingChunks) { - const OID epoch = OID::gen(); - ChunkMap chunkMap{epoch, Timestamp(1, 1)}; - ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; - auto newChunkMap = chunkMap.createMerged( + auto newChunkMap = makeChunkMap( {std::make_shared<ChunkInfo>( ChunkType{uuid(), ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)}, @@ -160,14 +486,101 @@ TEST_F(ChunkMapTest, TestEnumerateOverlappingChunks) { auto min = BSON("a" << -50); auto max = BSON("a" << 150); - int count = 0; newChunkMap.forEachOverlappingChunk(min, max, true, [&](const auto& chunk) { count++; return true; }); + ASSERT_EQ(count, 3); + min = BSON("a" << -50); + max = BSON("a" << getShardKeyPattern().globalMax()); + count = 0; + newChunkMap.forEachOverlappingChunk(min, max, false, [&](const auto& chunk) { + count++; + return true; + }); ASSERT_EQ(count, 3); + + min = BSON("a" << 50); + max = BSON("a" << 100); + count = 0; + newChunkMap.forEachOverlappingChunk(min, max, true, [&](const auto& chunk) { + count++; + return true; + }); + ASSERT_EQ(count, 2); + + min = BSON("a" << 50); + max = BSON("a" << 100); + count = 0; + newChunkMap.forEachOverlappingChunk(min, max, false, [&](const auto& chunk) { + count++; + return true; + }); + ASSERT_EQ(count, 1); +} + +TEST_F(ChunkMapTest, ForEachNoShardKey) { + auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); + + const auto chunkMap = makeChunkMap(chunks); + + auto lastChunkIdx = std::max(_random.nextInt64(chunks.size()), static_cast<int64_t>(1)); + + int i = 0; + chunkMap.forEach([&](const auto& chunkInfo) { + assertEqualChunkInfo(*chunks[i], *chunkInfo); + return ++i < lastChunkIdx; + }); + + ASSERT_EQ(i, lastChunkIdx); +} + +TEST_F(ChunkMapTest, ForEachWithShardKey) { + auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); + + const auto chunkMap = makeChunkMap(chunks); + + auto firstChunkIdx = static_cast<size_t>(_random.nextInt64(chunks.size())); + const auto& firstChunk = chunks[firstChunkIdx]; + auto skey = calculateIntermediateShardKey( + firstChunk->getMin(), firstChunk->getMax(), 0.2 /* minKeyProb */); + + size_t i = firstChunkIdx; + auto lastChunkIdx = firstChunkIdx + + std::max(_random.nextInt64(chunks.size() - firstChunkIdx), static_cast<int64_t>(1)); + chunkMap.forEach( + [&](const auto& chunkInfo) { + assertEqualChunkInfo(*chunks[i], *chunkInfo); + return ++i < lastChunkIdx; + }, + skey); + + ASSERT_EQ(i, lastChunkIdx); } +TEST_F(ChunkMapTest, TestEnumerateOverlappingChunksRandom) { + auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); + + const auto chunkMap = makeChunkMap(chunks); + + auto firstChunkIt = chunks.begin() + _random.nextInt64(chunks.size()); + auto lastChunkIt = firstChunkIt + _random.nextInt64(std::distance(firstChunkIt, chunks.end())); + + auto minBound = calculateIntermediateShardKey( + (*firstChunkIt)->getMin(), (*firstChunkIt)->getMax(), 0.2 /* minKeyProb */); + auto maxBound = calculateIntermediateShardKey( + (*lastChunkIt)->getMin(), (*lastChunkIt)->getMax(), 0.2 /* minKeyProb */); + + auto it = firstChunkIt; + chunkMap.forEachOverlappingChunk(minBound, maxBound, true, [&](const auto& chunkInfoPtr) { + assertEqualChunkInfo(**(it++), *chunkInfoPtr); + return true; + }); + ASSERT_EQ(0, std::distance(it, std::next(lastChunkIt))); +} + +} // namespace + } // namespace mongo |