diff options
Diffstat (limited to 'src/mongo/s/chunk_map_test.cpp')
| -rw-r--r-- | src/mongo/s/chunk_map_test.cpp | 425 |
1 files changed, 21 insertions, 404 deletions
diff --git a/src/mongo/s/chunk_map_test.cpp b/src/mongo/s/chunk_map_test.cpp index 78b2ca42097..6514fc00745 100644 --- a/src/mongo/s/chunk_map_test.cpp +++ b/src/mongo/s/chunk_map_test.cpp @@ -27,104 +27,18 @@ * it in the license file. */ -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest - #include "mongo/platform/basic.h" -#include "mongo/logv2/log.h" -#include "mongo/platform/random.h" #include "mongo/s/chunk_manager.h" -#include "mongo/s/chunk_writes_tracker.h" -#include "mongo/s/chunks_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { -using chunks_test_util::assertEqualChunkInfo; -using chunks_test_util::calculateCollVersion; -using chunks_test_util::calculateIntermediateShardKey; -using chunks_test_util::genChunkVector; -using chunks_test_util::genRandomSplitPoints; -using chunks_test_util::performRandomChunkOperations; - namespace { -PseudoRandom _random{SecureRandom().nextInt64()}; - +const NamespaceString kNss("TestDB", "TestColl"); const ShardId kThisShard("testShard"); -ShardVersionMap getShardVersionMap(const ChunkMap& chunkMap) { - return chunkMap.getShardVersionsMap(); -} - -std::map<ShardId, ChunkVersion> calculateShardVersions( - const std::vector<std::shared_ptr<ChunkInfo>>& chunkVector) { - std::map<ShardId, ChunkVersion> svMap; - for (const auto& chunk : chunkVector) { - auto mapIt = svMap.find(chunk->getShardId()); - if (mapIt == svMap.end()) { - svMap.emplace(chunk->getShardId(), chunk->getLastmod()); - continue; - } - if (mapIt->second.isOlderThan(chunk->getLastmod())) { - mapIt->second = chunk->getLastmod(); - } - } - return svMap; -} - -std::vector<std::shared_ptr<ChunkInfo>> toChunkInfoPtrVector( - const std::vector<ChunkType>& chunkTypes, bool initializeWriteTrackerRandom = true) { - std::vector<std::shared_ptr<ChunkInfo>> chunkPtrs; - chunkPtrs.reserve(chunkTypes.size()); - for (const auto& chunkType : chunkTypes) { - auto chunkInfoPtr = std::make_shared<ChunkInfo>(chunkType); - if (initializeWriteTrackerRandom) { - chunkInfoPtr->getWritesTracker()->addBytesWritten(_random.nextInt64(30)); - } - chunkPtrs.push_back(std::move(chunkInfoPtr)); - } - return chunkPtrs; -} - -void validateChunkMap(const ChunkMap& chunkMap, - const std::vector<std::shared_ptr<ChunkInfo>>& chunkInfoVector) { - - // The chunkMap should contain all the chunks - ASSERT_EQ(chunkInfoVector.size(), chunkMap.size()); - - // Check collection version - const auto expectedShardVersions = calculateShardVersions(chunkInfoVector); - const auto expectedCollVersion = calculateCollVersion(expectedShardVersions); - ASSERT_EQ(expectedCollVersion, chunkMap.getVersion()); - - size_t i = 0; - chunkMap.forEach([&](const auto& chunkPtr) { - const auto& expectedChunkPtr = chunkInfoVector[i++]; - // Check that the chunk pointer is valid - ASSERT(chunkPtr.get() != nullptr); - assertEqualChunkInfo(*expectedChunkPtr, *chunkPtr); - return true; - }); - - // Validate all shard versions - const auto shardVersions = getShardVersionMap(chunkMap); - ASSERT_EQ(expectedShardVersions.size(), shardVersions.size()); - for (const auto& mapIt : shardVersions) { - ASSERT_EQ(expectedShardVersions.at(mapIt.first), mapIt.second.shardVersion); - } - - // Check that vectors are balanced in size - auto maxVectorSize = static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() * 1.5)); - auto minVectorSize = std::min( - chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2))); - - for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) { - ASSERT_GTE(chunkVectorPtr->size(), minVectorSize); - ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize); - } -} - class ChunkMapTest : public unittest::Test { public: const KeyPattern& getShardKeyPattern() const { @@ -135,36 +49,16 @@ public: return _uuid; } - const OID& collEpoch() const { - return _epoch; - } - - const Timestamp& collTimestamp() const { - return _collTimestamp; - } - - ChunkMap makeChunkMap(const std::vector<std::shared_ptr<ChunkInfo>>& chunks) const { - const auto chunkBucketSize = - static_cast<size_t>(_random.nextInt64(chunks.size() * 1.2) + 1); - LOGV2(7162701, "Creating new chunk map", "chunkBucketSize"_attr = chunkBucketSize); - return ChunkMap{collEpoch(), collTimestamp(), chunkBucketSize}.createMerged(chunks); - } - - std::vector<ChunkType> genRandomChunkVector(size_t maxNumChunks = 30, - size_t minNumChunks = 1) const { - return chunks_test_util::genRandomChunkVector( - _uuid, _epoch, _collTimestamp, maxNumChunks, minNumChunks); - } - private: - KeyPattern _shardKeyPattern{chunks_test_util::kShardKeyPattern}; + KeyPattern _shardKeyPattern{BSON("a" << 1)}; const UUID _uuid = UUID::gen(); - const OID _epoch{OID::gen()}; - const Timestamp _collTimestamp{1, 1}; }; +} // namespace + TEST_F(ChunkMapTest, TestAddChunk) { - ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; + const OID epoch = OID::gen(); + ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; auto chunk = std::make_shared<ChunkInfo>( ChunkType{uuid(), @@ -172,190 +66,18 @@ TEST_F(ChunkMapTest, TestAddChunk) { version, kThisShard}); - auto newChunkMap = makeChunkMap({chunk}); + ChunkMap chunkMap{epoch, Timestamp(1, 1)}; + auto newChunkMap = chunkMap.createMerged({chunk}); ASSERT_EQ(newChunkMap.size(), 1); - - validateChunkMap(newChunkMap, {chunk}); -} - -TEST_F(ChunkMapTest, ConstructChunkMapRandom) { - auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); - - const auto chunkMap = makeChunkMap(chunkVector); - - validateChunkMap(chunkMap, chunkVector); -} - -TEST_F(ChunkMapTest, ConstructChunkMapRandomAllChunksSameVersion) { - auto chunkVector = genRandomChunkVector(); - auto commonVersion = chunkVector.front().getVersion(); - - // Set same version on all chunks - for (auto& chunk : chunkVector) { - chunk.setVersion(commonVersion); - } - - auto chunkInfoVector = toChunkInfoPtrVector(chunkVector); - const auto expectedShardVersions = calculateShardVersions(chunkInfoVector); - const auto expectedCollVersion = calculateCollVersion(expectedShardVersions); - - ASSERT_EQ(commonVersion, expectedCollVersion); - - const auto chunkMap = makeChunkMap(chunkInfoVector); - validateChunkMap(chunkMap, chunkInfoVector); -} - -/* - * Check that constucting a ChunkMap with chunks that have mismatching timestamp fails. - */ -TEST_F(ChunkMapTest, ConstructChunkMapMismatchingTimestamp) { - auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); - - // Set a different epoch in one of the chunks - const Timestamp wrongTimestamp{Date_t::now()}; - ASSERT_NE(wrongTimestamp, collTimestamp()); - const auto wrongChunkIdx = _random.nextInt32(chunkVector.size()); - const auto oldChunk = chunkVector.at(wrongChunkIdx); - const auto oldVersion = oldChunk->getLastmod(); - const ChunkVersion wrongVersion{ - oldVersion.majorVersion(), oldVersion.minorVersion(), collEpoch(), wrongTimestamp}; - chunkVector[wrongChunkIdx] = std::make_shared<ChunkInfo>( - ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()}); - - ASSERT_THROWS_CODE( - makeChunkMap(chunkVector), AssertionException, ErrorCodes::ConflictingOperationInProgress); -} - -TEST_F(ChunkMapTest, UpdateMapNotLeaveSmallVectors) { - const ChunkVersion initialVersion{1, 0, collEpoch(), collTimestamp()}; - auto chunkVector = toChunkInfoPtrVector( - genChunkVector(uuid(), genRandomSplitPoints(8), initialVersion, 1 /*numShards*/)); - - const auto chunkBucketSize = 4; - LOGV2(7162703, "Constructing new chunk map", "chunkBucketSize"_attr = chunkBucketSize); - const auto initialChunkMap = - ChunkMap(collEpoch(), collTimestamp(), chunkBucketSize).createMerged(chunkVector); - - // Check that it contains all the chunks - ASSERT_EQ(chunkVector.size(), initialChunkMap.size()); - - auto mergedVersion = initialChunkMap.getVersion(); - mergedVersion.incMinor(); - - auto mergedChunk = std::make_shared<ChunkInfo>(ChunkType{ - uuid(), - ChunkRange{chunkVector[4]->getRange().getMin(), chunkVector.back()->getRange().getMax()}, - mergedVersion, - kThisShard}); - const auto chunkMap = initialChunkMap.createMerged({mergedChunk}); - - // Check that vectors are balanced in size - auto maxVectorSize = std::lround(chunkMap.getMaxChunkVectorSize() * 1.5); - auto minVectorSize = std::min( - chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2))); - - for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) { - ASSERT_GTE(chunkVectorPtr->size(), minVectorSize); - ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize); - } - - // Check original map is sitll valid - validateChunkMap(initialChunkMap, chunkVector); -} - - -/* - * Check that updating a ChunkMap with chunks that have mismatching timestamp fails. - */ -TEST_F(ChunkMapTest, UpdateChunkMapMismatchingTimestamp) { - auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); - - auto chunkMap = makeChunkMap(chunkVector); - auto collVersion = chunkMap.getVersion(); - - // Set a different epoch in one of the chunks - const Timestamp wrongTimestamp{Date_t::now()}; - const auto wrongChunkIdx = _random.nextInt32(chunkVector.size()); - const auto oldChunk = chunkVector.at(wrongChunkIdx); - const ChunkVersion wrongVersion{ - collVersion.majorVersion(), collVersion.minorVersion(), collEpoch(), wrongTimestamp}; - auto updateChunk = std::make_shared<ChunkInfo>( - ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()}); - - ASSERT_THROWS_CODE(chunkMap.createMerged({updateChunk}), - AssertionException, - ErrorCodes::ConflictingOperationInProgress); -} - -/* - * Check that updating a ChunkMap with chunks that have lower version fails. - */ -TEST_F(ChunkMapTest, UpdateChunkMapLowerVersion) { - auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector()); - - auto chunkMap = makeChunkMap(chunkVector); - - const auto wrongChunkIdx = _random.nextInt32(chunkVector.size()); - const auto oldChunk = chunkVector.at(wrongChunkIdx); - const ChunkVersion wrongVersion{0, 1, collEpoch(), collTimestamp()}; - auto updateChunk = std::make_shared<ChunkInfo>( - ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()}); - - ASSERT_THROWS_CODE(chunkMap.createMerged({updateChunk}), AssertionException, 626840); -} -/* - * Test update of ChunkMap with random chunk manipulation (splits/merges/moves); - */ -TEST_F(ChunkMapTest, UpdateChunkMapRandom) { - auto initialChunks = genRandomChunkVector(); - auto initialChunksInfo = toChunkInfoPtrVector(initialChunks); - - const auto initialChunkMap = makeChunkMap(initialChunksInfo); - - const auto initialShardVersions = calculateShardVersions(initialChunksInfo); - const auto initialCollVersion = calculateCollVersion(initialShardVersions); - - auto chunks = initialChunks; - - const auto maxNumChunkOps = 2 * initialChunks.size(); - const auto numChunkOps = _random.nextInt32(maxNumChunkOps); - performRandomChunkOperations(&chunks, numChunkOps); - - auto chunksInfo = toChunkInfoPtrVector(initialChunks, false /* initializeWriteTrackerRandom */); - - std::vector<std::shared_ptr<ChunkInfo>> updatedChunksInfo; - for (auto& chunkPtr : chunksInfo) { - // First overlapping chunk in the initial vector - const auto& overlapInitChunk = - **std::lower_bound(initialChunksInfo.begin(), - initialChunksInfo.end(), - ShardKeyPattern::toKeyString(chunkPtr->getRange().getMin()), - [](const auto& chunkInfo, const std::string& shardKeyString) { - return chunkInfo->getMaxKeyString() <= shardKeyString; - }); - // The new chunks inherits the written bytes from the first overlapping old chunk - chunkPtr->getWritesTracker()->addBytesWritten( - overlapInitChunk.getWritesTracker()->getBytesWritten()); - - if (!chunkPtr->getLastmod().isOlderOrEqualThan(initialCollVersion)) { - updatedChunksInfo.push_back(std::make_shared<ChunkInfo>(ChunkType{ - uuid(), chunkPtr->getRange(), chunkPtr->getLastmod(), chunkPtr->getShardId()})); - } - } - - // Create updated chunk map and validate it - auto chunkMap = initialChunkMap.createMerged(updatedChunksInfo); - validateChunkMap(chunkMap, chunksInfo); - - // Check that the initialChunkMap is still valid and usable - validateChunkMap(initialChunkMap, initialChunksInfo); } TEST_F(ChunkMapTest, TestEnumerateAllChunks) { - ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; + const OID epoch = OID::gen(); + ChunkMap chunkMap{epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; - auto newChunkMap = makeChunkMap( + auto newChunkMap = chunkMap.createMerged( {std::make_shared<ChunkInfo>( ChunkType{uuid(), ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)}, @@ -385,11 +107,12 @@ TEST_F(ChunkMapTest, TestEnumerateAllChunks) { ASSERT_EQ(count, newChunkMap.size()); } - TEST_F(ChunkMapTest, TestIntersectingChunk) { - ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; + const OID epoch = OID::gen(); + ChunkMap chunkMap{epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; - auto newChunkMap = makeChunkMap( + auto newChunkMap = chunkMap.createMerged( {std::make_shared<ChunkInfo>( ChunkType{uuid(), ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)}, @@ -412,33 +135,14 @@ TEST_F(ChunkMapTest, TestIntersectingChunk) { SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMin() == BSON("a" << 0))); ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMax() == BSON("a" << 100))); - - // findIntersectingChunks returns last chunk if invoked with MaxKey - intersectingChunk = - newChunkMap.findIntersectingChunk(BSON("a" << getShardKeyPattern().globalMax())); - ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMin() == - BSON("a" << 100))); - ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMax() == - getShardKeyPattern().globalMax())); -} - -TEST_F(ChunkMapTest, TestIntersectingChunkRandom) { - auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); - - const auto chunkMap = makeChunkMap(chunks); - - auto targetChunkIt = chunks.begin() + _random.nextInt64(chunks.size()); - auto intermediateKey = calculateIntermediateShardKey( - (*targetChunkIt)->getMin(), (*targetChunkIt)->getMax(), 0.2 /* minKeyProb */); - - auto intersectingChunkPtr = chunkMap.findIntersectingChunk(intermediateKey); - assertEqualChunkInfo(**(targetChunkIt), *intersectingChunkPtr); } TEST_F(ChunkMapTest, TestEnumerateOverlappingChunks) { - ChunkVersion version{1, 0, collEpoch(), collTimestamp()}; + const OID epoch = OID::gen(); + ChunkMap chunkMap{epoch, Timestamp(1, 1)}; + ChunkVersion version{1, 0, epoch, Timestamp(1, 1)}; - auto newChunkMap = makeChunkMap( + auto newChunkMap = chunkMap.createMerged( {std::make_shared<ChunkInfo>( ChunkType{uuid(), ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)}, @@ -456,101 +160,14 @@ TEST_F(ChunkMapTest, TestEnumerateOverlappingChunks) { auto min = BSON("a" << -50); auto max = BSON("a" << 150); + int count = 0; newChunkMap.forEachOverlappingChunk(min, max, true, [&](const auto& chunk) { count++; return true; }); - ASSERT_EQ(count, 3); - min = BSON("a" << -50); - max = BSON("a" << getShardKeyPattern().globalMax()); - count = 0; - newChunkMap.forEachOverlappingChunk(min, max, false, [&](const auto& chunk) { - count++; - return true; - }); ASSERT_EQ(count, 3); - - min = BSON("a" << 50); - max = BSON("a" << 100); - count = 0; - newChunkMap.forEachOverlappingChunk(min, max, true, [&](const auto& chunk) { - count++; - return true; - }); - ASSERT_EQ(count, 2); - - min = BSON("a" << 50); - max = BSON("a" << 100); - count = 0; - newChunkMap.forEachOverlappingChunk(min, max, false, [&](const auto& chunk) { - count++; - return true; - }); - ASSERT_EQ(count, 1); -} - -TEST_F(ChunkMapTest, ForEachNoShardKey) { - auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); - - const auto chunkMap = makeChunkMap(chunks); - - auto lastChunkIdx = std::max(_random.nextInt64(chunks.size()), static_cast<int64_t>(1)); - - int i = 0; - chunkMap.forEach([&](const auto& chunkInfo) { - assertEqualChunkInfo(*chunks[i], *chunkInfo); - return ++i < lastChunkIdx; - }); - - ASSERT_EQ(i, lastChunkIdx); -} - -TEST_F(ChunkMapTest, ForEachWithShardKey) { - auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); - - const auto chunkMap = makeChunkMap(chunks); - - auto firstChunkIdx = static_cast<size_t>(_random.nextInt64(chunks.size())); - const auto& firstChunk = chunks[firstChunkIdx]; - auto skey = calculateIntermediateShardKey( - firstChunk->getMin(), firstChunk->getMax(), 0.2 /* minKeyProb */); - - size_t i = firstChunkIdx; - auto lastChunkIdx = firstChunkIdx + - std::max(_random.nextInt64(chunks.size() - firstChunkIdx), static_cast<int64_t>(1)); - chunkMap.forEach( - [&](const auto& chunkInfo) { - assertEqualChunkInfo(*chunks[i], *chunkInfo); - return ++i < lastChunkIdx; - }, - skey); - - ASSERT_EQ(i, lastChunkIdx); -} - -TEST_F(ChunkMapTest, TestEnumerateOverlappingChunksRandom) { - auto chunks = toChunkInfoPtrVector(genRandomChunkVector()); - - const auto chunkMap = makeChunkMap(chunks); - - auto firstChunkIt = chunks.begin() + _random.nextInt64(chunks.size()); - auto lastChunkIt = firstChunkIt + _random.nextInt64(std::distance(firstChunkIt, chunks.end())); - - auto minBound = calculateIntermediateShardKey( - (*firstChunkIt)->getMin(), (*firstChunkIt)->getMax(), 0.2 /* minKeyProb */); - auto maxBound = calculateIntermediateShardKey( - (*lastChunkIt)->getMin(), (*lastChunkIt)->getMax(), 0.2 /* minKeyProb */); - - auto it = firstChunkIt; - chunkMap.forEachOverlappingChunk(minBound, maxBound, true, [&](const auto& chunkInfoPtr) { - assertEqualChunkInfo(**(it++), *chunkInfoPtr); - return true; - }); - ASSERT_EQ(0, std::distance(it, std::next(lastChunkIt))); } -} // namespace - } // namespace mongo |