Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 21 insertions, 404 deletions

diff --git a/src/mongo/s/chunk_map_test.cpp b/src/mongo/s/chunk_map_test.cpp
index 78b2ca42097..6514fc00745 100644
--- a/src/mongo/s/chunk_map_test.cpp
+++ b/src/mongo/s/chunk_map_test.cpp
@@ -27,104 +27,18 @@
  *    it in the license file.
  */
 
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest
-
 #include "mongo/platform/basic.h"
 
-#include "mongo/logv2/log.h"
-#include "mongo/platform/random.h"
 #include "mongo/s/chunk_manager.h"
-#include "mongo/s/chunk_writes_tracker.h"
-#include "mongo/s/chunks_test_util.h"
 #include "mongo/unittest/unittest.h"
 
 namespace mongo {
 
-using chunks_test_util::assertEqualChunkInfo;
-using chunks_test_util::calculateCollVersion;
-using chunks_test_util::calculateIntermediateShardKey;
-using chunks_test_util::genChunkVector;
-using chunks_test_util::genRandomSplitPoints;
-using chunks_test_util::performRandomChunkOperations;
-
 namespace {
 
-PseudoRandom _random{SecureRandom().nextInt64()};
-
+const NamespaceString kNss("TestDB", "TestColl");
 const ShardId kThisShard("testShard");
 
-ShardVersionMap getShardVersionMap(const ChunkMap& chunkMap) {
-    return chunkMap.getShardVersionsMap();
-}
-
-std::map<ShardId, ChunkVersion> calculateShardVersions(
-    const std::vector<std::shared_ptr<ChunkInfo>>& chunkVector) {
-    std::map<ShardId, ChunkVersion> svMap;
-    for (const auto& chunk : chunkVector) {
-        auto mapIt = svMap.find(chunk->getShardId());
-        if (mapIt == svMap.end()) {
-            svMap.emplace(chunk->getShardId(), chunk->getLastmod());
-            continue;
-        }
-        if (mapIt->second.isOlderThan(chunk->getLastmod())) {
-            mapIt->second = chunk->getLastmod();
-        }
-    }
-    return svMap;
-}
-
-std::vector<std::shared_ptr<ChunkInfo>> toChunkInfoPtrVector(
-    const std::vector<ChunkType>& chunkTypes, bool initializeWriteTrackerRandom = true) {
-    std::vector<std::shared_ptr<ChunkInfo>> chunkPtrs;
-    chunkPtrs.reserve(chunkTypes.size());
-    for (const auto& chunkType : chunkTypes) {
-        auto chunkInfoPtr = std::make_shared<ChunkInfo>(chunkType);
-        if (initializeWriteTrackerRandom) {
-            chunkInfoPtr->getWritesTracker()->addBytesWritten(_random.nextInt64(30));
-        }
-        chunkPtrs.push_back(std::move(chunkInfoPtr));
-    }
-    return chunkPtrs;
-}
-
-void validateChunkMap(const ChunkMap& chunkMap,
-                      const std::vector<std::shared_ptr<ChunkInfo>>& chunkInfoVector) {
-
-    // The chunkMap should contain all the chunks
-    ASSERT_EQ(chunkInfoVector.size(), chunkMap.size());
-
-    // Check collection version
-    const auto expectedShardVersions = calculateShardVersions(chunkInfoVector);
-    const auto expectedCollVersion = calculateCollVersion(expectedShardVersions);
-    ASSERT_EQ(expectedCollVersion, chunkMap.getVersion());
-
-    size_t i = 0;
-    chunkMap.forEach([&](const auto& chunkPtr) {
-        const auto& expectedChunkPtr = chunkInfoVector[i++];
-        // Check that the chunk pointer is valid
-        ASSERT(chunkPtr.get() != nullptr);
-        assertEqualChunkInfo(*expectedChunkPtr, *chunkPtr);
-        return true;
-    });
-
-    // Validate all shard versions
-    const auto shardVersions = getShardVersionMap(chunkMap);
-    ASSERT_EQ(expectedShardVersions.size(), shardVersions.size());
-    for (const auto& mapIt : shardVersions) {
-        ASSERT_EQ(expectedShardVersions.at(mapIt.first), mapIt.second.shardVersion);
-    }
-
-    // Check that vectors are balanced in size
-    auto maxVectorSize = static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() * 1.5));
-    auto minVectorSize = std::min(
-        chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2)));
-
-    for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) {
-        ASSERT_GTE(chunkVectorPtr->size(), minVectorSize);
-        ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize);
-    }
-}
-
 class ChunkMapTest : public unittest::Test {
 public:
     const KeyPattern& getShardKeyPattern() const {
@@ -135,36 +49,16 @@ public:
         return _uuid;
     }
 
-    const OID& collEpoch() const {
-        return _epoch;
-    }
-
-    const Timestamp& collTimestamp() const {
-        return _collTimestamp;
-    }
-
-    ChunkMap makeChunkMap(const std::vector<std::shared_ptr<ChunkInfo>>& chunks) const {
-        const auto chunkBucketSize =
-            static_cast<size_t>(_random.nextInt64(chunks.size() * 1.2) + 1);
-        LOGV2(7162701, "Creating new chunk map", "chunkBucketSize"_attr = chunkBucketSize);
-        return ChunkMap{collEpoch(), collTimestamp(), chunkBucketSize}.createMerged(chunks);
-    }
-
-    std::vector<ChunkType> genRandomChunkVector(size_t maxNumChunks = 30,
-                                                size_t minNumChunks = 1) const {
-        return chunks_test_util::genRandomChunkVector(
-            _uuid, _epoch, _collTimestamp, maxNumChunks, minNumChunks);
-    }
-
 private:
-    KeyPattern _shardKeyPattern{chunks_test_util::kShardKeyPattern};
+    KeyPattern _shardKeyPattern{BSON("a" << 1)};
     const UUID _uuid = UUID::gen();
-    const OID _epoch{OID::gen()};
-    const Timestamp _collTimestamp{1, 1};
 };
 
+}  // namespace
+
 TEST_F(ChunkMapTest, TestAddChunk) {
-    ChunkVersion version{1, 0, collEpoch(), collTimestamp()};
+    const OID epoch = OID::gen();
+    ChunkVersion version{1, 0, epoch, Timestamp(1, 1)};
 
     auto chunk = std::make_shared<ChunkInfo>(
         ChunkType{uuid(),
@@ -172,190 +66,18 @@ TEST_F(ChunkMapTest, TestAddChunk) {
                   version,
                   kThisShard});
 
-    auto newChunkMap = makeChunkMap({chunk});
+    ChunkMap chunkMap{epoch, Timestamp(1, 1)};
+    auto newChunkMap = chunkMap.createMerged({chunk});
 
     ASSERT_EQ(newChunkMap.size(), 1);
-
-    validateChunkMap(newChunkMap, {chunk});
-}
-
-TEST_F(ChunkMapTest, ConstructChunkMapRandom) {
-    auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector());
-
-    const auto chunkMap = makeChunkMap(chunkVector);
-
-    validateChunkMap(chunkMap, chunkVector);
-}
-
-TEST_F(ChunkMapTest, ConstructChunkMapRandomAllChunksSameVersion) {
-    auto chunkVector = genRandomChunkVector();
-    auto commonVersion = chunkVector.front().getVersion();
-
-    // Set same version on all chunks
-    for (auto& chunk : chunkVector) {
-        chunk.setVersion(commonVersion);
-    }
-
-    auto chunkInfoVector = toChunkInfoPtrVector(chunkVector);
-    const auto expectedShardVersions = calculateShardVersions(chunkInfoVector);
-    const auto expectedCollVersion = calculateCollVersion(expectedShardVersions);
-
-    ASSERT_EQ(commonVersion, expectedCollVersion);
-
-    const auto chunkMap = makeChunkMap(chunkInfoVector);
-    validateChunkMap(chunkMap, chunkInfoVector);
-}
-
-/*
- * Check that constucting a ChunkMap with chunks that have mismatching timestamp fails.
- */
-TEST_F(ChunkMapTest, ConstructChunkMapMismatchingTimestamp) {
-    auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector());
-
-    // Set a different epoch in one of the chunks
-    const Timestamp wrongTimestamp{Date_t::now()};
-    ASSERT_NE(wrongTimestamp, collTimestamp());
-    const auto wrongChunkIdx = _random.nextInt32(chunkVector.size());
-    const auto oldChunk = chunkVector.at(wrongChunkIdx);
-    const auto oldVersion = oldChunk->getLastmod();
-    const ChunkVersion wrongVersion{
-        oldVersion.majorVersion(), oldVersion.minorVersion(), collEpoch(), wrongTimestamp};
-    chunkVector[wrongChunkIdx] = std::make_shared<ChunkInfo>(
-        ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()});
-
-    ASSERT_THROWS_CODE(
-        makeChunkMap(chunkVector), AssertionException, ErrorCodes::ConflictingOperationInProgress);
-}
-
-TEST_F(ChunkMapTest, UpdateMapNotLeaveSmallVectors) {
-    const ChunkVersion initialVersion{1, 0, collEpoch(), collTimestamp()};
-    auto chunkVector = toChunkInfoPtrVector(
-        genChunkVector(uuid(), genRandomSplitPoints(8), initialVersion, 1 /*numShards*/));
-
-    const auto chunkBucketSize = 4;
-    LOGV2(7162703, "Constructing new chunk map", "chunkBucketSize"_attr = chunkBucketSize);
-    const auto initialChunkMap =
-        ChunkMap(collEpoch(), collTimestamp(), chunkBucketSize).createMerged(chunkVector);
-
-    // Check that it contains all the chunks
-    ASSERT_EQ(chunkVector.size(), initialChunkMap.size());
-
-    auto mergedVersion = initialChunkMap.getVersion();
-    mergedVersion.incMinor();
-
-    auto mergedChunk = std::make_shared<ChunkInfo>(ChunkType{
-        uuid(),
-        ChunkRange{chunkVector[4]->getRange().getMin(), chunkVector.back()->getRange().getMax()},
-        mergedVersion,
-        kThisShard});
-    const auto chunkMap = initialChunkMap.createMerged({mergedChunk});
-
-    // Check that vectors are balanced in size
-    auto maxVectorSize = std::lround(chunkMap.getMaxChunkVectorSize() * 1.5);
-    auto minVectorSize = std::min(
-        chunkMap.size(), static_cast<size_t>(std::lround(chunkMap.getMaxChunkVectorSize() / 2)));
-
-    for (const auto& [maxKeyString, chunkVectorPtr] : chunkMap.getChunkVectorMap()) {
-        ASSERT_GTE(chunkVectorPtr->size(), minVectorSize);
-        ASSERT_LTE(chunkVectorPtr->size(), maxVectorSize);
-    }
-
-    // Check original map is sitll valid
-    validateChunkMap(initialChunkMap, chunkVector);
-}
-
-
-/*
- * Check that updating a ChunkMap with chunks that have mismatching timestamp fails.
- */
-TEST_F(ChunkMapTest, UpdateChunkMapMismatchingTimestamp) {
-    auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector());
-
-    auto chunkMap = makeChunkMap(chunkVector);
-    auto collVersion = chunkMap.getVersion();
-
-    // Set a different epoch in one of the chunks
-    const Timestamp wrongTimestamp{Date_t::now()};
-    const auto wrongChunkIdx = _random.nextInt32(chunkVector.size());
-    const auto oldChunk = chunkVector.at(wrongChunkIdx);
-    const ChunkVersion wrongVersion{
-        collVersion.majorVersion(), collVersion.minorVersion(), collEpoch(), wrongTimestamp};
-    auto updateChunk = std::make_shared<ChunkInfo>(
-        ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()});
-
-    ASSERT_THROWS_CODE(chunkMap.createMerged({updateChunk}),
-                       AssertionException,
-                       ErrorCodes::ConflictingOperationInProgress);
-}
-
-/*
- * Check that updating a ChunkMap with chunks that have lower version fails.
- */
-TEST_F(ChunkMapTest, UpdateChunkMapLowerVersion) {
-    auto chunkVector = toChunkInfoPtrVector(genRandomChunkVector());
-
-    auto chunkMap = makeChunkMap(chunkVector);
-
-    const auto wrongChunkIdx = _random.nextInt32(chunkVector.size());
-    const auto oldChunk = chunkVector.at(wrongChunkIdx);
-    const ChunkVersion wrongVersion{0, 1, collEpoch(), collTimestamp()};
-    auto updateChunk = std::make_shared<ChunkInfo>(
-        ChunkType{uuid(), oldChunk->getRange(), wrongVersion, oldChunk->getShardId()});
-
-    ASSERT_THROWS_CODE(chunkMap.createMerged({updateChunk}), AssertionException, 626840);
-}
-/*
- * Test update of ChunkMap with random chunk manipulation (splits/merges/moves);
- */
-TEST_F(ChunkMapTest, UpdateChunkMapRandom) {
-    auto initialChunks = genRandomChunkVector();
-    auto initialChunksInfo = toChunkInfoPtrVector(initialChunks);
-
-    const auto initialChunkMap = makeChunkMap(initialChunksInfo);
-
-    const auto initialShardVersions = calculateShardVersions(initialChunksInfo);
-    const auto initialCollVersion = calculateCollVersion(initialShardVersions);
-
-    auto chunks = initialChunks;
-
-    const auto maxNumChunkOps = 2 * initialChunks.size();
-    const auto numChunkOps = _random.nextInt32(maxNumChunkOps);
-    performRandomChunkOperations(&chunks, numChunkOps);
-
-    auto chunksInfo = toChunkInfoPtrVector(initialChunks, false /* initializeWriteTrackerRandom */);
-
-    std::vector<std::shared_ptr<ChunkInfo>> updatedChunksInfo;
-    for (auto& chunkPtr : chunksInfo) {
-        // First overlapping chunk in the initial vector
-        const auto& overlapInitChunk =
-            **std::lower_bound(initialChunksInfo.begin(),
-                               initialChunksInfo.end(),
-                               ShardKeyPattern::toKeyString(chunkPtr->getRange().getMin()),
-                               [](const auto& chunkInfo, const std::string& shardKeyString) {
-                                   return chunkInfo->getMaxKeyString() <= shardKeyString;
-                               });
-        // The new chunks inherits the written bytes from the first overlapping old chunk
-        chunkPtr->getWritesTracker()->addBytesWritten(
-            overlapInitChunk.getWritesTracker()->getBytesWritten());
-
-        if (!chunkPtr->getLastmod().isOlderOrEqualThan(initialCollVersion)) {
-            updatedChunksInfo.push_back(std::make_shared<ChunkInfo>(ChunkType{
-                uuid(), chunkPtr->getRange(), chunkPtr->getLastmod(), chunkPtr->getShardId()}));
-        }
-    }
-
-    // Create updated chunk map and validate it
-    auto chunkMap = initialChunkMap.createMerged(updatedChunksInfo);
-    validateChunkMap(chunkMap, chunksInfo);
-
-    // Check that the initialChunkMap is still valid and usable
-    validateChunkMap(initialChunkMap, initialChunksInfo);
 }
 
 TEST_F(ChunkMapTest, TestEnumerateAllChunks) {
-    ChunkVersion version{1, 0, collEpoch(), collTimestamp()};
+    const OID epoch = OID::gen();
+    ChunkMap chunkMap{epoch, Timestamp(1, 1)};
+    ChunkVersion version{1, 0, epoch, Timestamp(1, 1)};
 
-    auto newChunkMap = makeChunkMap(
+    auto newChunkMap = chunkMap.createMerged(
         {std::make_shared<ChunkInfo>(
              ChunkType{uuid(),
                        ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)},
@@ -385,11 +107,12 @@ TEST_F(ChunkMapTest, TestEnumerateAllChunks) {
     ASSERT_EQ(count, newChunkMap.size());
 }
 
-
 TEST_F(ChunkMapTest, TestIntersectingChunk) {
-    ChunkVersion version{1, 0, collEpoch(), collTimestamp()};
+    const OID epoch = OID::gen();
+    ChunkMap chunkMap{epoch, Timestamp(1, 1)};
+    ChunkVersion version{1, 0, epoch, Timestamp(1, 1)};
 
-    auto newChunkMap = makeChunkMap(
+    auto newChunkMap = chunkMap.createMerged(
         {std::make_shared<ChunkInfo>(
              ChunkType{uuid(),
                        ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)},
@@ -412,33 +135,14 @@ TEST_F(ChunkMapTest, TestIntersectingChunk) {
         SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMin() == BSON("a" << 0)));
     ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMax() ==
                                                        BSON("a" << 100)));
-
-    // findIntersectingChunks returns last chunk if invoked with MaxKey
-    intersectingChunk =
-        newChunkMap.findIntersectingChunk(BSON("a" << getShardKeyPattern().globalMax()));
-    ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMin() ==
-                                                       BSON("a" << 100)));
-    ASSERT(SimpleBSONObjComparator::kInstance.evaluate(intersectingChunk->getMax() ==
-                                                       getShardKeyPattern().globalMax()));
-}
-
-TEST_F(ChunkMapTest, TestIntersectingChunkRandom) {
-    auto chunks = toChunkInfoPtrVector(genRandomChunkVector());
-
-    const auto chunkMap = makeChunkMap(chunks);
-
-    auto targetChunkIt = chunks.begin() + _random.nextInt64(chunks.size());
-    auto intermediateKey = calculateIntermediateShardKey(
-        (*targetChunkIt)->getMin(), (*targetChunkIt)->getMax(), 0.2 /* minKeyProb */);
-
-    auto intersectingChunkPtr = chunkMap.findIntersectingChunk(intermediateKey);
-    assertEqualChunkInfo(**(targetChunkIt), *intersectingChunkPtr);
 }
 
 TEST_F(ChunkMapTest, TestEnumerateOverlappingChunks) {
-    ChunkVersion version{1, 0, collEpoch(), collTimestamp()};
+    const OID epoch = OID::gen();
+    ChunkMap chunkMap{epoch, Timestamp(1, 1)};
+    ChunkVersion version{1, 0, epoch, Timestamp(1, 1)};
 
-    auto newChunkMap = makeChunkMap(
+    auto newChunkMap = chunkMap.createMerged(
         {std::make_shared<ChunkInfo>(
              ChunkType{uuid(),
                        ChunkRange{getShardKeyPattern().globalMin(), BSON("a" << 0)},
@@ -456,101 +160,14 @@ TEST_F(ChunkMapTest, TestEnumerateOverlappingChunks) {
 
     auto min = BSON("a" << -50);
     auto max = BSON("a" << 150);
+
     int count = 0;
     newChunkMap.forEachOverlappingChunk(min, max, true, [&](const auto& chunk) {
         count++;
         return true;
     });
-    ASSERT_EQ(count, 3);
 
-    min = BSON("a" << -50);
-    max = BSON("a" << getShardKeyPattern().globalMax());
-    count = 0;
-    newChunkMap.forEachOverlappingChunk(min, max, false, [&](const auto& chunk) {
-        count++;
-        return true;
-    });
     ASSERT_EQ(count, 3);
-
-    min = BSON("a" << 50);
-    max = BSON("a" << 100);
-    count = 0;
-    newChunkMap.forEachOverlappingChunk(min, max, true, [&](const auto& chunk) {
-        count++;
-        return true;
-    });
-    ASSERT_EQ(count, 2);
-
-    min = BSON("a" << 50);
-    max = BSON("a" << 100);
-    count = 0;
-    newChunkMap.forEachOverlappingChunk(min, max, false, [&](const auto& chunk) {
-        count++;
-        return true;
-    });
-    ASSERT_EQ(count, 1);
-}
-
-TEST_F(ChunkMapTest, ForEachNoShardKey) {
-    auto chunks = toChunkInfoPtrVector(genRandomChunkVector());
-
-    const auto chunkMap = makeChunkMap(chunks);
-
-    auto lastChunkIdx = std::max(_random.nextInt64(chunks.size()), static_cast<int64_t>(1));
-
-    int i = 0;
-    chunkMap.forEach([&](const auto& chunkInfo) {
-        assertEqualChunkInfo(*chunks[i], *chunkInfo);
-        return ++i < lastChunkIdx;
-    });
-
-    ASSERT_EQ(i, lastChunkIdx);
-}
-
-TEST_F(ChunkMapTest, ForEachWithShardKey) {
-    auto chunks = toChunkInfoPtrVector(genRandomChunkVector());
-
-    const auto chunkMap = makeChunkMap(chunks);
-
-    auto firstChunkIdx = static_cast<size_t>(_random.nextInt64(chunks.size()));
-    const auto& firstChunk = chunks[firstChunkIdx];
-    auto skey = calculateIntermediateShardKey(
-        firstChunk->getMin(), firstChunk->getMax(), 0.2 /* minKeyProb */);
-
-    size_t i = firstChunkIdx;
-    auto lastChunkIdx = firstChunkIdx +
-        std::max(_random.nextInt64(chunks.size() - firstChunkIdx), static_cast<int64_t>(1));
-    chunkMap.forEach(
-        [&](const auto& chunkInfo) {
-            assertEqualChunkInfo(*chunks[i], *chunkInfo);
-            return ++i < lastChunkIdx;
-        },
-        skey);
-
-    ASSERT_EQ(i, lastChunkIdx);
-}
-
-TEST_F(ChunkMapTest, TestEnumerateOverlappingChunksRandom) {
-    auto chunks = toChunkInfoPtrVector(genRandomChunkVector());
-
-    const auto chunkMap = makeChunkMap(chunks);
-
-    auto firstChunkIt = chunks.begin() + _random.nextInt64(chunks.size());
-    auto lastChunkIt = firstChunkIt + _random.nextInt64(std::distance(firstChunkIt, chunks.end()));
-
-    auto minBound = calculateIntermediateShardKey(
-        (*firstChunkIt)->getMin(), (*firstChunkIt)->getMax(), 0.2 /* minKeyProb */);
-    auto maxBound = calculateIntermediateShardKey(
-        (*lastChunkIt)->getMin(), (*lastChunkIt)->getMax(), 0.2 /* minKeyProb */);
-
-    auto it = firstChunkIt;
-    chunkMap.forEachOverlappingChunk(minBound, maxBound, true, [&](const auto& chunkInfoPtr) {
-        assertEqualChunkInfo(**(it++), *chunkInfoPtr);
-        return true;
-    });
-    ASSERT_EQ(0, std::distance(it, std::next(lastChunkIt)));
 }
 
-}  // namespace
-
 }  // namespace mongo