Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 76 insertions, 159 deletions

diff --git a/src/mongo/db/catalog/index_consistency.cpp b/src/mongo/db/catalog/index_consistency.cpp
index 64b2dfe40ba..55e35b744f8 100644
--- a/src/mongo/db/catalog/index_consistency.cpp
+++ b/src/mongo/db/catalog/index_consistency.cpp
@@ -38,7 +38,7 @@
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/catalog/index_repair.h"
 #include "mongo/db/catalog/validate_gen.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/index/index_descriptor.h"
@@ -106,7 +106,7 @@ IndexConsistency::IndexConsistency(OperationContext* opCtx,
 
 void IndexConsistency::addMultikeyMetadataPath(const KeyString::Value& ks, IndexInfo* indexInfo) {
     auto hash = _hashKeyString(ks, indexInfo->indexNameHash);
-    if (MONGO_unlikely(_validateState->logDiagnostics())) {
+    if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
         LOGV2(6208500,
               "[validate](multikeyMetadataPath) Adding with the hash",
               "hash"_attr = hash,
@@ -118,7 +118,7 @@ void IndexConsistency::addMultikeyMetadataPath(const KeyString::Value& ks, Index
 void IndexConsistency::removeMultikeyMetadataPath(const KeyString::Value& ks,
                                                   IndexInfo* indexInfo) {
     auto hash = _hashKeyString(ks, indexInfo->indexNameHash);
-    if (MONGO_unlikely(_validateState->logDiagnostics())) {
+    if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
         LOGV2(6208501,
               "[validate](multikeyMetadataPath) Removing with the hash",
               "hash"_attr = hash,
@@ -132,26 +132,9 @@ size_t IndexConsistency::getMultikeyMetadataPathCount(IndexInfo* indexInfo) {
 }
 
 bool IndexConsistency::haveEntryMismatch() const {
-    bool haveMismatch =
-        std::any_of(_indexKeyBuckets.begin(),
-                    _indexKeyBuckets.end(),
-                    [](const IndexKeyBucket& bucket) -> bool { return bucket.indexKeyCount; });
-
-    if (haveMismatch && _validateState->logDiagnostics()) {
-        for (size_t i = 0; i < _indexKeyBuckets.size(); i++) {
-            if (_indexKeyBuckets[i].indexKeyCount == 0) {
-                continue;
-            }
-
-            LOGV2(7404500,
-                  "[validate](bucket entry mismatch)",
-                  "hash"_attr = i,
-                  "indexKeyCount"_attr = _indexKeyBuckets[i].indexKeyCount,
-                  "bucketBytesSize"_attr = _indexKeyBuckets[i].bucketSizeBytes);
-        }
-    }
-
-    return haveMismatch;
+    return std::any_of(_indexKeyBuckets.begin(),
+                       _indexKeyBuckets.end(),
+                       [](const IndexKeyBucket& bucket) -> bool { return bucket.indexKeyCount; });
 }
 
 void IndexConsistency::setSecondPhase() {
@@ -209,7 +192,7 @@ void IndexConsistency::repairMissingIndexEntries(OperationContext* opCtx,
     }
 }
 
-void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResults* results) {
+void IndexConsistency::addIndexEntryErrors(ValidateResults* results) {
     invariant(!_firstPhase);
 
     // We'll report up to 1MB for extra index entry errors and missing index entry errors.
@@ -223,26 +206,11 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
         numExtraIndexEntryErrors += item.second.size();
     }
 
-    // Sort missing index entries by size so we can process in order of increasing size and return
-    // as many as possible within memory limits.
-    using MissingIt = decltype(_missingIndexEntries)::const_iterator;
-    std::vector<MissingIt> missingIndexEntriesBySize;
-    missingIndexEntriesBySize.reserve(_missingIndexEntries.size());
-    for (auto it = _missingIndexEntries.begin(); it != _missingIndexEntries.end(); ++it) {
-        missingIndexEntriesBySize.push_back(it);
-    }
-    std::sort(missingIndexEntriesBySize.begin(),
-              missingIndexEntriesBySize.end(),
-              [](const MissingIt& a, const MissingIt& b) {
-                  return a->second.keyString.getSize() < b->second.keyString.getSize();
-              });
-
     // Inform which indexes have inconsistencies and add the BSON objects of the inconsistent index
     // entries to the results vector.
     bool missingIndexEntrySizeLimitWarning = false;
-    bool first = true;
-    for (const auto& missingIndexEntry : missingIndexEntriesBySize) {
-        const IndexEntryInfo& entryInfo = missingIndexEntry->second;
+    for (const auto& missingIndexEntry : _missingIndexEntries) {
+        const IndexEntryInfo& entryInfo = missingIndexEntry.second;
         KeyString::Value ks = entryInfo.keyString;
         auto indexKey =
             KeyString::toBsonSafe(ks.getBuffer(), ks.getSize(), entryInfo.ord, ks.getTypeBits());
@@ -253,9 +221,8 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
                                             entryInfo.idKey);
 
         numMissingIndexEntriesSizeBytes += entry.objsize();
-        if (first || numMissingIndexEntriesSizeBytes <= kErrorSizeBytes) {
+        if (numMissingIndexEntriesSizeBytes <= kErrorSizeBytes) {
             results->missingIndexEntries.push_back(entry);
-            first = false;
         } else if (!missingIndexEntrySizeLimitWarning) {
             StringBuilder ss;
             ss << "Not all missing index entry inconsistencies are listed due to size limitations.";
@@ -264,8 +231,6 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
             missingIndexEntrySizeLimitWarning = true;
         }
 
-        _printMetadata(opCtx, results, entryInfo);
-
         std::string indexName = entry["indexName"].String();
         if (!results->indexResultsMap.at(indexName).valid) {
             continue;
@@ -278,58 +243,33 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
         results->indexResultsMap.at(indexName).valid = false;
     }
 
-    // Sort extra index entries by size so we can process in order of increasing size and return as
-    // many as possible within memory limits.
-    using ExtraIt = SimpleBSONObjSet::const_iterator;
-    std::vector<ExtraIt> extraIndexEntriesBySize;
-    // Since the extra entries are stored in a map of sets, we have to iterate the entries in the
-    // map and sum the size of the sets in order to get the total number. Given that we can have at
-    // most 64 indexes per collection, and the total number of entries could potentially be in the
-    // millions, we expect that iterating the map will be much less costly than the additional
-    // allocations and copies that could result from not calling 'reserve' on the vector.
-    size_t totalExtraIndexEntriesCount =
-        std::accumulate(_extraIndexEntries.begin(),
-                        _extraIndexEntries.end(),
-                        0,
-                        [](size_t total, const std::pair<IndexKey, SimpleBSONObjSet>& set) {
-                            return total + set.second.size();
-                        });
-    extraIndexEntriesBySize.reserve(totalExtraIndexEntriesCount);
+    bool extraIndexEntrySizeLimitWarning = false;
     for (const auto& extraIndexEntry : _extraIndexEntries) {
         const SimpleBSONObjSet& entries = extraIndexEntry.second;
-        for (auto it = entries.begin(); it != entries.end(); ++it) {
-            extraIndexEntriesBySize.push_back(it);
-        }
-    }
-    std::sort(extraIndexEntriesBySize.begin(),
-              extraIndexEntriesBySize.end(),
-              [](const ExtraIt& a, const ExtraIt& b) { return a->objsize() < b->objsize(); });
+        for (const auto& entry : entries) {
+            numExtraIndexEntriesSizeBytes += entry.objsize();
+            if (numExtraIndexEntriesSizeBytes <= kErrorSizeBytes) {
+                results->extraIndexEntries.push_back(entry);
+            } else if (!extraIndexEntrySizeLimitWarning) {
+                StringBuilder ss;
+                ss << "Not all extra index entry inconsistencies are listed due to size "
+                      "limitations.";
+                results->errors.push_back(ss.str());
+
+                extraIndexEntrySizeLimitWarning = true;
+            }
+
+            std::string indexName = entry["indexName"].String();
+            if (!results->indexResultsMap.at(indexName).valid) {
+                continue;
+            }
 
-    bool extraIndexEntrySizeLimitWarning = false;
-    for (const auto& entry : extraIndexEntriesBySize) {
-        numExtraIndexEntriesSizeBytes += entry->objsize();
-        if (first || numExtraIndexEntriesSizeBytes <= kErrorSizeBytes) {
-            results->extraIndexEntries.push_back(*entry);
-            first = false;
-        } else if (!extraIndexEntrySizeLimitWarning) {
             StringBuilder ss;
-            ss << "Not all extra index entry inconsistencies are listed due to size "
-                  "limitations.";
+            ss << "Index with name '" << indexName << "' has inconsistencies.";
             results->errors.push_back(ss.str());
 
-            extraIndexEntrySizeLimitWarning = true;
+            results->indexResultsMap.at(indexName).valid = false;
         }
-
-        std::string indexName = (*entry)["indexName"].String();
-        if (!results->indexResultsMap.at(indexName).valid) {
-            continue;
-        }
-
-        StringBuilder ss;
-        ss << "Index with name '" << indexName << "' has inconsistencies.";
-        results->errors.push_back(ss.str());
-
-        results->indexResultsMap.at(indexName).valid = false;
     }
 
     // Inform how many inconsistencies were detected.
@@ -362,8 +302,7 @@ void IndexConsistency::addDocumentMultikeyPaths(IndexInfo* indexInfo,
 void IndexConsistency::addDocKey(OperationContext* opCtx,
                                  const KeyString::Value& ks,
                                  IndexInfo* indexInfo,
-                                 RecordId recordId,
-                                 ValidateResults* results) {
+                                 RecordId recordId) {
     auto rawHash = ks.hash(indexInfo->indexNameHash);
     auto hashLower = rawHash % kNumHashBuckets;
     auto hashUpper = (rawHash / kNumHashBuckets) % kNumHashBuckets;
@@ -379,7 +318,7 @@ void IndexConsistency::addDocKey(OperationContext* opCtx,
         upper.bucketSizeBytes += ks.getSize();
         indexInfo->numRecords++;
 
-        if (MONGO_unlikely(_validateState->logDiagnostics())) {
+        if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
             LOGV2(4666602,
                   "[validate](record) Adding with hashes",
                   "hashUpper"_attr = hashUpper,
@@ -409,6 +348,9 @@ void IndexConsistency::addDocKey(OperationContext* opCtx,
         invariant(_missingIndexEntries.count(key) == 0);
         _missingIndexEntries.insert(
             std::make_pair(key, IndexEntryInfo(*indexInfo, recordId, idKeyBuilder.obj(), ks)));
+
+        // Prints the collection document's metadata.
+        _validateState->getCollection()->getRecordStore()->printRecordMetadata(opCtx, recordId);
     }
 }
 
@@ -432,7 +374,7 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx,
         upper.bucketSizeBytes += ks.getSize();
         indexInfo->numKeys++;
 
-        if (MONGO_unlikely(_validateState->logDiagnostics())) {
+        if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
             LOGV2(4666603,
                   "[validate](index) Adding with hashes",
                   "hashUpper"_attr = hashUpper,
@@ -481,12 +423,9 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx,
                 SimpleBSONObjSet infoSet = {info};
                 _extraIndexEntries.insert(std::make_pair(key, infoSet));
 
-                // Prints the collection document's and index entry's metadata.
-                _validateState->getCollection()->getRecordStore()->printRecordMetadata(
-                    opCtx, recordId, &(results->recordTimestamps));
-                indexInfo->accessMethod->asSortedData()
-                    ->getSortedDataInterface()
-                    ->printIndexEntryMetadata(opCtx, ks);
+                // Prints the collection document's metadata.
+                _validateState->getCollection()->getRecordStore()->printRecordMetadata(opCtx,
+                                                                                       recordId);
                 return;
             }
             search->second.insert(info);
@@ -499,8 +438,7 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx,
 bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) {
     invariant(!_firstPhase);
 
-    const uint64_t maxMemoryUsageBytes =
-        static_cast<uint64_t>(maxValidateMemoryUsageMB.load()) * 1024 * 1024;
+    const uint32_t maxMemoryUsageBytes = maxValidateMemoryUsageMB.load() * 1024 * 1024;
     const uint64_t totalMemoryNeededBytes =
         std::accumulate(_indexKeyBuckets.begin(),
                         _indexKeyBuckets.end(),
@@ -515,57 +453,48 @@ bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) {
         return true;
     }
 
-    // At this point we know we'll exceed the memory limit, and will pare back some of the buckets.
-    // First we'll see what the smallest bucket is, and if that's over the limit by itself, then
-    // we can zero out all the other buckets. Otherwise we'll keep as many buckets as we can.
+    bool hasNonZeroBucket = false;
+    uint64_t memoryUsedSoFarBytes = 0;
+    uint32_t smallestBucketBytes = std::numeric_limits<uint32_t>::max();
+    // Zero out any nonzero buckets that would put us over maxMemoryUsageBytes.
+    std::for_each(_indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) {
+        if (bucket.indexKeyCount == 0) {
+            return;
+        }
 
-    auto smallestBucketWithAnInconsistency = std::min_element(
-        _indexKeyBuckets.begin(),
-        _indexKeyBuckets.end(),
-        [](const IndexKeyBucket& lhs, const IndexKeyBucket& rhs) {
-            if (lhs.indexKeyCount != 0) {
-                return rhs.indexKeyCount == 0 || lhs.bucketSizeBytes < rhs.bucketSizeBytes;
-            }
-            return false;
-        });
-    invariant(smallestBucketWithAnInconsistency->indexKeyCount != 0);
-
-    if (smallestBucketWithAnInconsistency->bucketSizeBytes > maxMemoryUsageBytes) {
-        // We're going to just keep the smallest bucket, and zero everything else.
-        std::for_each(
-            _indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) {
-                if (&bucket == &(*smallestBucketWithAnInconsistency)) {
-                    // We keep the smallest bucket.
-                    return;
-                }
+        smallestBucketBytes = std::min(smallestBucketBytes, bucket.bucketSizeBytes);
+        if (bucket.bucketSizeBytes + memoryUsedSoFarBytes > maxMemoryUsageBytes) {
+            // Including this bucket would put us over the memory limit, so zero this bucket. We
+            // don't want to keep any entry that will exceed the memory limit in the second phase so
+            // we don't double the 'maxMemoryUsageBytes' here.
+            bucket.indexKeyCount = 0;
+            return;
+        }
+        memoryUsedSoFarBytes += bucket.bucketSizeBytes;
+        hasNonZeroBucket = true;
+    });
 
-                bucket.indexKeyCount = 0;
-            });
-    } else {
-        // We're going to scan through the buckets and keep as many as we can.
-        std::uint32_t memoryUsedSoFarBytes = 0;
-        std::for_each(
-            _indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) {
-                if (bucket.indexKeyCount == 0) {
-                    return;
-                }
+    StringBuilder memoryLimitMessage;
+    memoryLimitMessage << "Memory limit for validation is currently set to "
+                       << maxValidateMemoryUsageMB.load()
+                       << "MB and can be configured via the 'maxValidateMemoryUsageMB' parameter.";
 
-                if (bucket.bucketSizeBytes + memoryUsedSoFarBytes > maxMemoryUsageBytes) {
-                    // Including this bucket would put us over the memory limit, so zero this
-                    // bucket. We don't want to keep any entry that will exceed the memory limit in
-                    // the second phase so we don't double the 'maxMemoryUsageBytes' here.
-                    bucket.indexKeyCount = 0;
-                    return;
-                }
-                memoryUsedSoFarBytes += bucket.bucketSizeBytes;
-            });
+    if (!hasNonZeroBucket) {
+        const uint32_t minMemoryNeededMB = (smallestBucketBytes / (1024 * 1024)) + 1;
+        StringBuilder ss;
+        ss << "Unable to report index entry inconsistencies due to memory limitations. Need at "
+              "least "
+           << minMemoryNeededMB << "MB to report at least one index entry inconsistency. "
+           << memoryLimitMessage.str();
+        result->errors.push_back(ss.str());
+        result->valid = false;
+
+        return false;
     }
 
     StringBuilder ss;
-    ss << "Not all index entry inconsistencies are reported due to memory limitations. Memory "
-          "limit for validation is currently set to "
-       << maxValidateMemoryUsageMB.load()
-       << "MB and can be configured via the 'maxValidateMemoryUsageMB' parameter.";
+    ss << "Not all index entry inconsistencies are reported due to memory limitations. "
+       << memoryLimitMessage.str();
     result->errors.push_back(ss.str());
     result->valid = false;
 
@@ -611,16 +540,4 @@ uint32_t IndexConsistency::_hashKeyString(const KeyString::Value& ks,
                                           uint32_t indexNameHash) const {
     return ks.hash(indexNameHash);
 }
-
-void IndexConsistency::_printMetadata(OperationContext* opCtx,
-                                      ValidateResults* results,
-                                      const IndexEntryInfo& entryInfo) {
-    _validateState->getCollection()->getRecordStore()->printRecordMetadata(
-        opCtx, entryInfo.recordId, &(results->recordTimestamps));
-    getIndexInfo(entryInfo.indexName)
-        .accessMethod->asSortedData()
-        ->getSortedDataInterface()
-        ->printIndexEntryMetadata(opCtx, entryInfo.keyString);
-}
-
 }  // namespace mongo