diff options
Diffstat (limited to 'src/mongo/db/catalog/validate_adaptor.cpp')
| -rw-r--r-- | src/mongo/db/catalog/validate_adaptor.cpp | 390 |
1 files changed, 55 insertions, 335 deletions
diff --git a/src/mongo/db/catalog/validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp index 9dc4a8dc71b..acbef39ba9c 100644 --- a/src/mongo/db/catalog/validate_adaptor.cpp +++ b/src/mongo/db/catalog/validate_adaptor.cpp @@ -41,7 +41,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_consistency.h" #include "mongo/db/catalog/throttle_cursor.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/index/index_descriptor.h" @@ -53,10 +53,6 @@ #include "mongo/db/storage/execution_context.h" #include "mongo/db/storage/key_string.h" #include "mongo/db/storage/record_store.h" -#include "mongo/db/storage/storage_parameters_gen.h" -#include "mongo/db/timeseries/flat_bson.h" -#include "mongo/db/timeseries/timeseries_constants.h" -#include "mongo/db/timeseries/timeseries_options.h" #include "mongo/logv2/log.h" #include "mongo/rpc/object_check.h" #include "mongo/util/fail_point.h" @@ -67,8 +63,6 @@ namespace mongo { namespace { MONGO_FAIL_POINT_DEFINE(crashOnMultikeyValidateFailure); -MONGO_FAIL_POINT_DEFINE(failIndexKeyOrdering); -MONGO_FAIL_POINT_DEFINE(failRecordStoreTraversal); // Set limit for size of corrupted records that will be reported. const long long kMaxErrorSizeBytes = 1 * 1024 * 1024; @@ -78,9 +72,6 @@ const long long kInterruptIntervalNumBytes = 50 * 1024 * 1024; // 50MB. static constexpr const char* kSchemaValidationFailedReason = "Detected one or more documents not compliant with the collection's schema. Check logs for log " "id 5363500."; -static constexpr const char* kTimeseriesValidationInconsistencyReason = - "Detected one or more documents in this collection incompatible with time-series " - "specifications. For more info, see logs with log id 6698300."; /** * Validate that for each record in a clustered RecordStore the record key (RecordId) matches the @@ -127,197 +118,32 @@ void schemaValidationFailed(CollectionValidation::ValidateState* state, state->setCollectionSchemaViolated(); - if (result != Collection::SchemaValidationResult::kPass) { + // TODO SERVER-65078: remove the testing proctor check. + // When testing is enabled, only warn about non-compliant documents to prevent test failures. + if (TestingProctor::instance().isEnabled() || + Collection::SchemaValidationResult::kWarn == result) { results->warnings.push_back(kSchemaValidationFailedReason); + } else if (Collection::SchemaValidationResult::kError == result) { + results->errors.push_back(kSchemaValidationFailedReason); + results->valid = false; } } -/** - * Checks the value of the bucket's version and if it matches the types of 'data' fields. - */ -Status _validateTimeseriesControlVersion(const BSONObj& recordBson) { - int controlVersion = recordBson.getField(timeseries::kBucketControlFieldName) - .Obj() - .getField(timeseries::kBucketControlVersionFieldName) - .Number(); - if (controlVersion != 1 && controlVersion != 2) { - return Status( - ErrorCodes::BadValue, - fmt::format("Invalid value for 'control.version'. Expected 1 or 2, but got {}.", - controlVersion)); - } - auto dataType = controlVersion == 1 ? BSONType::Object : BSONType::BinData; - // In addition to checking dataType, make sure that closed buckets have BinData Column subtype - auto isCorrectType = [&](BSONElement el) { - if (controlVersion == 1) { - return el.type() == BSONType::Object; - } else { - return el.type() == BSONType::BinData && el.binDataType() == BinDataType::Column; - } - }; - BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj(); - for (BSONObjIterator bi(data); bi.more();) { - BSONElement e = bi.next(); - if (!isCorrectType(e)) { - return Status(ErrorCodes::TypeMismatch, - fmt::format("Mismatch between time-series schema version and data field " - "type. Expected type {}, but got {}.", - mongo::typeName(dataType), - mongo::typeName(e.type()))); - } - } - return Status::OK(); -} - -/** - * Checks the equivalence between the min and max fields in 'control' for a bucket and - * the corresponding value in 'data'. - */ -Status _validateTimeseriesMinMax(const BSONObj& recordBson, const CollectionPtr& coll) { - BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj(); - BSONObj control = recordBson.getField(timeseries::kBucketControlFieldName).Obj(); - BSONObj controlMin = control.getField(timeseries::kBucketControlMinFieldName).Obj(); - BSONObj controlMax = control.getField(timeseries::kBucketControlMaxFieldName).Obj(); - - auto dataFields = data.getFieldNames<std::set<std::string>>(); - auto controlMinFields = controlMin.getFieldNames<std::set<std::string>>(); - auto controlMaxFields = controlMax.getFieldNames<std::set<std::string>>(); - - // Checks that the number of 'control.min' and 'control.max' fields agrees with number of 'data' - // fields. - if (dataFields.size() != controlMinFields.size() || - dataFields.size() != controlMaxFields.size()) { - return Status( - ErrorCodes::BadValue, - fmt::format( - "Mismatch between the number of time-series control fields and the number " - "of data fields. " - "Control had {} min fields and {} max fields, but observed data had {} fields.", - controlMinFields.size(), - controlMaxFields.size(), - dataFields.size())); - }; - - // Used when checking min timestamp, which is rounded down by granularity. - auto granularity = coll->getTimeseriesOptions()->getGranularity(); - - // Validates that the 'control.min' and 'control.max' field values agree with 'data' field - // values. - for (auto fieldName : dataFields) { - timeseries::MinMax minmax; - auto field = data.getField(fieldName); - - for (BSONElement el : field.Obj()) { - minmax.update(el.wrap(fieldName), boost::none, coll->getDefaultCollator()); - } - - auto controlFieldMin = controlMin.getField(fieldName); - auto controlFieldMax = controlMax.getField(fieldName); - auto min = minmax.min(); - auto max = minmax.max(); - - // Checks whether the min and max values between 'control' and 'data' match, taking - // timestamp granularity into account. - auto checkMinAndMaxMatch = [&]() { - if (fieldName == coll->getTimeseriesOptions()->getTimeField()) { - return controlFieldMin.Date() == - timeseries::roundTimestampToGranularity(min.getField(fieldName).Date(), - granularity) && - controlFieldMax.Date() == max.getField(fieldName).Date(); - } else { - return controlFieldMin.wrap().woCompare(min) == 0 && - controlFieldMax.wrap().woCompare(max) == 0; - } - }; - - if (!checkMinAndMaxMatch()) { - return Status( - ErrorCodes::BadValue, - fmt::format( - "Mismatch between time-series control and observed min or max for field {}. " - "Control had min {} and max {}, but observed data had min {} and max {}.", - fieldName, - controlFieldMin.toString(), - controlFieldMax.toString(), - min.toString(), - max.toString())); - } - } - - return Status::OK(); -} - -/** - * Validates the consistency of a time-series bucket. - */ -Status _validateTimeSeriesBucketRecord(const CollectionPtr& collection, - const BSONObj& recordBson, - ValidateResults* results) { - - if (Status status = _validateTimeseriesControlVersion(recordBson); !status.isOK()) { - return status; - } - - int version = recordBson.getField(timeseries::kBucketControlFieldName) - .Obj() - .getField(timeseries::kBucketControlVersionFieldName) - .Number(); - - // TODO(SERVER-67023): Check closed bucket as part of validation. - if (version == 1) { - if (Status status = _validateTimeseriesMinMax(recordBson, collection); !status.isOK()) { - return status; - } - } - - - return Status::OK(); -} - - -void _timeseriesValidationFailed(CollectionValidation::ValidateState* state, - ValidateResults* results) { - if (state->isTimeseriesDataInconsistent()) { - // Only report the warning message once. - return; - } - state->setTimeseriesDataInconsistent(); - - results->warnings.push_back(kTimeseriesValidationInconsistencyReason); -} - -BSONObj rehydrateKey(const BSONObj& keyPattern, const BSONObj& indexKey) { - // We need to rehydrate the indexKey for improved readability. - // {"": ObjectId(...)} -> {"_id": ObjectId(...)} - auto keysIt = keyPattern.begin(); - auto valuesIt = indexKey.begin(); - - BSONObjBuilder b; - while (keysIt != keyPattern.end()) { - // keysIt and valuesIt must have the same number of elements. - invariant(valuesIt != indexKey.end()); - b.appendAs(*valuesIt, keysIt->fieldName()); - keysIt++; - valuesIt++; - } - return b.obj(); -} } // namespace Status ValidateAdaptor::validateRecord(OperationContext* opCtx, const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results, - ValidationVersion validationVersion) { - const Status status = validateBSON(record.data(), record.size(), validationVersion); + ValidateResults* results) { + const Status status = validateBSON(record.data(), record.size()); if (!status.isOK()) return status; BSONObj recordBson = record.toBson(); *dataSize = recordBson.objsize(); - if (MONGO_unlikely(_validateState->logDiagnostics())) { + if (MONGO_unlikely(_validateState->extraLoggingForTest())) { LOGV2(4666601, "[validate]", "recordId"_attr = recordId, "recordData"_attr = recordBson); } @@ -361,26 +187,6 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, {multikeyMetadataKeys->begin(), multikeyMetadataKeys->end()}, *documentMultikeyPaths); - auto printMultikeyMetadata = [&]() { - LOGV2(7556100, - "Index is not multikey but document has multikey data", - "indexName"_attr = descriptor->indexName(), - "recordId"_attr = recordId, - "record"_attr = redact(recordBson)); - for (auto& key : *documentKeySet) { - auto indexKey = KeyString::toBsonSafe(key.getBuffer(), - key.getSize(), - iam->getSortedDataInterface()->getOrdering(), - key.getTypeBits()); - const BSONObj rehydratedKey = rehydrateKey(descriptor->keyPattern(), indexKey); - LOGV2(7556101, - "Index key for document with multikey inconsistency", - "indexName"_attr = descriptor->indexName(), - "recordId"_attr = recordId, - "indexKey"_attr = redact(rehydratedKey)); - } - }; - if (!index->isMultikey(opCtx, coll) && shouldBeMultikey) { if (_validateState->fixErrors()) { writeConflictRetry(opCtx, "setIndexAsMultikey", coll->ns().ns(), [&] { @@ -398,17 +204,10 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, << " set to multikey."); results->repaired = true; } else { - printMultikeyMetadata(); - auto& curRecordResults = (results->indexResultsMap)[descriptor->indexName()]; - const std::string msg = fmt::format( - "Index {} is not multikey but document with RecordId({}) and {} has multikey " - "data, " - "{} key(s)", - descriptor->indexName(), - recordId.toString(), - recordBson.getField("_id").toString(), - documentKeySet->size()); + std::string msg = str::stream() << "Index " << descriptor->indexName() + << " is not multikey but has more than one" + << " key in document " << recordId; curRecordResults.errors.push_back(msg); curRecordResults.valid = false; if (crashOnMultikeyValidateFailure.shouldFail()) { @@ -436,8 +235,6 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, << " multikey paths updated."); results->repaired = true; } else { - printMultikeyMetadata(); - std::string msg = str::stream() << "Index " << descriptor->indexName() << " multikey paths do not cover a document. RecordId: " << recordId; @@ -470,7 +267,7 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, for (const auto& keyString : *documentKeySet) { try { _totalIndexKeys++; - _indexConsistency->addDocKey(opCtx, keyString, &indexInfo, recordId, results); + _indexConsistency->addDocKey(opCtx, keyString, &indexInfo, recordId); } catch (...) { return exceptionToStatus(); } @@ -483,16 +280,15 @@ namespace { // Ensures that index entries are in increasing or decreasing order. void _validateKeyOrder(OperationContext* opCtx, const IndexCatalogEntry* index, - const KeyStringEntry& currKey, - const KeyStringEntry& prevKey, + const KeyString::Value& currKey, + const KeyString::Value& prevKey, IndexValidateResults* results) { auto descriptor = index->descriptor(); bool unique = descriptor->unique(); // KeyStrings will be in strictly increasing order because all keys are sorted and they are in // the format (Key, RID), and all RecordIDs are unique. - if (currKey.keyString.compare(prevKey.keyString) <= 0 || - MONGO_unlikely(failIndexKeyOrdering.shouldFail())) { + if (currKey.compare(prevKey) <= 0) { if (results && results->valid) { results->errors.push_back(str::stream() << "index '" << descriptor->indexName() @@ -506,20 +302,21 @@ void _validateKeyOrder(OperationContext* opCtx, if (unique) { // Unique indexes must not have duplicate keys. - int cmp = currKey.loc.isLong() - ? currKey.keyString.compareWithoutRecordIdLong(prevKey.keyString) - : currKey.keyString.compareWithoutRecordIdStr(prevKey.keyString); + int cmp = currKey.compareWithoutRecordIdLong(prevKey); if (cmp != 0) { return; } if (results && results->valid) { - auto bsonKey = - KeyString::toBson(currKey.keyString, Ordering::make(descriptor->keyPattern())); + auto bsonKey = KeyString::toBson(currKey, Ordering::make(descriptor->keyPattern())); + auto firstRecordId = + KeyString::decodeRecordIdLongAtEnd(prevKey.getBuffer(), prevKey.getSize()); + auto secondRecordId = + KeyString::decodeRecordIdLongAtEnd(currKey.getBuffer(), currKey.getSize()); results->errors.push_back(str::stream() << "Unique index '" << descriptor->indexName() << "' has duplicate key: " << bsonKey - << ", first record: " << prevKey.loc - << ", second record: " << currKey.loc); + << ", first record: " << firstRecordId + << ", second record: " << secondRecordId); } if (results) { results->valid = false; @@ -538,6 +335,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, IndexInfo& indexInfo = _indexConsistency->getIndexInfo(indexName); int64_t numKeys = 0; + bool isFirstEntry = true; + // The progress meter will be inactive after traversing the record store to allow the message // and the total to be set to different values. if (!_progress->isActive()) { @@ -552,7 +351,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, KeyString::Builder firstKeyStringBuilder( version, BSONObj(), indexInfo.ord, KeyString::Discriminator::kExclusiveBefore); KeyString::Value firstKeyString = firstKeyStringBuilder.getValueCopy(); - boost::optional<KeyStringEntry> prevIndexKeyStringEntry; + KeyString::Value prevIndexKeyStringValue; // Ensure that this index has an open index cursor. const auto indexCursorIt = _validateState->getIndexCursors().find(indexName); @@ -583,8 +382,9 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, bool foundOldUniqueIndexKeys = false; while (indexEntry) { - if (prevIndexKeyStringEntry) { - _validateKeyOrder(opCtx, index, *indexEntry, *prevIndexKeyStringEntry, &indexResults); + if (!isFirstEntry) { + _validateKeyOrder( + opCtx, index, indexEntry->keyString, prevIndexKeyStringValue, &indexResults); } if (!foundOldUniqueIndexKeys && !descriptor->isIdIndex() && descriptor->unique() && @@ -615,7 +415,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, _progress->hit(); numKeys++; - prevIndexKeyStringEntry = indexEntry; + isFirstEntry = false; + prevIndexKeyStringValue = indexEntry->keyString; if (numKeys % kInterruptIntervalNumRecords == 0) { // Periodically checks for interrupts and yields. @@ -631,7 +432,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, "Error advancing index cursor", "error"_attr = ex.toString(), "index"_attr = indexName, - "prevKey"_attr = prevIndexKeyStringEntry->keyString.toString()); + "prevKey"_attr = prevIndexKeyStringValue.toString()); } throw; } @@ -717,8 +518,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output, - ValidationVersion validationVersion) { + BSONObjBuilder* output) { _numRecords = 0; // need to reset it because this function can be called more than once. long long dataSizeTotal = 0; long long interruptIntervalNumBytes = 0; @@ -742,10 +542,9 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, // Because the progress meter is intended as an approximation, it's sufficient to get the number // of records when we begin traversing, even if this number may deviate from the final number. - const auto& coll = _validateState->getCollection(); const char* curopMessage = "Validate: scanning documents"; - const auto totalRecords = coll->getRecordStore()->numRecords(opCtx); - const auto rs = coll->getRecordStore(); + const auto totalRecords = _validateState->getCollection()->getRecordStore()->numRecords(opCtx); + const auto rs = _validateState->getCollection()->getRecordStore(); { stdx::unique_lock<Client> lk(*opCtx->getClient()); _progress.set(CurOp::get(opCtx)->setProgress_inlock(curopMessage, totalRecords)); @@ -756,9 +555,6 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, return; } - bool bucketMixedSchemaDataError = false; - bool bucketMinMaxMalformedError = false; - bool bucketMixedSchemaDataWarning = false; bool corruptRecordsSizeLimitWarning = false; const std::unique_ptr<SeekableRecordThrottleCursor>& traverseRecordStoreCursor = _validateState->getTraverseRecordStoreCursor(); @@ -772,25 +568,11 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, interruptIntervalNumBytes += dataSize; dataSizeTotal += dataSize; size_t validatedSize = 0; - Status status = validateRecord( - opCtx, record->id, record->data, &validatedSize, results, validationVersion); - - // Log the out-of-order entries as errors. - // - // Validate uses a DataCorruptionDetectionMode::kLogAndContinue mode such that data - // corruption errors are logged without throwing, so certain checks must be duplicated here - // as well. - if ((prevRecordId.isValid() && prevRecordId > record->id) || - MONGO_unlikely(failRecordStoreTraversal.shouldFail())) { - // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list and - // not care about deduplication. - static constexpr auto kErrorMessage = "Detected out-of-order documents. See logs."; - if (results->valid || - std::find(results->errors.begin(), results->errors.end(), kErrorMessage) == - results->errors.end()) { - results->errors.push_back(kErrorMessage); - results->valid = false; - } + Status status = validateRecord(opCtx, record->id, record->data, &validatedSize, results); + + // RecordStores are required to return records in RecordId order. + if (prevRecordId.isValid()) { + invariant(prevRecordId < record->id); } // validatedSize = dataSize is not a general requirement as some storage engines may use @@ -821,14 +603,8 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, results->numRemovedCorruptRecords++; _numRecords--; } else { - // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list - // and not care about deduplication. - static constexpr auto kErrorMessage = - "Detected one or more invalid documents. See logs."; - if (results->valid || - std::find(results->errors.begin(), results->errors.end(), kErrorMessage) == - results->errors.end()) { - results->errors.push_back(kErrorMessage); + if (results->valid) { + results->errors.push_back("Detected one or more invalid documents. See logs."); results->valid = false; } @@ -847,76 +623,18 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, // If the document is not corrupted, validate the document against this collection's // schema validator. Don't treat invalid documents as errors since documents can bypass // document validation when being inserted or updated. - auto result = coll->checkValidation(opCtx, record->data.toBson()); + auto result = + _validateState->getCollection()->checkValidation(opCtx, record->data.toBson()); if (result.first != Collection::SchemaValidationResult::kPass) { LOGV2_WARNING(5363500, "Document is not compliant with the collection's schema", - logAttrs(coll->ns()), + logAttrs(_validateState->getCollection()->ns()), "recordId"_attr = record->id, "reason"_attr = result.second); nNonCompliantDocuments++; schemaValidationFailed(_validateState, result.first, results); - } else if (coll->getTimeseriesOptions()) { - // Checks for time-series collection consistency. - Status bucketStatus = - _validateTimeSeriesBucketRecord(coll, record->data.toBson(), results); - - // This log id should be kept in sync with the associated warning messages that are - // returned to the client. - if (!bucketStatus.isOK()) { - LOGV2_WARNING(6698300, - "Document is not compliant with time-series specifications", - logAttrs(coll->ns()), - "recordId"_attr = record->id, - "reason"_attr = bucketStatus); - nNonCompliantDocuments++; - _timeseriesValidationFailed(_validateState, results); - } else { - auto containsMixedSchemaDataResponse = - coll->doesTimeseriesBucketsDocContainMixedSchemaData(record->data.toBson()); - if (!containsMixedSchemaDataResponse.isOK() && !bucketMinMaxMalformedError) { - bucketMinMaxMalformedError = true; - LOGV2_WARNING(8469900, - "Detected a time-series bucket with malformed min/max values", - logAttrs(coll->ns()), - "bucketId"_attr = record->id, - "error"_attr = containsMixedSchemaDataResponse.getStatus()); - results->errors.push_back( - str::stream() - << "Detected a time-series bucket with malformed min/max values"); - results->valid = false; - } else if (containsMixedSchemaDataResponse.isOK() && - containsMixedSchemaDataResponse.getValue()) { - bool mixedSchemaAllowed = - coll->getTimeseriesBucketsMayHaveMixedSchemaData().value_or(true); - if (mixedSchemaAllowed && !bucketMixedSchemaDataWarning) { - bucketMixedSchemaDataWarning = true; - LOGV2_WARNING(8469901, - "Detected a time-series bucket with mixed schema data", - logAttrs(coll->ns()), - "bucketId"_attr = record->id); - results->warnings.push_back( - str::stream() - << "Detected a time-series bucket with mixed schema data"); - } else if (!mixedSchemaAllowed && !bucketMixedSchemaDataError) { - bucketMixedSchemaDataError = true; - LOGV2_WARNING(8469902, - "Detected a time-series bucket with mixed schema data " - "when timeseriesBucketsMayHaveMixedSchemaData is false. " - "You can run the collMod command to set this flag", - logAttrs(coll->ns()), - "bucketId"_attr = record->id); - results->errors.push_back( - str::stream() - << "Detected a time-series bucket with mixed schema data when " - "timeseriesBucketsMayHaveMixedSchemaData is false. You can run " - "the collMod command to set this flag"); - results->valid = false; - } - } - } } } @@ -939,18 +657,20 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, << " invalid documents."); } - const auto fastCount = coll->numRecords(opCtx); + const auto fastCount = _validateState->getCollection()->numRecords(opCtx); if (_validateState->shouldEnforceFastCount() && fastCount != _numRecords) { - results->errors.push_back( - str::stream() << "fast count (" << fastCount << ") does not match number of records (" - << _numRecords << ") for collection '" << coll->ns() << "'"); + results->errors.push_back(str::stream() << "fast count (" << fastCount + << ") does not match number of records (" + << _numRecords << ") for collection '" + << _validateState->getCollection()->ns() << "'"); results->valid = false; } // Do not update the record store stats if we're in the background as we've validated a // checkpoint and it may not have the most up-to-date changes. if (results->valid && !_validateState->isBackground()) { - coll->getRecordStore()->updateStatsAfterRepair(opCtx, _numRecords, dataSizeTotal); + _validateState->getCollection()->getRecordStore()->updateStatsAfterRepair( + opCtx, _numRecords, dataSizeTotal); } } |
