diff options
Diffstat (limited to 'src/mongo/db/catalog/collection_impl.cpp')
| -rw-r--r-- | src/mongo/db/catalog/collection_impl.cpp | 325 |
1 files changed, 89 insertions, 236 deletions
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index cdf3730fa1e..b79c8c78914 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -40,8 +40,6 @@ #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/crypto/fle_crypto.h" #include "mongo/db/auth/security_token.h" -#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" -#include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/catalog/document_validation.h" @@ -49,7 +47,6 @@ #include "mongo/db/catalog/index_consistency.h" #include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/catalog/local_oplog_info.h" -#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" #include "mongo/db/catalog/uncommitted_multikey.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands/server_status_metric.h" @@ -83,7 +80,6 @@ #include "mongo/db/storage/record_store.h" #include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/db/timeseries/timeseries_constants.h" -#include "mongo/db/timeseries/timeseries_extended_range.h" #include "mongo/db/timeseries/timeseries_index_schema_conversion_functions.h" #include "mongo/db/transaction_participant.h" #include "mongo/db/ttl_collection_cache.h" @@ -354,42 +350,35 @@ bool indexTypeSupportsPathLevelMultikeyTracking(StringData accessMethod) { return accessMethod == IndexNames::BTREE || accessMethod == IndexNames::GEO_2DSPHERE; } -StatusWith<bool> doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) { +bool doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) { auto minIt = min.begin(); auto minEnd = min.end(); auto maxIt = max.begin(); auto maxEnd = max.end(); while (minIt != minEnd && maxIt != maxEnd) { - // The 'control.min' and 'control.max' fields have the same ordering. - if (minIt->fieldNameStringData() != maxIt->fieldNameStringData()) { - return Status{ - ErrorCodes::BadValue, - "Encountered inconsistent field name ordering in time-series bucket min/max"}; - } - - if (minIt->canonicalType() != maxIt->canonicalType()) { + bool typeMatch = minIt->canonicalType() == maxIt->canonicalType(); + if (!typeMatch) { return true; } else if (minIt->type() == Object) { - auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj()); - if (!result.isOK() || result.getValue()) { - return result; + // The 'control.min' and 'control.max' fields have the same ordering. + invariant(minIt->fieldNameStringData() == maxIt->fieldNameStringData()); + if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) { + return true; } } else if (minIt->type() == Array) { - auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj()); - if (!result.isOK() || result.getValue()) { - return result; + if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) { + return true; } } + invariant(typeMatch); minIt++; maxIt++; } - if (minIt != minEnd || maxIt != maxEnd) { - return Status{ErrorCodes::BadValue, - "Encountered extra field(s) in time-series bucket min/max"}; - } + // The 'control.min' and 'control.max' fields have the same cardinality. + invariant(minIt == minEnd && maxIt == maxEnd); return false; } @@ -550,11 +539,11 @@ void CollectionImpl::init(OperationContext* opCtx) { if (opCtx->lockState()->inAWriteUnitOfWork()) { opCtx->recoveryUnit()->onCommit([svcCtx, uuid](auto ts) { TTLCollectionCache::get(svcCtx).registerTTLInfo( - uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}}); + uuid, TTLCollectionCache::ClusteredId{}); }); } else { - TTLCollectionCache::get(svcCtx).registerTTLInfo( - uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}}); + TTLCollectionCache::get(svcCtx).registerTTLInfo(uuid, + TTLCollectionCache::ClusteredId{}); } } } @@ -736,8 +725,6 @@ Collection::Validator CollectionImpl::parseValidator( auto expCtx = make_intrusive<ExpressionContext>( opCtx, CollatorInterface::cloneCollator(_shared->_collator.get()), ns()); - expCtx->variables.setDefaultRuntimeConstants(opCtx); - // The MatchExpression and contained ExpressionContext created as part of the validator are // owned by the Collection and will outlive the OperationContext they were created under. expCtx->opCtx = nullptr; @@ -824,9 +811,9 @@ Status CollectionImpl::insertDocumentsForOplog(OperationContext* opCtx, _cappedDeleteAsNeeded(opCtx, records->begin()->id); - // We do not need to notify capped waiters, as we have not yet updated oplog visibility, so - // these inserts will not be visible. When visibility updates, it will notify capped - // waiters. + opCtx->recoveryUnit()->onCommit( + [this](boost::optional<Timestamp>) { _shared->notifyCappedWaitersIfNeeded(); }); + return status; } @@ -1598,19 +1585,6 @@ bool CollectionImpl::isTemporary() const { } boost::optional<bool> CollectionImpl::getTimeseriesBucketsMayHaveMixedSchemaData() const { - if (!getTimeseriesOptions()) { - return boost::none; - } - - // If present, reuse storageEngine options to work around the issue described in SERVER-91194 - boost::optional<bool> optBackwardsCompatibleFlag = getFlagFromStorageEngineBson( - _metadata->options.storageEngine, - backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData); - if (optBackwardsCompatibleFlag) { - return *optBackwardsCompatibleFlag; - } - - // Else, fallback to legacy parameter return _metadata->timeseriesBucketsMayHaveMixedSchemaData; } @@ -1626,21 +1600,11 @@ void CollectionImpl::setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext "setting"_attr = setting); _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) { - // Reuse storageEngine options to work around the issue described in SERVER-91194 - if (setting.has_value()) { - md.options.storageEngine = setFlagToStorageEngineBson( - md.options.storageEngine, - backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData, - *setting); - } - - // Also update legacy parameter for compatibility when downgrading to older sub-versions - // only relying on this option (best-effort because it may be lost due to SERVER-91194) md.timeseriesBucketsMayHaveMixedSchemaData = setting; }); } -StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( +bool CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( const BSONObj& bucketsDoc) const { if (!getTimeseriesOptions()) { return false; @@ -1653,29 +1617,6 @@ StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( return doesMinMaxHaveMixedSchemaData(minObj, maxObj); } -bool CollectionImpl::getRequiresTimeseriesExtendedRangeSupport() const { - return _shared->_requiresTimeseriesExtendedRangeSupport.load(); -} - -void CollectionImpl::setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const { - uassert(6679401, "This is not a time-series collection", _metadata->options.timeseries); - - bool expected = false; - bool set = _shared->_requiresTimeseriesExtendedRangeSupport.compareAndSwap(&expected, true); - if (set) { - catalog_stats::requiresTimeseriesExtendedRangeSupport.fetchAndAdd(1); - if (!timeseries::collectionHasTimeIndex(opCtx, *this)) { - LOGV2_WARNING( - 6679402, - "Time-series collection contains dates outside the standard range. Some query " - "optimizations may be disabled. Please consider building an index on timeField to " - "re-enable them.", - "nss"_attr = ns().getTimeseriesViewNamespace(), - "timeField"_attr = _metadata->options.timeseries->getTimeField()); - } - } -} - bool CollectionImpl::isClustered() const { return getClusteredInfo().is_initialized(); } @@ -1814,8 +1755,7 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx, int scale) const { const IndexCatalog* idxCatalog = getIndexCatalog(); - auto ii = idxCatalog->getIndexIterator( - opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished); + std::unique_ptr<IndexCatalog::IndexIterator> ii = idxCatalog->getIndexIterator(opCtx, true); uint64_t totalSize = 0; @@ -1836,18 +1776,9 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx, } uint64_t CollectionImpl::getIndexFreeStorageBytes(OperationContext* const opCtx) const { - // Unfinished index builds are excluded to avoid a potential deadlock when trying to collect - // statistics from the index table while the index build is in the bulk load phase. See - // SERVER-77018. This should not be too impactful as: - // - During the collection scan phase, the index table is unused. - // - During the bulk load phase, getFreeStorageBytes will probably return EBUSY, as the ident is - // in use by the index builder. (And worst case results in the deadlock). - // - It might be possible to return meaningful data post bulk-load, but reusable bytes should be - // low anyways as the collection has been bulk loaded. Additionally, this would be a inaccurate - // anyways as the build is in progress. - // - Once the index build is finished, this will be eventually accounted for. const auto idxCatalog = getIndexCatalog(); - auto indexIt = idxCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + const bool includeUnfinished = true; + auto indexIt = idxCatalog->getIndexIterator(opCtx, includeUnfinished); uint64_t totalSize = 0; while (indexIt->more()) { @@ -1871,7 +1802,8 @@ Status CollectionImpl::truncate(OperationContext* opCtx) { // 1) store index specs std::vector<BSONObj> indexSpecs; { - auto ii = _indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + std::unique_ptr<IndexCatalog::IndexIterator> ii = + _indexCatalog->getIndexIterator(opCtx, false); while (ii->more()) { const IndexDescriptor* idx = ii->next()->descriptor(); indexSpecs.push_back(idx->infoObj().getOwned()); @@ -2206,22 +2138,20 @@ void CollectionImpl::updatePrepareUniqueSetting(OperationContext* opCtx, }); } -std::vector<std::string> CollectionImpl::repairInvalidIndexOptions(OperationContext* opCtx) { +std::vector<std::string> CollectionImpl::removeInvalidIndexOptions(OperationContext* opCtx) { std::vector<std::string> indexesWithInvalidOptions; _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) { for (auto& index : md.indexes) { - if (index.isPresent()) { - BSONObj oldSpec = index.spec; - - Status status = index_key_validate::validateIndexSpec(opCtx, oldSpec).getStatus(); - if (status.isOK()) { - continue; - } + BSONObj oldSpec = index.spec; - indexesWithInvalidOptions.push_back(std::string(index.nameStringData())); - index.spec = index_key_validate::repairIndexSpec(NamespaceString(md.ns), oldSpec); + Status status = index_key_validate::validateIndexSpecFieldNames(oldSpec); + if (status.isOK()) { + continue; } + + indexesWithInvalidOptions.push_back(std::string(index.nameStringData())); + index.spec = index_key_validate::removeUnknownFields(NamespaceString(md.ns), oldSpec); } }); @@ -2298,65 +2228,44 @@ bool CollectionImpl::isIndexMultikey(OperationContext* opCtx, StringData indexName, MultikeyPaths* multikeyPaths, int indexOffset) const { - int offset = indexOffset; - if (offset < 0) { - offset = _metadata->findIndexOffset(indexName); - invariant(offset >= 0, - str::stream() << "cannot get multikey for index " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON()); - } else { - invariant(offset < int(_metadata->indexes.size()), - str::stream() << "out of bounds index offset for multikey info " << indexName - << " @ " << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - invariant(indexName == _metadata->indexes[offset].nameStringData(), - str::stream() << "invalid index offset for multikey info " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - } - - // If we have uncommitted multikey writes we need to check here to read our own writes + auto isMultikey = [this, multikeyPaths, indexName, indexOffset]( + const BSONCollectionCatalogEntry::MetaData& metadata) { + int offset = indexOffset; + if (offset < 0) { + offset = metadata.findIndexOffset(indexName); + invariant(offset >= 0, + str::stream() << "cannot get multikey for index " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON()); + } else { + invariant(offset < int(metadata.indexes.size()), + str::stream() + << "out of bounds index offset for multikey info " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + invariant(indexName == metadata.indexes[offset].nameStringData(), + str::stream() + << "invalid index offset for multikey info " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + } + + const auto& index = metadata.indexes[offset]; + stdx::lock_guard lock(index.multikeyMutex); + if (multikeyPaths && !index.multikeyPaths.empty()) { + *multikeyPaths = index.multikeyPaths; + } + + return index.multikey; + }; + const auto& uncommittedMultikeys = UncommittedMultikey::get(opCtx).resources(); if (uncommittedMultikeys) { if (auto it = uncommittedMultikeys->find(this); it != uncommittedMultikeys->end()) { - const auto& index = it->second.indexes[offset]; - if (multikeyPaths && !index.multikeyPaths.empty()) { - *multikeyPaths = index.multikeyPaths; - } - return index.multikey; - } - } - - // Otherwise read from the metadata cache if there are no concurrent multikey writers - { - const auto& index = _metadata->indexes[offset]; - // Check for concurrent writers, this can race with writers where it can be set immediately - // after checking. This is fine we know that the reader in that case opened its snapshot - // before the writer and we do not need to observe its result. - if (index.concurrentWriters.load() == 0) { - stdx::lock_guard lock(index.multikeyMutex); - if (multikeyPaths && !index.multikeyPaths.empty()) { - *multikeyPaths = index.multikeyPaths; - } - return index.multikey; + return isMultikey(it->second); } } - // We need to read from the durable catalog if there are concurrent multikey writers to avoid - // reading between the multikey write committing in the storage engine but before its onCommit - // handler made the write visible for readers. - auto snapshotMetadata = DurableCatalog::get(opCtx)->getMetaData(opCtx, getCatalogId()); - int snapshotOffset = snapshotMetadata->findIndexOffset(indexName); - invariant(snapshotOffset >= 0, - str::stream() << "cannot get multikey for index " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON()); - const auto& index = snapshotMetadata->indexes[snapshotOffset]; - if (multikeyPaths && !index.multikeyPaths.empty()) { - *multikeyPaths = index.multikeyPaths; - } - return index.multikey; + return isMultikey(*_metadata); } bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, @@ -2364,31 +2273,31 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, const MultikeyPaths& multikeyPaths, int indexOffset) const { - int offset = indexOffset; - if (offset < 0) { - offset = _metadata->findIndexOffset(indexName); - invariant(offset >= 0, - str::stream() << "cannot set multikey for index " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON()); - } else { - invariant(offset < int(_metadata->indexes.size()), - str::stream() << "out of bounds index offset for multikey update" << indexName - << " @ " << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - invariant(indexName == _metadata->indexes[offset].nameStringData(), - str::stream() << "invalid index offset for multikey update " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - } - - auto setMultikey = [offset, - multikeyPaths](const BSONCollectionCatalogEntry::MetaData& metadata) { + auto setMultikey = [this, indexName, multikeyPaths, indexOffset]( + const BSONCollectionCatalogEntry::MetaData& metadata) { + int offset = indexOffset; + if (offset < 0) { + offset = metadata.findIndexOffset(indexName); + invariant(offset >= 0, + str::stream() << "cannot set multikey for index " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON()); + } else { + invariant(offset < int(metadata.indexes.size()), + str::stream() + << "out of bounds index offset for multikey update" << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + invariant(indexName == metadata.indexes[offset].nameStringData(), + str::stream() + << "invalid index offset for multikey update " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + } + auto* index = &metadata.indexes[offset]; stdx::lock_guard lock(index->multikeyMutex); - auto tracksPathLevelMultikeyInfo = !index->multikeyPaths.empty(); + auto tracksPathLevelMultikeyInfo = !metadata.indexes[offset].multikeyPaths.empty(); if (!tracksPathLevelMultikeyInfo) { invariant(multikeyPaths.empty()); @@ -2404,7 +2313,7 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, // We are tracking path-level multikey information for this index. invariant(!multikeyPaths.empty()); - invariant(multikeyPaths.size() == index->multikeyPaths.size()); + invariant(multikeyPaths.size() == metadata.indexes[offset].multikeyPaths.size()); index->multikey = true; @@ -2443,31 +2352,11 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, } BSONCollectionCatalogEntry::MetaData* metadata = nullptr; bool hasSetMultikey = false; - if (auto it = uncommittedMultikeys->find(this); it != uncommittedMultikeys->end()) { metadata = &it->second; hasSetMultikey = setMultikey(*metadata); } else { - // First time this OperationContext needs to change multikey information for this - // collection. We cannot use the cached metadata in this collection as we may have just - // committed a multikey change concurrently to the storage engine without being able to - // observe it if its onCommit handlers haven't run yet. - auto metadataLocal = *DurableCatalog::get(opCtx)->getMetaData(opCtx, getCatalogId()); - // When reading from the durable catalog the index offsets are different because when - // removing indexes in-memory just zeros out the slot instead of actually removing it. We - // must adjust the entries so they match how they are stored in _metadata so we can rely on - // the index offsets being stable. The order of valid indexes are the same, so we can - // iterate from the end and move them into the right positions. - int localIdx = metadataLocal.indexes.size() - 1; - metadataLocal.indexes.resize(_metadata->indexes.size()); - for (int i = _metadata->indexes.size() - 1; i >= 0 && localIdx != i; --i) { - if (_metadata->indexes[i].isPresent()) { - metadataLocal.indexes[i] = std::move(metadataLocal.indexes[localIdx]); - metadataLocal.indexes[localIdx] = {}; - --localIdx; - } - } - + BSONCollectionCatalogEntry::MetaData metadataLocal(*_metadata); hasSetMultikey = setMultikey(metadataLocal); if (hasSetMultikey) { metadata = &uncommittedMultikeys->emplace(this, std::move(metadataLocal)).first->second; @@ -2482,44 +2371,8 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, DurableCatalog::get(opCtx)->putMetaData(opCtx, getCatalogId(), *metadata); - // RAII Helper object to ensure we decrement the concurrent counter if and only if we - // incremented it in a preCommit handler. - class ConcurrentMultikeyWriteTracker { - public: - ConcurrentMultikeyWriteTracker( - std::shared_ptr<const BSONCollectionCatalogEntry::MetaData> meta, int indexOffset) - : metadata(std::move(meta)), offset(indexOffset) {} - - ~ConcurrentMultikeyWriteTracker() { - if (hasIncremented) { - metadata->indexes[offset].concurrentWriters.fetchAndSubtract(1); - } - } - - void preCommit() { - metadata->indexes[offset].concurrentWriters.fetchAndAdd(1); - hasIncremented = true; - } - - private: - std::shared_ptr<const BSONCollectionCatalogEntry::MetaData> metadata; - int offset; - bool hasIncremented = false; - }; - - auto concurrentWriteTracker = - std::make_shared<ConcurrentMultikeyWriteTracker>(_metadata, offset); - - // Mark this index that there is an ongoing multikey write. This forces readers to read from the - // durable catalog to determine if the index is multikey or not. - opCtx->recoveryUnit()->registerPreCommitHook( - [concurrentWriteTracker](OperationContext*) { concurrentWriteTracker->preCommit(); }); - - // Capture a reference to 'concurrentWriteTracker' to extend the lifetime of this object until - // commiting/rolling back the transaction is fully complete. opCtx->recoveryUnit()->onCommit( - [this, uncommittedMultikeys, setMultikey = std::move(setMultikey), concurrentWriteTracker]( - auto ts) { + [this, uncommittedMultikeys, setMultikey = std::move(setMultikey)](auto ts) { // Merge in changes to this index, other indexes may have been updated since we made our // copy. Don't check for result as another thread could be setting multikey at the same // time |