diff options
Diffstat (limited to 'src/mongo/db/query/canonical_query.cpp')
| -rw-r--r-- | src/mongo/db/query/canonical_query.cpp | 436 |
1 files changed, 307 insertions, 129 deletions
diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp index 32ebde51cfb..865cd10b245 100644 --- a/src/mongo/db/query/canonical_query.cpp +++ b/src/mongo/db/query/canonical_query.cpp @@ -36,6 +36,7 @@ #include "mongo/crypto/encryption_fields_gen.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/commands/test_commands_enabled.h" +#include "mongo/db/cst/cst_parser.h" #include "mongo/db/jsobj.h" #include "mongo/db/matcher/expression_array.h" #include "mongo/db/namespace_string.h" @@ -44,158 +45,162 @@ #include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/query/fle/server_rewrite.h" #include "mongo/db/query/indexability.h" -#include "mongo/db/query/parsed_find_command.h" #include "mongo/db/query/projection_parser.h" #include "mongo/db/query/query_planner_common.h" #include "mongo/logv2/log.h" namespace mongo { +namespace { + +bool parsingCanProduceNoopMatchNodes(const ExtensionsCallback& extensionsCallback, + MatchExpressionParser::AllowedFeatureSet allowedFeatures) { + return extensionsCallback.hasNoopExtensions() && + (allowedFeatures & MatchExpressionParser::AllowedFeatures::kText || + allowedFeatures & MatchExpressionParser::AllowedFeatures::kJavascript); +} + +} // namespace // static StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( OperationContext* opCtx, std::unique_ptr<FindCommandRequest> findCommand, bool explain, - const boost::intrusive_ptr<ExpressionContext>& givenExpCtx, + const boost::intrusive_ptr<ExpressionContext>& expCtx, const ExtensionsCallback& extensionsCallback, MatchExpressionParser::AllowedFeatureSet allowedFeatures, const ProjectionPolicies& projectionPolicies, std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) { + tassert(5746107, + "ntoreturn should not be set on the findCommand", + findCommand->getNtoreturn() == boost::none); - if (givenExpCtx) { - // Caller provided an ExpressionContext, let's go ahead and use that. - auto swParsedFind = parsed_find_command::parse(givenExpCtx, - std::move(findCommand), - extensionsCallback, - allowedFeatures, - projectionPolicies); - if (!swParsedFind.isOK()) { - return swParsedFind.getStatus(); + auto status = query_request_helper::validateFindCommandRequest(*findCommand); + if (!status.isOK()) { + return status; + } + + std::unique_ptr<CollatorInterface> collator; + if (!findCommand->getCollation().isEmpty()) { + auto statusWithCollator = CollatorFactoryInterface::get(opCtx->getServiceContext()) + ->makeFromBSON(findCommand->getCollation()); + if (!statusWithCollator.isOK()) { + return statusWithCollator.getStatus(); } - return canonicalize(std::move(givenExpCtx), - std::move(swParsedFind.getValue()), - explain, - std::move(pipeline)); + collator = std::move(statusWithCollator.getValue()); + } + + // Make MatchExpression. + boost::intrusive_ptr<ExpressionContext> newExpCtx; + if (!expCtx.get()) { + invariant(findCommand->getNamespaceOrUUID().nss()); + newExpCtx = make_intrusive<ExpressionContext>(opCtx, + std::move(collator), + *findCommand->getNamespaceOrUUID().nss(), + findCommand->getLegacyRuntimeConstants(), + findCommand->getLet()); } else { - // No ExpressionContext provided, let's call the override that makes one for us. - auto swResults = parsed_find_command::parse( - opCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies); - if (!swResults.isOK()) { - return swResults.getStatus(); + newExpCtx = expCtx; + // A collator can enter through both the FindCommandRequest and ExpressionContext arguments. + // This invariant ensures that both collators are the same because downstream we + // pull the collator from only one of the ExpressionContext carrier. + if (collator.get() && expCtx->getCollator()) { + invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator())); } - auto&& [expCtx, parsedFind] = std::move(swResults.getValue()); - return canonicalize(std::move(expCtx), std::move(parsedFind), explain, std::move(pipeline)); } -} - -// static -StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( - boost::intrusive_ptr<ExpressionContext> expCtx, - std::unique_ptr<ParsedFindCommand> parsedFind, - bool explain, - std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) { // Make the CQ we'll hopefully return. - auto cq = std::make_unique<CanonicalQuery>(); + std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery()); cq->setExplain(explain); - if (auto initStatus = cq->init(std::move(expCtx), - std::move(parsedFind), - std::move(pipeline), - true /*optimizeMatchExpression*/); - !initStatus.isOK()) { + + StatusWithMatchExpression statusWithMatcher = [&]() -> StatusWithMatchExpression { + if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) { + try { + return cst::parseToMatchExpression( + findCommand->getFilter(), newExpCtx, extensionsCallback); + } catch (const DBException& ex) { + return ex.toStatus(); + } + } else { + return MatchExpressionParser::parse( + findCommand->getFilter(), newExpCtx, extensionsCallback, allowedFeatures); + } + }(); + if (!statusWithMatcher.isOK()) { + return statusWithMatcher.getStatus(); + } + + // Stop counting expressions after they have been parsed to exclude expressions created + // during optimization and other processing steps. + newExpCtx->stopExpressionCounters(); + + std::unique_ptr<MatchExpression> me = std::move(statusWithMatcher.getValue()); + + Status initStatus = + cq->init(opCtx, + std::move(newExpCtx), + std::move(findCommand), + parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures), + std::move(me), + projectionPolicies, + std::move(pipeline)); + + if (!initStatus.isOK()) { return initStatus; } - return {std::move(cq)}; + return std::move(cq); } // static -StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::makeForSubplanner( - OperationContext* opCtx, const CanonicalQuery& baseQuery, size_t i) { - tassert(8401301, - "expected MatchExpression with rooted $or", - baseQuery.root()->matchType() == MatchExpression::OR); - tassert(8401302, - "attempted to get out of bounds child of $or", - baseQuery.root()->numChildren() > i); - auto root = baseQuery.root()->getChild(i); +StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( + OperationContext* opCtx, const CanonicalQuery& baseQuery, MatchExpression* root) { auto findCommand = std::make_unique<FindCommandRequest>(baseQuery.nss()); - findCommand->setFilter(root->serialize()); + BSONObjBuilder builder; + root->serialize(&builder, true); + findCommand->setFilter(builder.obj()); findCommand->setProjection(baseQuery.getFindCommandRequest().getProjection().getOwned()); findCommand->setSort(baseQuery.getFindCommandRequest().getSort().getOwned()); findCommand->setCollation(baseQuery.getFindCommandRequest().getCollation().getOwned()); + auto status = query_request_helper::validateFindCommandRequest(*findCommand); + if (!status.isOK()) { + return status; + } // Make the CQ we'll hopefully return. - auto cq = std::make_unique<CanonicalQuery>(); + std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery()); cq->setExplain(baseQuery.getExplain()); - auto swParsedFind = ParsedFindCommand::withExistingFilter( - baseQuery.getExpCtx(), - baseQuery.getCollator() ? baseQuery.getCollator()->clone() : nullptr, - root->shallowClone(), - std::move(findCommand)); - if (!swParsedFind.isOK()) { - return swParsedFind.getStatus(); - } - // Note: we do not optimize the MatchExpression representing the branch of the top-level $or - // that we are currently examining. This is because repeated invocations of - // MatchExpression::optimize() may change the order of predicates in the MatchExpression, due to - // new rewrites being unlocked by previous ones. We need to preserve the order of predicates to - // allow index tagging to work properly. See SERVER-84013 for more details. - Status initStatus = cq->init(baseQuery.getExpCtx(), - std::move(swParsedFind.getValue()), - {} /* an empty pipeline */, - false /*optimizeMatchExpression*/); - - invariant(initStatus.isOK()); - return {std::move(cq)}; + Status initStatus = cq->init(opCtx, + baseQuery.getExpCtx(), + std::move(findCommand), + baseQuery.canHaveNoopMatchNodes(), + root->shallowClone(), + ProjectionPolicies::findProjectionPolicies(), + {} /* an empty pipeline */); + + if (!initStatus.isOK()) { + return initStatus; + } + return std::move(cq); } -Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx, - std::unique_ptr<ParsedFindCommand> parsedFind, - std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline, - bool optimizeMatchExpression) { +Status CanonicalQuery::init(OperationContext* opCtx, + boost::intrusive_ptr<ExpressionContext> expCtx, + std::unique_ptr<FindCommandRequest> findCommand, + bool canHaveNoopMatchNodes, + std::unique_ptr<MatchExpression> root, + const ProjectionPolicies& projectionPolicies, + std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) { _expCtx = expCtx; - _findCommand = std::move(parsedFind->findCommandRequest); - _canHaveNoopMatchNodes = parsedFind->canHaveNoopMatchNodes; + _findCommand = std::move(findCommand); + _canHaveNoopMatchNodes = canHaveNoopMatchNodes; _forceClassicEngine = internalQueryForceClassicEngine.load(); - if (optimizeMatchExpression) { - _root = MatchExpression::normalize(std::move(parsedFind->filter)); - } else { - _root = std::move(parsedFind->filter); - } - - if (parsedFind->proj) { - // The projection will be optimized only if the query is not compatible with SBE or there's - // no user-specified "let" variable. This is to prevent the user-defined variable being - // optimized out. We will optimize the projection later after we are certain that the query - // is ineligible for SBE. - bool shouldOptimizeProj = !expCtx->sbeCompatible || !_findCommand->getLet(); - if (parsedFind->proj->requiresMatchDetails()) { - // Sadly, in some cases the match details cannot be generated from the unoptimized - // MatchExpression. For example, a rooted-$or of equalities won't work to produce the - // details, but if you optimize that query to an $in, it will work. If we were starting - // from scratch, we may disallow this. But it has already been released as working so we - // will keep it so, and here have to re-parse the projection using the new, normalized - // MatchExpression, before we save this projection for later execution. - _proj.emplace(projection_ast::parseAndAnalyze(expCtx, - _findCommand->getProjection(), - _root.get(), - _findCommand->getFilter(), - *parsedFind->savedProjectionPolicies, - shouldOptimizeProj)); - } else { - _proj.emplace(std::move(*parsedFind->proj)); - if (shouldOptimizeProj) { - _proj->optimize(); - } - } - } - if (parsedFind->sort) { - _sortPattern = std::move(parsedFind->sort); + auto validStatus = isValid(root.get(), *_findCommand); + if (!validStatus.isOK()) { + return validStatus.getStatus(); } - _pipeline = std::move(pipeline); - - // Perform auto-parameterization only if the query is SBE-compatible and caching is enabled. + auto unavailableMetadata = validStatus.getValue(); + _root = MatchExpression::normalize(std::move(root)); if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) { const bool hasNoTextNodes = !QueryPlannerCommon::hasNode(_root.get(), MatchExpression::TEXT); @@ -212,45 +217,89 @@ Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx, } } // The tree must always be valid after normalization. - dassert(parsed_find_command::isValid(_root.get(), *_findCommand).isOK()); + dassert(isValid(_root.get(), *_findCommand).isOK()); if (auto status = isValidNormalized(_root.get()); !status.isOK()) { return status; } - if (_proj) { - _metadataDeps = _proj->metadataDeps(); + // Validate the projection if there is one. + if (!_findCommand->getProjection().isEmpty()) { + try { + _proj.emplace(projection_ast::parseAndAnalyze(expCtx, + _findCommand->getProjection(), + _root.get(), + _findCommand->getFilter(), + projectionPolicies, + true /* Should optimize? */)); - if (_proj->metadataDeps()[DocumentMetadataFields::kSortKey] && - _findCommand->getSort().isEmpty()) { - return {ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort"}; + // Fail if any of the projection's dependencies are unavailable. + DepsTracker{unavailableMetadata}.requestMetadata(_proj->metadataDeps()); + } catch (const DBException& e) { + return e.toStatus(); } + + _metadataDeps = _proj->metadataDeps(); } - if (_sortPattern) { - // Be sure to track and add any metadata dependencies from the sort (e.g. text score). - _metadataDeps |= _sortPattern->metadataDeps(parsedFind->unavailableMetadata); + _pipeline = std::move(pipeline); - // If the results of this query might have to be merged on a remote node, then that node - // might need the sort key metadata. Request that the plan generates this metadata. - if (_expCtx->needsMerge) { - _metadataDeps.set(DocumentMetadataFields::kSortKey); - } + if (_proj && _proj->metadataDeps()[DocumentMetadataFields::kSortKey] && + _findCommand->getSort().isEmpty()) { + return Status(ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort"); + } + + // If there is a sort, parse it and add any metadata dependencies it induces. + try { + initSortPattern(unavailableMetadata); + } catch (const DBException& ex) { + return ex.toStatus(); } // If the 'returnKey' option is set, then the plan should produce index key metadata. if (_findCommand->getReturnKey()) { _metadataDeps.set(DocumentMetadataFields::kIndexKey); } + return Status::OK(); } +void CanonicalQuery::initSortPattern(QueryMetadataBitSet unavailableMetadata) { + if (_findCommand->getSort().isEmpty()) { + return; + } + + // A $natural sort is really a hint, and should be handled as such. Furthermore, the downstream + // sort handling code may not expect a $natural sort. + // + // We have already validated that if there is a $natural sort and a hint, that the hint + // also specifies $natural with the same direction. Therefore, it is safe to clear the $natural + // sort and rewrite it as a $natural hint. + if (_findCommand->getSort()[query_request_helper::kNaturalSortField]) { + _findCommand->setHint(_findCommand->getSort().getOwned()); + _findCommand->setSort(BSONObj{}); + } + + if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) { + _sortPattern = cst::parseToSortPattern(_findCommand->getSort(), _expCtx); + } else { + _sortPattern = SortPattern{_findCommand->getSort(), _expCtx}; + } + _metadataDeps |= _sortPattern->metadataDeps(unavailableMetadata); + + // If the results of this query might have to be merged on a remote node, then that node might + // need the sort key metadata. Request that the plan generates this metadata. + if (_expCtx->needsMerge) { + _metadataDeps.set(DocumentMetadataFields::kSortKey); + } +} + void CanonicalQuery::setCollator(std::unique_ptr<CollatorInterface> collator) { auto collatorRaw = collator.get(); // We must give the ExpressionContext the same collator. _expCtx->setCollator(std::move(collator)); - // The collator associated with the match expression tree is now invalid, since we have - // reset the collator owned by the ExpressionContext. + // The collator associated with the match expression tree is now invalid, since we have reset + // the collator owned by the ExpressionContext. _root->setCollator(collatorRaw); } @@ -284,9 +333,138 @@ bool CanonicalQuery::isSimpleIdQuery(const BSONObj& query) { return hasID; } +size_t CanonicalQuery::countNodes(const MatchExpression* root, MatchExpression::MatchType type) { + size_t sum = 0; + if (type == root->matchType()) { + sum = 1; + } + for (size_t i = 0; i < root->numChildren(); ++i) { + sum += countNodes(root->getChild(i), type); + } + return sum; +} + +/** + * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside? + */ +bool hasNodeInSubtree(const MatchExpression* root, + MatchExpression::MatchType childType, + MatchExpression::MatchType subtreeType) { + if (subtreeType == root->matchType()) { + return QueryPlannerCommon::hasNode(root, childType); + } + for (size_t i = 0; i < root->numChildren(); ++i) { + if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) { + return true; + } + } + return false; +} + +StatusWith<QueryMetadataBitSet> CanonicalQuery::isValid(const MatchExpression* root, + const FindCommandRequest& findCommand) { + QueryMetadataBitSet unavailableMetadata{}; + + // There can only be one TEXT. If there is a TEXT, it cannot appear inside a NOR. + // + // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT + // inside of value-expression clauses like NOT, so we don't check those here. + size_t numText = countNodes(root, MatchExpression::TEXT); + if (numText > 1) { + return Status(ErrorCodes::BadValue, "Too many text expressions"); + } else if (1 == numText) { + if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) { + return Status(ErrorCodes::BadValue, "text expression not allowed in nor"); + } + } else { + // Text metadata is not available. + unavailableMetadata.set(DocumentMetadataFields::kTextScore); + } + + // There can only be one NEAR. If there is a NEAR, it must be either the root or the root + // must be an AND and its child must be a NEAR. + size_t numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); + if (numGeoNear > 1) { + return Status(ErrorCodes::BadValue, "Too many geoNear expressions"); + } else if (1 == numGeoNear) { + // Do nothing, we will perform extra checks in CanonicalQuery::isValidNormalized. + } else { + // Geo distance and geo point metadata are unavailable. + unavailableMetadata |= DepsTracker::kAllGeoNearData; + } + + const BSONObj& sortObj = findCommand.getSort(); + BSONElement sortNaturalElt = sortObj["$natural"]; + const BSONObj& hintObj = findCommand.getHint(); + BSONElement hintNaturalElt = hintObj["$natural"]; + + if (sortNaturalElt && sortObj.nFields() != 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Cannot include '$natural' in compound sort: " << sortObj); + } + + if (hintNaturalElt && hintObj.nFields() != 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Cannot include '$natural' in compound hint: " << hintObj); + } + + // NEAR cannot have a $natural sort or $natural hint. + if (numGeoNear > 0) { + if (sortNaturalElt) { + return Status(ErrorCodes::BadValue, + "geoNear expression not allowed with $natural sort order"); + } + + if (hintNaturalElt) { + return Status(ErrorCodes::BadValue, + "geoNear expression not allowed with $natural hint"); + } + } + + // TEXT and NEAR cannot both be in the query. + if (numText > 0 && numGeoNear > 0) { + return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query"); + } + + // TEXT and {$natural: ...} sort order cannot both be in the query. + if (numText > 0 && sortNaturalElt) { + return Status(ErrorCodes::BadValue, "text expression not allowed with $natural sort order"); + } + + // TEXT and hint cannot both be in the query. + if (numText > 0 && !hintObj.isEmpty()) { + return Status(ErrorCodes::BadValue, "text and hint not allowed in same query"); + } + + // TEXT and tailable are incompatible. + if (numText > 0 && findCommand.getTailable()) { + return Status(ErrorCodes::BadValue, "text and tailable cursor not allowed in same query"); + } + + // NEAR and tailable are incompatible. + if (numGeoNear > 0 && findCommand.getTailable()) { + return Status(ErrorCodes::BadValue, + "Tailable cursors and geo $near cannot be used together"); + } + + // $natural sort order must agree with hint. + if (sortNaturalElt) { + if (!hintObj.isEmpty() && !hintNaturalElt) { + return Status(ErrorCodes::BadValue, "index hint not allowed with $natural sort order"); + } + if (hintNaturalElt) { + if (hintNaturalElt.numberInt() != sortNaturalElt.numberInt()) { + return Status(ErrorCodes::BadValue, + "$natural hint must be in the same direction as $natural sort order"); + } + } + } + + return unavailableMetadata; +} + Status CanonicalQuery::isValidNormalized(const MatchExpression* root) { - if (auto numGeoNear = QueryPlannerCommon::countNodes(root, MatchExpression::GEO_NEAR); - numGeoNear > 0) { + if (auto numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); numGeoNear > 0) { tassert(5705300, "Only one geo $near expression is expected", numGeoNear == 1); auto topLevel = false; |