Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 307 insertions, 129 deletions

diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp
index 32ebde51cfb..865cd10b245 100644
--- a/src/mongo/db/query/canonical_query.cpp
+++ b/src/mongo/db/query/canonical_query.cpp
@@ -36,6 +36,7 @@
 #include "mongo/crypto/encryption_fields_gen.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/commands/test_commands_enabled.h"
+#include "mongo/db/cst/cst_parser.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/matcher/expression_array.h"
 #include "mongo/db/namespace_string.h"
@@ -44,158 +45,162 @@
 #include "mongo/db/query/collation/collator_factory_interface.h"
 #include "mongo/db/query/fle/server_rewrite.h"
 #include "mongo/db/query/indexability.h"
-#include "mongo/db/query/parsed_find_command.h"
 #include "mongo/db/query/projection_parser.h"
 #include "mongo/db/query/query_planner_common.h"
 #include "mongo/logv2/log.h"
 namespace mongo {
+namespace {
+
+bool parsingCanProduceNoopMatchNodes(const ExtensionsCallback& extensionsCallback,
+                                     MatchExpressionParser::AllowedFeatureSet allowedFeatures) {
+    return extensionsCallback.hasNoopExtensions() &&
+        (allowedFeatures & MatchExpressionParser::AllowedFeatures::kText ||
+         allowedFeatures & MatchExpressionParser::AllowedFeatures::kJavascript);
+}
+
+}  // namespace
 
 // static
 StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize(
     OperationContext* opCtx,
     std::unique_ptr<FindCommandRequest> findCommand,
     bool explain,
-    const boost::intrusive_ptr<ExpressionContext>& givenExpCtx,
+    const boost::intrusive_ptr<ExpressionContext>& expCtx,
     const ExtensionsCallback& extensionsCallback,
     MatchExpressionParser::AllowedFeatureSet allowedFeatures,
     const ProjectionPolicies& projectionPolicies,
     std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) {
+    tassert(5746107,
+            "ntoreturn should not be set on the findCommand",
+            findCommand->getNtoreturn() == boost::none);
 
-    if (givenExpCtx) {
-        // Caller provided an ExpressionContext, let's go ahead and use that.
-        auto swParsedFind = parsed_find_command::parse(givenExpCtx,
-                                                       std::move(findCommand),
-                                                       extensionsCallback,
-                                                       allowedFeatures,
-                                                       projectionPolicies);
-        if (!swParsedFind.isOK()) {
-            return swParsedFind.getStatus();
+    auto status = query_request_helper::validateFindCommandRequest(*findCommand);
+    if (!status.isOK()) {
+        return status;
+    }
+
+    std::unique_ptr<CollatorInterface> collator;
+    if (!findCommand->getCollation().isEmpty()) {
+        auto statusWithCollator = CollatorFactoryInterface::get(opCtx->getServiceContext())
+                                      ->makeFromBSON(findCommand->getCollation());
+        if (!statusWithCollator.isOK()) {
+            return statusWithCollator.getStatus();
         }
-        return canonicalize(std::move(givenExpCtx),
-                            std::move(swParsedFind.getValue()),
-                            explain,
-                            std::move(pipeline));
+        collator = std::move(statusWithCollator.getValue());
+    }
+
+    // Make MatchExpression.
+    boost::intrusive_ptr<ExpressionContext> newExpCtx;
+    if (!expCtx.get()) {
+        invariant(findCommand->getNamespaceOrUUID().nss());
+        newExpCtx = make_intrusive<ExpressionContext>(opCtx,
+                                                      std::move(collator),
+                                                      *findCommand->getNamespaceOrUUID().nss(),
+                                                      findCommand->getLegacyRuntimeConstants(),
+                                                      findCommand->getLet());
     } else {
-        // No ExpressionContext provided, let's call the override that makes one for us.
-        auto swResults = parsed_find_command::parse(
-            opCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies);
-        if (!swResults.isOK()) {
-            return swResults.getStatus();
+        newExpCtx = expCtx;
+        // A collator can enter through both the FindCommandRequest and ExpressionContext arguments.
+        // This invariant ensures that both collators are the same because downstream we
+        // pull the collator from only one of the ExpressionContext carrier.
+        if (collator.get() && expCtx->getCollator()) {
+            invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator()));
         }
-        auto&& [expCtx, parsedFind] = std::move(swResults.getValue());
-        return canonicalize(std::move(expCtx), std::move(parsedFind), explain, std::move(pipeline));
     }
-}
-
-// static
-StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize(
-    boost::intrusive_ptr<ExpressionContext> expCtx,
-    std::unique_ptr<ParsedFindCommand> parsedFind,
-    bool explain,
-    std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) {
 
     // Make the CQ we'll hopefully return.
-    auto cq = std::make_unique<CanonicalQuery>();
+    std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery());
     cq->setExplain(explain);
-    if (auto initStatus = cq->init(std::move(expCtx),
-                                   std::move(parsedFind),
-                                   std::move(pipeline),
-                                   true /*optimizeMatchExpression*/);
-        !initStatus.isOK()) {
+
+    StatusWithMatchExpression statusWithMatcher = [&]() -> StatusWithMatchExpression {
+        if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) {
+            try {
+                return cst::parseToMatchExpression(
+                    findCommand->getFilter(), newExpCtx, extensionsCallback);
+            } catch (const DBException& ex) {
+                return ex.toStatus();
+            }
+        } else {
+            return MatchExpressionParser::parse(
+                findCommand->getFilter(), newExpCtx, extensionsCallback, allowedFeatures);
+        }
+    }();
+    if (!statusWithMatcher.isOK()) {
+        return statusWithMatcher.getStatus();
+    }
+
+    // Stop counting expressions after they have been parsed to exclude expressions created
+    // during optimization and other processing steps.
+    newExpCtx->stopExpressionCounters();
+
+    std::unique_ptr<MatchExpression> me = std::move(statusWithMatcher.getValue());
+
+    Status initStatus =
+        cq->init(opCtx,
+                 std::move(newExpCtx),
+                 std::move(findCommand),
+                 parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures),
+                 std::move(me),
+                 projectionPolicies,
+                 std::move(pipeline));
+
+    if (!initStatus.isOK()) {
         return initStatus;
     }
-    return {std::move(cq)};
+    return std::move(cq);
 }
 
 // static
-StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::makeForSubplanner(
-    OperationContext* opCtx, const CanonicalQuery& baseQuery, size_t i) {
-    tassert(8401301,
-            "expected MatchExpression with rooted $or",
-            baseQuery.root()->matchType() == MatchExpression::OR);
-    tassert(8401302,
-            "attempted to get out of bounds child of $or",
-            baseQuery.root()->numChildren() > i);
-    auto root = baseQuery.root()->getChild(i);
+StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize(
+    OperationContext* opCtx, const CanonicalQuery& baseQuery, MatchExpression* root) {
     auto findCommand = std::make_unique<FindCommandRequest>(baseQuery.nss());
-    findCommand->setFilter(root->serialize());
+    BSONObjBuilder builder;
+    root->serialize(&builder, true);
+    findCommand->setFilter(builder.obj());
     findCommand->setProjection(baseQuery.getFindCommandRequest().getProjection().getOwned());
     findCommand->setSort(baseQuery.getFindCommandRequest().getSort().getOwned());
     findCommand->setCollation(baseQuery.getFindCommandRequest().getCollation().getOwned());
+    auto status = query_request_helper::validateFindCommandRequest(*findCommand);
+    if (!status.isOK()) {
+        return status;
+    }
 
     // Make the CQ we'll hopefully return.
-    auto cq = std::make_unique<CanonicalQuery>();
+    std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery());
     cq->setExplain(baseQuery.getExplain());
-    auto swParsedFind = ParsedFindCommand::withExistingFilter(
-        baseQuery.getExpCtx(),
-        baseQuery.getCollator() ? baseQuery.getCollator()->clone() : nullptr,
-        root->shallowClone(),
-        std::move(findCommand));
-    if (!swParsedFind.isOK()) {
-        return swParsedFind.getStatus();
-    }
-    // Note: we do not optimize the MatchExpression representing the branch of the top-level $or
-    // that we are currently examining. This is because repeated invocations of
-    // MatchExpression::optimize() may change the order of predicates in the MatchExpression, due to
-    // new rewrites being unlocked by previous ones. We need to preserve the order of predicates to
-    // allow index tagging to work properly. See SERVER-84013 for more details.
-    Status initStatus = cq->init(baseQuery.getExpCtx(),
-                                 std::move(swParsedFind.getValue()),
-                                 {} /* an empty pipeline */,
-                                 false /*optimizeMatchExpression*/);
-
-    invariant(initStatus.isOK());
-    return {std::move(cq)};
+    Status initStatus = cq->init(opCtx,
+                                 baseQuery.getExpCtx(),
+                                 std::move(findCommand),
+                                 baseQuery.canHaveNoopMatchNodes(),
+                                 root->shallowClone(),
+                                 ProjectionPolicies::findProjectionPolicies(),
+                                 {} /* an empty pipeline */);
+
+    if (!initStatus.isOK()) {
+        return initStatus;
+    }
+    return std::move(cq);
 }
 
-Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx,
-                            std::unique_ptr<ParsedFindCommand> parsedFind,
-                            std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline,
-                            bool optimizeMatchExpression) {
+Status CanonicalQuery::init(OperationContext* opCtx,
+                            boost::intrusive_ptr<ExpressionContext> expCtx,
+                            std::unique_ptr<FindCommandRequest> findCommand,
+                            bool canHaveNoopMatchNodes,
+                            std::unique_ptr<MatchExpression> root,
+                            const ProjectionPolicies& projectionPolicies,
+                            std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) {
     _expCtx = expCtx;
-    _findCommand = std::move(parsedFind->findCommandRequest);
-    _canHaveNoopMatchNodes = parsedFind->canHaveNoopMatchNodes;
+    _findCommand = std::move(findCommand);
 
+    _canHaveNoopMatchNodes = canHaveNoopMatchNodes;
     _forceClassicEngine = internalQueryForceClassicEngine.load();
 
-    if (optimizeMatchExpression) {
-        _root = MatchExpression::normalize(std::move(parsedFind->filter));
-    } else {
-        _root = std::move(parsedFind->filter);
-    }
-
-    if (parsedFind->proj) {
-        // The projection will be optimized only if the query is not compatible with SBE or there's
-        // no user-specified "let" variable. This is to prevent the user-defined variable being
-        // optimized out. We will optimize the projection later after we are certain that the query
-        // is ineligible for SBE.
-        bool shouldOptimizeProj = !expCtx->sbeCompatible || !_findCommand->getLet();
-        if (parsedFind->proj->requiresMatchDetails()) {
-            // Sadly, in some cases the match details cannot be generated from the unoptimized
-            // MatchExpression. For example, a rooted-$or of equalities won't work to produce the
-            // details, but if you optimize that query to an $in, it will work. If we were starting
-            // from scratch, we may disallow this. But it has already been released as working so we
-            // will keep it so, and here have to re-parse the projection using the new, normalized
-            // MatchExpression, before we save this projection for later execution.
-            _proj.emplace(projection_ast::parseAndAnalyze(expCtx,
-                                                          _findCommand->getProjection(),
-                                                          _root.get(),
-                                                          _findCommand->getFilter(),
-                                                          *parsedFind->savedProjectionPolicies,
-                                                          shouldOptimizeProj));
-        } else {
-            _proj.emplace(std::move(*parsedFind->proj));
-            if (shouldOptimizeProj) {
-                _proj->optimize();
-            }
-        }
-    }
-    if (parsedFind->sort) {
-        _sortPattern = std::move(parsedFind->sort);
+    auto validStatus = isValid(root.get(), *_findCommand);
+    if (!validStatus.isOK()) {
+        return validStatus.getStatus();
     }
-    _pipeline = std::move(pipeline);
-
-    // Perform auto-parameterization only if the query is SBE-compatible and caching is enabled.
+    auto unavailableMetadata = validStatus.getValue();
+    _root = MatchExpression::normalize(std::move(root));
     if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) {
         const bool hasNoTextNodes =
             !QueryPlannerCommon::hasNode(_root.get(), MatchExpression::TEXT);
@@ -212,45 +217,89 @@ Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx,
         }
     }
     // The tree must always be valid after normalization.
-    dassert(parsed_find_command::isValid(_root.get(), *_findCommand).isOK());
+    dassert(isValid(_root.get(), *_findCommand).isOK());
     if (auto status = isValidNormalized(_root.get()); !status.isOK()) {
         return status;
     }
 
-    if (_proj) {
-        _metadataDeps = _proj->metadataDeps();
+    // Validate the projection if there is one.
+    if (!_findCommand->getProjection().isEmpty()) {
+        try {
+            _proj.emplace(projection_ast::parseAndAnalyze(expCtx,
+                                                          _findCommand->getProjection(),
+                                                          _root.get(),
+                                                          _findCommand->getFilter(),
+                                                          projectionPolicies,
+                                                          true /* Should optimize? */));
 
-        if (_proj->metadataDeps()[DocumentMetadataFields::kSortKey] &&
-            _findCommand->getSort().isEmpty()) {
-            return {ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort"};
+            // Fail if any of the projection's dependencies are unavailable.
+            DepsTracker{unavailableMetadata}.requestMetadata(_proj->metadataDeps());
+        } catch (const DBException& e) {
+            return e.toStatus();
         }
+
+        _metadataDeps = _proj->metadataDeps();
     }
 
-    if (_sortPattern) {
-        // Be sure to track and add any metadata dependencies from the sort (e.g. text score).
-        _metadataDeps |= _sortPattern->metadataDeps(parsedFind->unavailableMetadata);
+    _pipeline = std::move(pipeline);
 
-        // If the results of this query might have to be merged on a remote node, then that node
-        // might need the sort key metadata. Request that the plan generates this metadata.
-        if (_expCtx->needsMerge) {
-            _metadataDeps.set(DocumentMetadataFields::kSortKey);
-        }
+    if (_proj && _proj->metadataDeps()[DocumentMetadataFields::kSortKey] &&
+        _findCommand->getSort().isEmpty()) {
+        return Status(ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort");
+    }
+
+    // If there is a sort, parse it and add any metadata dependencies it induces.
+    try {
+        initSortPattern(unavailableMetadata);
+    } catch (const DBException& ex) {
+        return ex.toStatus();
     }
 
     // If the 'returnKey' option is set, then the plan should produce index key metadata.
     if (_findCommand->getReturnKey()) {
         _metadataDeps.set(DocumentMetadataFields::kIndexKey);
     }
+
     return Status::OK();
 }
 
+void CanonicalQuery::initSortPattern(QueryMetadataBitSet unavailableMetadata) {
+    if (_findCommand->getSort().isEmpty()) {
+        return;
+    }
+
+    // A $natural sort is really a hint, and should be handled as such. Furthermore, the downstream
+    // sort handling code may not expect a $natural sort.
+    //
+    // We have already validated that if there is a $natural sort and a hint, that the hint
+    // also specifies $natural with the same direction. Therefore, it is safe to clear the $natural
+    // sort and rewrite it as a $natural hint.
+    if (_findCommand->getSort()[query_request_helper::kNaturalSortField]) {
+        _findCommand->setHint(_findCommand->getSort().getOwned());
+        _findCommand->setSort(BSONObj{});
+    }
+
+    if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) {
+        _sortPattern = cst::parseToSortPattern(_findCommand->getSort(), _expCtx);
+    } else {
+        _sortPattern = SortPattern{_findCommand->getSort(), _expCtx};
+    }
+    _metadataDeps |= _sortPattern->metadataDeps(unavailableMetadata);
+
+    // If the results of this query might have to be merged on a remote node, then that node might
+    // need the sort key metadata. Request that the plan generates this metadata.
+    if (_expCtx->needsMerge) {
+        _metadataDeps.set(DocumentMetadataFields::kSortKey);
+    }
+}
+
 void CanonicalQuery::setCollator(std::unique_ptr<CollatorInterface> collator) {
     auto collatorRaw = collator.get();
     // We must give the ExpressionContext the same collator.
     _expCtx->setCollator(std::move(collator));
 
-    // The collator associated with the match expression tree is now invalid, since we have
-    // reset the collator owned by the ExpressionContext.
+    // The collator associated with the match expression tree is now invalid, since we have reset
+    // the collator owned by the ExpressionContext.
     _root->setCollator(collatorRaw);
 }
 
@@ -284,9 +333,138 @@ bool CanonicalQuery::isSimpleIdQuery(const BSONObj& query) {
     return hasID;
 }
 
+size_t CanonicalQuery::countNodes(const MatchExpression* root, MatchExpression::MatchType type) {
+    size_t sum = 0;
+    if (type == root->matchType()) {
+        sum = 1;
+    }
+    for (size_t i = 0; i < root->numChildren(); ++i) {
+        sum += countNodes(root->getChild(i), type);
+    }
+    return sum;
+}
+
+/**
+ * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside?
+ */
+bool hasNodeInSubtree(const MatchExpression* root,
+                      MatchExpression::MatchType childType,
+                      MatchExpression::MatchType subtreeType) {
+    if (subtreeType == root->matchType()) {
+        return QueryPlannerCommon::hasNode(root, childType);
+    }
+    for (size_t i = 0; i < root->numChildren(); ++i) {
+        if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+StatusWith<QueryMetadataBitSet> CanonicalQuery::isValid(const MatchExpression* root,
+                                                        const FindCommandRequest& findCommand) {
+    QueryMetadataBitSet unavailableMetadata{};
+
+    // There can only be one TEXT.  If there is a TEXT, it cannot appear inside a NOR.
+    //
+    // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT
+    // inside of value-expression clauses like NOT, so we don't check those here.
+    size_t numText = countNodes(root, MatchExpression::TEXT);
+    if (numText > 1) {
+        return Status(ErrorCodes::BadValue, "Too many text expressions");
+    } else if (1 == numText) {
+        if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) {
+            return Status(ErrorCodes::BadValue, "text expression not allowed in nor");
+        }
+    } else {
+        // Text metadata is not available.
+        unavailableMetadata.set(DocumentMetadataFields::kTextScore);
+    }
+
+    // There can only be one NEAR.  If there is a NEAR, it must be either the root or the root
+    // must be an AND and its child must be a NEAR.
+    size_t numGeoNear = countNodes(root, MatchExpression::GEO_NEAR);
+    if (numGeoNear > 1) {
+        return Status(ErrorCodes::BadValue, "Too many geoNear expressions");
+    } else if (1 == numGeoNear) {
+        // Do nothing, we will perform extra checks in CanonicalQuery::isValidNormalized.
+    } else {
+        // Geo distance and geo point metadata are unavailable.
+        unavailableMetadata |= DepsTracker::kAllGeoNearData;
+    }
+
+    const BSONObj& sortObj = findCommand.getSort();
+    BSONElement sortNaturalElt = sortObj["$natural"];
+    const BSONObj& hintObj = findCommand.getHint();
+    BSONElement hintNaturalElt = hintObj["$natural"];
+
+    if (sortNaturalElt && sortObj.nFields() != 1) {
+        return Status(ErrorCodes::BadValue,
+                      str::stream() << "Cannot include '$natural' in compound sort: " << sortObj);
+    }
+
+    if (hintNaturalElt && hintObj.nFields() != 1) {
+        return Status(ErrorCodes::BadValue,
+                      str::stream() << "Cannot include '$natural' in compound hint: " << hintObj);
+    }
+
+    // NEAR cannot have a $natural sort or $natural hint.
+    if (numGeoNear > 0) {
+        if (sortNaturalElt) {
+            return Status(ErrorCodes::BadValue,
+                          "geoNear expression not allowed with $natural sort order");
+        }
+
+        if (hintNaturalElt) {
+            return Status(ErrorCodes::BadValue,
+                          "geoNear expression not allowed with $natural hint");
+        }
+    }
+
+    // TEXT and NEAR cannot both be in the query.
+    if (numText > 0 && numGeoNear > 0) {
+        return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query");
+    }
+
+    // TEXT and {$natural: ...} sort order cannot both be in the query.
+    if (numText > 0 && sortNaturalElt) {
+        return Status(ErrorCodes::BadValue, "text expression not allowed with $natural sort order");
+    }
+
+    // TEXT and hint cannot both be in the query.
+    if (numText > 0 && !hintObj.isEmpty()) {
+        return Status(ErrorCodes::BadValue, "text and hint not allowed in same query");
+    }
+
+    // TEXT and tailable are incompatible.
+    if (numText > 0 && findCommand.getTailable()) {
+        return Status(ErrorCodes::BadValue, "text and tailable cursor not allowed in same query");
+    }
+
+    // NEAR and tailable are incompatible.
+    if (numGeoNear > 0 && findCommand.getTailable()) {
+        return Status(ErrorCodes::BadValue,
+                      "Tailable cursors and geo $near cannot be used together");
+    }
+
+    // $natural sort order must agree with hint.
+    if (sortNaturalElt) {
+        if (!hintObj.isEmpty() && !hintNaturalElt) {
+            return Status(ErrorCodes::BadValue, "index hint not allowed with $natural sort order");
+        }
+        if (hintNaturalElt) {
+            if (hintNaturalElt.numberInt() != sortNaturalElt.numberInt()) {
+                return Status(ErrorCodes::BadValue,
+                              "$natural hint must be in the same direction as $natural sort order");
+            }
+        }
+    }
+
+    return unavailableMetadata;
+}
+
 Status CanonicalQuery::isValidNormalized(const MatchExpression* root) {
-    if (auto numGeoNear = QueryPlannerCommon::countNodes(root, MatchExpression::GEO_NEAR);
-        numGeoNear > 0) {
+    if (auto numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); numGeoNear > 0) {
         tassert(5705300, "Only one geo $near expression is expected", numGeoNear == 1);
 
         auto topLevel = false;