Update upstream source from tag 'upstream/6.0.10'debian/6.0.10-1

Update to upstream version '6.0.10'
with Debian dir 2d176fa254eee97b139f712fec5709641335a8c3

4 files changed, 141 insertions, 101 deletions

diff --git a/src/mongo/db/stats/SConscript b/src/mongo/db/stats/SConscript
index 17bbe0cb144..845600bfacb 100644
--- a/src/mongo/db/stats/SConscript
+++ b/src/mongo/db/stats/SConscript
@@ -126,10 +126,9 @@ env.Library(
         '$BUILD_DIR/mongo/db/catalog/index_catalog',
         '$BUILD_DIR/mongo/db/commands/server_status',
         '$BUILD_DIR/mongo/db/db_raii',
-        '$BUILD_DIR/mongo/db/dbdirectclient', # TODO (SERVER-64162) remove
         '$BUILD_DIR/mongo/db/index/index_access_method',
-        '$BUILD_DIR/mongo/db/pipeline/aggregation_request_helper', # TODO (SERVER-64162) remove
         '$BUILD_DIR/mongo/db/pipeline/document_sources_idl',
+        '$BUILD_DIR/mongo/db/s/balancer_stats_registry',
         '$BUILD_DIR/mongo/db/timeseries/bucket_catalog',
         '$BUILD_DIR/mongo/db/timeseries/timeseries_stats',
         'fill_locker_info',
diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp
index 1f3f4c901bb..e344141052d 100644
--- a/src/mongo/db/stats/counters.cpp
+++ b/src/mongo/db/stats/counters.cpp
@@ -335,7 +335,20 @@ NetworkCounter networkCounter;
 AuthCounter authCounter;
 AggStageCounters aggStageCounters;
 DotsAndDollarsFieldsCounters dotsAndDollarsFieldsCounters;
-QueryEngineCounters queryEngineCounters;
-OperatorCountersAggExpressions operatorCountersAggExpressions;
-OperatorCountersMatchExpressions operatorCountersMatchExpressions;
+QueryFrameworkCounters queryFrameworkCounters;
+ValidatorCounters validatorCounters;
+
+OperatorCounters operatorCountersAggExpressions{"operatorCounters.expressions."};
+OperatorCounters operatorCountersMatchExpressions{"operatorCounters.match."};
+OperatorCounters operatorCountersGroupAccumulatorExpressions{"operatorCounters.groupAccumulators."};
+OperatorCounters operatorCountersWindowAccumulatorExpressions{
+    "operatorCounters.windowAccumulators."};
+
+Counter64 updateManyCount;
+ServerStatusMetricField<Counter64> displayUpdateManyCount("query.updateManyCount",
+                                                          &updateManyCount);
+Counter64 deleteManyCount;
+ServerStatusMetricField<Counter64> displayDeleteManyCount("query.deleteManyCount",
+                                                          &deleteManyCount);
+
 }  // namespace mongo
diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h
index 5e22a46aece..e41de1e31b0 100644
--- a/src/mongo/db/stats/counters.h
+++ b/src/mongo/db/stats/counters.h
@@ -337,20 +337,21 @@ public:
 
 extern DotsAndDollarsFieldsCounters dotsAndDollarsFieldsCounters;
 
-class QueryEngineCounters {
+class QueryFrameworkCounters {
 public:
-    QueryEngineCounters()
-        : sbeFindQueryMetric("query.queryExecutionEngine.find.sbe", &sbeFindQueryCounter),
-          classicFindQueryMetric("query.queryExecutionEngine.find.classic",
-                                 &classicFindQueryCounter),
-          sbeOnlyAggregationMetric("query.queryExecutionEngine.aggregate.sbeOnly",
+    QueryFrameworkCounters()
+        : sbeFindQueryMetric("query.queryFramework.find.sbe", &sbeFindQueryCounter),
+          classicFindQueryMetric("query.queryFramework.find.classic", &classicFindQueryCounter),
+          cqfFindQueryMetric("query.queryFramework.find.cqf", &cqfFindQueryCounter),
+          sbeOnlyAggregationMetric("query.queryFramework.aggregate.sbeOnly",
                                    &sbeOnlyAggregationCounter),
-          classicOnlyAggregationMetric("query.queryExecutionEngine.aggregate.classicOnly",
+          classicOnlyAggregationMetric("query.queryFramework.aggregate.classicOnly",
                                        &classicOnlyAggregationCounter),
-          sbeHybridAggregationMetric("query.queryExecutionEngine.aggregate.sbeHybrid",
+          sbeHybridAggregationMetric("query.queryFramework.aggregate.sbeHybrid",
                                      &sbeHybridAggregationCounter),
-          classicHybridAggregationMetric("query.queryExecutionEngine.aggregate.classicHybrid",
-                                         &classicHybridAggregationCounter) {}
+          classicHybridAggregationMetric("query.queryFramework.aggregate.classicHybrid",
+                                         &classicHybridAggregationCounter),
+          cqfAggregationMetric("query.queryFramework.aggregate.cqf", &cqfAggregationQueryCounter) {}
 
     void incrementQueryEngineCounters(CurOp* curop) {
         auto& debug = curop->debug();
@@ -376,91 +377,131 @@ public:
                     sbeOnlyAggregationCounter.increment();
                 }
             }
+        } else if (debug.cqfUsed) {
+            if (cmdName == "find") {
+                cqfFindQueryCounter.increment();
+            } else {
+                cqfAggregationQueryCounter.increment();
+            }
         }
     }
 
-    // Query counters that record whether a find query was fully or partially executed in SBE, or
-    // fully executed using the classic engine. One or the other will always be incremented during a
-    // query.
+    // Query counters that record whether a find query was fully or partially executed in SBE, fully
+    // executed using the classic engine, or fully executed using the common query framework (CQF).
+    // One of these will always be incremented during a query.
     Counter64 sbeFindQueryCounter;
     Counter64 classicFindQueryCounter;
+    Counter64 cqfFindQueryCounter;
     ServerStatusMetricField<Counter64> sbeFindQueryMetric;
     ServerStatusMetricField<Counter64> classicFindQueryMetric;
+    ServerStatusMetricField<Counter64> cqfFindQueryMetric;
     // Aggregation query counters that record whether an aggregation was fully or partially executed
-    // in DocumentSource (an sbe/classic hybrid plan), or fully pushed down to the sbe/classic
-    // layer. Only incremented during aggregations.
+    // in DocumentSource (an sbe/classic hybrid plan), fully pushed down to the sbe/classic layer,
+    // or executed using CQF. These are only incremented during aggregations.
     Counter64 sbeOnlyAggregationCounter;
     Counter64 classicOnlyAggregationCounter;
     Counter64 sbeHybridAggregationCounter;
     Counter64 classicHybridAggregationCounter;
+    Counter64 cqfAggregationQueryCounter;
     ServerStatusMetricField<Counter64> sbeOnlyAggregationMetric;
     ServerStatusMetricField<Counter64> classicOnlyAggregationMetric;
     ServerStatusMetricField<Counter64> sbeHybridAggregationMetric;
     ServerStatusMetricField<Counter64> classicHybridAggregationMetric;
+    ServerStatusMetricField<Counter64> cqfAggregationMetric;
 };
-extern QueryEngineCounters queryEngineCounters;
+extern QueryFrameworkCounters queryFrameworkCounters;
 
-class OperatorCountersAggExpressions {
+/**
+ * Generic class for counters of expressions inside various MQL statements.
+ */
+class OperatorCounters {
 private:
-    struct AggExprCounter {
-        AggExprCounter(StringData name)
-            : metric("operatorCounters.expressions." + name, &counter) {}
-
+    struct ExprCounter {
+        ExprCounter(const std::string name) : metric(name, &counter) {}
         Counter64 counter;
         ServerStatusMetricField<Counter64> metric;
     };
 
 public:
-    void addAggExpressionCounter(StringData name) {
-        operatorCountersAggExpressionMap[name] = std::make_unique<AggExprCounter>(name);
+    OperatorCounters(const std::string prefix) : _prefix{prefix} {}
+
+    void addCounter(const std::string name) {
+        const StringData sdName(name);
+        operatorCountersExprMap[sdName] = std::make_unique<ExprCounter>(_prefix + name);
     }
 
     void mergeCounters(StringMap<uint64_t>& toMerge) {
         for (auto&& [name, cnt] : toMerge) {
-            if (auto it = operatorCountersAggExpressionMap.find(name);
-                it != operatorCountersAggExpressionMap.end()) {
+            if (auto it = operatorCountersExprMap.find(name); it != operatorCountersExprMap.end()) {
                 it->second->counter.increment(cnt);
             }
         }
     }
 
 private:
-    // Map of aggregation expressions to the number of occurrences in aggregation pipelines.
-    StringMap<std::unique_ptr<AggExprCounter>> operatorCountersAggExpressionMap = {};
+    const std::string _prefix;
+    // Map of expressions to the number of occurrences in queries.
+    StringMap<std::unique_ptr<ExprCounter>> operatorCountersExprMap = {};
 };
 
-extern OperatorCountersAggExpressions operatorCountersAggExpressions;
-
-/**
- * Global counters for match expressions.
- */
-class OperatorCountersMatchExpressions {
-private:
-    struct MatchExprCounter {
-        MatchExprCounter(StringData name) : metric("operatorCounters.match." + name, &counter) {}
-
-        Counter64 counter;
-        ServerStatusMetricField<Counter64> metric;
-    };
-
+class ValidatorCounters {
 public:
-    void addMatchExprCounter(StringData name) {
-        operatorCountersMatchExprMap[name] = std::make_unique<MatchExprCounter>(name);
-    }
-
-    void mergeCounters(StringMap<uint64_t>& toMerge) {
-        for (auto&& [name, cnt] : toMerge) {
-            if (auto it = operatorCountersMatchExprMap.find(name);
-                it != operatorCountersMatchExprMap.end()) {
-                it->second->counter.increment(cnt);
+    ValidatorCounters() {
+        _validatorCounterMap["create"] = std::make_unique<ValidatorCounter>("create");
+        _validatorCounterMap["collMod"] = std::make_unique<ValidatorCounter>("collMod");
+    }
+
+    void incrementCounters(const StringData cmdName,
+                           const BSONObj& validator,
+                           bool parsingSucceeded) {
+        if (!validator.isEmpty()) {
+            auto validatorCounter = _validatorCounterMap.find(cmdName);
+            tassert(7139200,
+                    str::stream() << "The validator counters are not support for the command: "
+                                  << cmdName,
+                    validatorCounter != _validatorCounterMap.end());
+            validatorCounter->second->total.increment();
+
+            if (!parsingSucceeded) {
+                validatorCounter->second->failed.increment();
+            }
+            if (validator.hasField("$jsonSchema")) {
+                validatorCounter->second->jsonSchema.increment();
             }
         }
     }
 
 private:
-    // Map of match expressions to the number of occurrences in queries.
-    StringMap<std::unique_ptr<MatchExprCounter>> operatorCountersMatchExprMap = {};
+    struct ValidatorCounter {
+        ValidatorCounter(const StringData name)
+            : totalMetric("commands." + name + ".validator.total", &total),
+              failedMetric("commands." + name + ".validator.failed", &failed),
+              jsonSchemaMetric("commands." + name + ".validator.jsonSchema", &jsonSchema) {}
+        Counter64 total;
+        Counter64 failed;
+        Counter64 jsonSchema;
+        ServerStatusMetricField<Counter64> totalMetric;
+        ServerStatusMetricField<Counter64> failedMetric;
+        ServerStatusMetricField<Counter64> jsonSchemaMetric;
+    };
+
+    StringMap<std::unique_ptr<ValidatorCounter>> _validatorCounterMap = {};
 };
 
-extern OperatorCountersMatchExpressions operatorCountersMatchExpressions;
+extern ValidatorCounters validatorCounters;
+
+// Global counters for expressions inside aggregation pipelines.
+extern OperatorCounters operatorCountersAggExpressions;
+// Global counters for match expressions.
+extern OperatorCounters operatorCountersMatchExpressions;
+// Global counters for accumulator expressions apply to $group.
+extern OperatorCounters operatorCountersGroupAccumulatorExpressions;
+// Global counters for accumulator expressions apply to $setWindowFields.
+extern OperatorCounters operatorCountersWindowAccumulatorExpressions;
+
+// Track the number of {multi:true} updates.
+extern Counter64 updateManyCount;
+// Track the number of deleteMany calls.
+extern Counter64 deleteManyCount;
+
 }  // namespace mongo
diff --git a/src/mongo/db/stats/storage_stats.cpp b/src/mongo/db/stats/storage_stats.cpp
index 870dc908021..203c762856d 100644
--- a/src/mongo/db/stats/storage_stats.cpp
+++ b/src/mongo/db/stats/storage_stats.cpp
@@ -36,10 +36,9 @@
 #include "mongo/db/catalog/database_holder.h"
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/db_raii.h"
-#include "mongo/db/dbdirectclient.h"  // TODO (SERVER-64162) remove
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/pipeline/aggregate_command_gen.h"  // TODO (SERVER-64162) remove
+#include "mongo/db/s/balancer_stats_registry.h"
 #include "mongo/db/timeseries/bucket_catalog.h"
 #include "mongo/db/timeseries/timeseries_stats.h"
 #include "mongo/logv2/log.h"
@@ -49,36 +48,6 @@
 
 namespace mongo {
 
-namespace {
-long long countOrphanDocsForCollection(OperationContext* opCtx, const UUID& uuid) {
-    // TODO (SERVER-64162): move this function to range_deletion_util.cpp and replace
-    // "collectionUuid" and "numOrphanDocs" with RangeDeletionTask field names.
-    DBDirectClient client(opCtx);
-    std::vector<BSONObj> pipeline;
-    pipeline.push_back(BSON("$match" << BSON("collectionUuid" << uuid)));
-    pipeline.push_back(BSON("$group" << BSON("_id"
-                                             << "numOrphans"
-                                             << "count"
-                                             << BSON("$sum"
-                                                     << "$numOrphanDocs"))));
-    AggregateCommandRequest aggRequest(NamespaceString::kRangeDeletionNamespace, pipeline);
-    auto swCursor = DBClientCursor::fromAggregationRequest(
-        &client, aggRequest, false /* secondaryOk */, true /* useExhaust */);
-    if (!swCursor.isOK()) {
-        return 0;
-    }
-    auto cursor = std::move(swCursor.getValue());
-    if (!cursor->more()) {
-        return 0;
-    }
-    auto res = cursor->nextSafe();
-    invariant(!cursor->more());
-    auto numOrphans = res.getField("count");
-    invariant(numOrphans);
-    return numOrphans.exactNumberLong();
-}
-}  // namespace
-
 Status appendCollectionStorageStats(OperationContext* opCtx,
                                     const NamespaceString& nss,
                                     const StorageStatsSpec& storageStatsSpec,
@@ -90,11 +59,14 @@ Status appendCollectionStorageStats(OperationContext* opCtx,
     bool waitForLock = storageStatsSpec.getWaitForLock();
     bool numericOnly = storageStatsSpec.getNumericOnly();
 
-    const auto bucketNss = nss.makeTimeseriesBucketsNamespace();
-    const auto isTimeseries = nss.isTimeseriesBucketsCollection() ||
-        CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(opCtx, bucketNss);
+    const auto bucketNss =
+        nss.isTimeseriesBucketsCollection() ? nss : nss.makeTimeseriesBucketsNamespace();
+    // Hold reference to the catalog for collection lookup without locks to be safe.
+    auto catalog = CollectionCatalog::get(opCtx);
+    auto bucketsColl = catalog->lookupCollectionByNamespace(opCtx, bucketNss);
+    const bool mayBeTimeseries = bucketsColl && bucketsColl->getTimeseriesOptions();
     const auto collNss =
-        (isTimeseries && !nss.isTimeseriesBucketsCollection()) ? std::move(bucketNss) : nss;
+        (mayBeTimeseries && !nss.isTimeseriesBucketsCollection()) ? std::move(bucketNss) : nss;
 
     boost::optional<AutoGetCollectionForReadCommandMaybeLockFree> autoColl;
     try {
@@ -108,7 +80,15 @@ Status appendCollectionStorageStats(OperationContext* opCtx,
     }
 
     const auto& collection = autoColl->getCollection();  // Will be set if present
-    if (!collection) {
+    const bool isTimeseries = collection && collection->getTimeseriesOptions().has_value();
+
+    // We decided the requested namespace was a time series view, so we redirected to the underlying
+    // buckets collection. However, when we tried to acquire that collection, it did not exist or it
+    // did not have time series options, which means it was dropped and potentially recreated in
+    // between the two calls. Logically, the collection that we were looking for does not exist.
+    bool logicallyNotFound = collNss != nss && !isTimeseries;
+
+    if (!collection || logicallyNotFound) {
         result->appendNumber("size", 0);
         result->appendNumber("count", 0);
         result->appendNumber(kOrphanCountField, 0);
@@ -143,10 +123,17 @@ Status appendCollectionStorageStats(OperationContext* opCtx,
         }
     }
 
-    if (serverGlobalParams.featureCompatibility.isVersionInitialized() &&
-        feature_flags::gOrphanTracking.isEnabled(serverGlobalParams.featureCompatibility)) {
-        result->appendNumber(kOrphanCountField,
-                             countOrphanDocsForCollection(opCtx, collection->uuid()));
+    if (serverGlobalParams.clusterRole == ClusterRole::ShardServer &&
+        !nss.isNamespaceAlwaysUnsharded()) {
+        if (serverGlobalParams.featureCompatibility.isVersionInitialized() &&
+            feature_flags::gOrphanTracking.isEnabled(serverGlobalParams.featureCompatibility)) {
+            result->appendNumber(
+                kOrphanCountField,
+                BalancerStatsRegistry::get(opCtx)->getCollNumOrphanDocsFromDiskIfNeeded(
+                    opCtx, collection->uuid()));
+        }
+    } else {
+        result->appendNumber(kOrphanCountField, 0);
     }
 
     const RecordStore* recordStore = collection->getRecordStore();
@@ -175,8 +162,8 @@ Status appendCollectionStorageStats(OperationContext* opCtx,
     BSONObjBuilder indexDetails;
     std::vector<std::string> indexBuilds;
 
-    std::unique_ptr<IndexCatalog::IndexIterator> it =
-        indexCatalog->getIndexIterator(opCtx, /*includeUnfinishedIndexes=*/true);
+    auto it = indexCatalog->getIndexIterator(
+        opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
     while (it->more()) {
         const IndexCatalogEntry* entry = it->next();
         const IndexDescriptor* descriptor = entry->descriptor();