diff options
| author | Erin Zhu <erin.zhu@mongodb.com> | 2024-02-16 17:39:46 +0000 |
|---|---|---|
| committer | MongoDB Bot <mongo-bot@mongodb.com> | 2024-02-16 18:58:24 +0000 |
| commit | f8a2c84946db93762b46bd9946d8c8d87db5d93f (patch) | |
| tree | 9ca02fde324bb16e3f4d33df6beac897c9f56b61 | |
| parent | 71b8d3570f69d3c9bc05793b2421ff98de04e990 (diff) | |
SERVER-86652: Query Stats Backport to 7.0 Batch #6r7.0.7-rc0
Includes changes from
SERVER-79071: Add new privilege action type for non-tokenized $queryStats invocation
SERVER-79568: Allow Non-Tokenized $queryStats Invocation without Test Commands Enabled
SERVER-79453: Add featureFlagQueryStatsFindCommand for Milestone 0
SERVER-78985 specify PartitionedCache edge cases
SERVER-78943 Test LRUCache eviction behavior for large queryStats entries
SERVER-79828: Only compute query shape for mongos find command when necessary
SERVER-79984: Remove query stats passthroughs from amazon-linux2-arm64-fixed-concurrent-transactions variant
SERVER-77001 Improve implementation of RateLimiting
SERVER-80151 Move query stats files to a new directory
SERVER-78083: Add ability to log $queryStats output on each invocation
SERVER-80201 split query_stats into multiple files
SERVER-76328: Scrutinize expression parser for representative query round-tripping
SERVER-78049: Track query stats on changeStream and virtual collections
SERVER-80360 Fix query stats in list serialization perf regression
SERVER-80304: Make $queryStats parameter hmacKey required for hmac-sha-256
SERVER-80411 Move 'hint' out of query shape and into query stats store
SERVER-80934 Add google benchmark for shapification
SERVER-80821 Use original request namespace in query shape
SERVER-79794 silently catch DBException when $queryStats shapification process bloats the query's shape past the max BSON size limit
SERVER-80941 Speed up shapifying path expressions
SERVER-80878 Add QueryStatsStore insertion/eviction listener to track size
SERVER-78025 Remove skip tag and TODO from working test
and Enterprise
SERVER-80151 Move query stats files to a new directory
GitOrigin-RevId: af54ee776360f35dd25de76279e21785c7cdb681
113 files changed, 2796 insertions, 1136 deletions
diff --git a/buildscripts/resmokeconfig/suites/benchmarks.yml b/buildscripts/resmokeconfig/suites/benchmarks.yml index 49ed4e9dcb1..fe1349e1a83 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks.yml @@ -21,6 +21,7 @@ selector: # These benchmarks are being run as part of the benchmarks_query.yml - build/install/bin/percentile_algo_bm* - build/install/bin/window_function_percentile_bm* + - build/install/bin/rate_limiting_bm* # These benchmarks are being run as part of the benchmarks_expression*.yml - build/install/bin/expression_bm* - build/install/bin/sbe_expression_bm* diff --git a/buildscripts/resmokeconfig/suites/benchmarks_query.yml b/buildscripts/resmokeconfig/suites/benchmarks_query.yml index d8e1fd3f93f..cf324ce8f69 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_query.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_query.yml @@ -7,6 +7,8 @@ selector: # The trailing asterisk is for handling the .exe extension on Windows. - build/install/bin/percentile_algo_bm* - build/install/bin/window_function_percentile_bm* + - build/install/bin/rate_limiting_bm* + - build/install/bin/shapifying_bm* executor: config: {} diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 757423a91c9..275ee570ff8 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -2941,9 +2941,6 @@ buildvariants: - name: vector_search - name: vector_search_auth - name: vector_search_ssl - - name: query_stats_passthrough - - name: query_stats_passthrough_writeonly - - name: query_stats_mongos_passthrough - <<: *enterprise-rhel-80-64-bit-dynamic-classic-engine name: &enterprise-rhel-80-64-bit-dynamic-classic-engine-query-patch-only enterprise-rhel-80-64-bit-dynamic-classic-engine-query-patch-only diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js index bb35e07db8d..dd349450ea4 100644 --- a/jstests/auth/lib/commands_lib.js +++ b/jstests/auth/lib/commands_lib.js @@ -6515,14 +6515,24 @@ export const authCommandsLib = { ] }, { - // Test that only clusterManager has permission to run $queryStats - testname: "testTelemetryReadPrivilege", - command: {aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}}, - skipSharded: false, - skipTest: (conn) => { - return !TestData.setParameters.featureFlagQueryStats; - }, - testcases: [{runOnDb: adminDbName, roles: roles_clusterManager}] + // Test that only clusterManager has permission to run $queryStats without transformation + testname: "testQueryStatsReadPrivilege", + command: {aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}}, + skipSharded: false, + skipTest: (conn) => { + return !TestData.setParameters.featureFlagQueryStats && !TestData.setParameters.featureFlagQueryStatsFindCommand; + }, + testcases: [{runOnDb: adminDbName, roles: roles_clusterManager}] + }, + { + // Test that only clusterManager has permission to run $queryStats with transformation + testname: "testQueryStatsReadTransformedPrivilege", + command: {aggregate: 1, pipeline: [{$queryStats: {transformIdentifiers: {algorithm: "hmac-sha-256", hmacKey: BinData(8, "MjM0NTY3ODkxMDExMTIxMzE0MTUxNjE3MTgxOTIwMjE=")}}}], cursor: {}}, + skipSharded: false, + skipTest: (conn) => { + return !TestData.setParameters.featureFlagQueryStats && !TestData.setParameters.featureFlagQueryStatsFindCommand; + }, + testcases: [{runOnDb: adminDbName, roles: roles_clusterManager}] }, { testname: "top", diff --git a/jstests/core/views/invalid_system_views.js b/jstests/core/views/invalid_system_views.js index 109ac9e1a94..859138ef67e 100644 --- a/jstests/core/views/invalid_system_views.js +++ b/jstests/core/views/invalid_system_views.js @@ -17,8 +17,6 @@ * tenant_migration_incompatible, * uses_compact, * references_foreign_collection, - * # TODO SERVER-78025 reenable query stats coverage on this test - * skip_for_query_stats * ] */ diff --git a/jstests/libs/query_stats_utils.js b/jstests/libs/query_stats_utils.js index 62154ddb916..9cc0c306245 100644 --- a/jstests/libs/query_stats_utils.js +++ b/jstests/libs/query_stats_utils.js @@ -199,6 +199,61 @@ function confirmAllExpectedFieldsPresent(expectedKey, resultingKey) { assert.eq(fieldsCounter, Object.keys(expectedKey).length, resultingKey); } +function assertExpectedResults(results, + expectedQueryStatsKey, + expectedExecCount, + expectedDocsReturnedSum, + expectedDocsReturnedMax, + expectedDocsReturnedMin, + expectedDocsReturnedSumOfSq, + getMores) { + const {key, metrics} = results; + confirmAllExpectedFieldsPresent(expectedQueryStatsKey, key); + assert.eq(expectedExecCount, metrics.execCount); + assert.docEq({ + sum: NumberLong(expectedDocsReturnedSum), + max: NumberLong(expectedDocsReturnedMax), + min: NumberLong(expectedDocsReturnedMin), + sumOfSquares: NumberLong(expectedDocsReturnedSumOfSq) + }, + metrics.docsReturned); + + const { + firstSeenTimestamp, + latestSeenTimestamp, + lastExecutionMicros, + totalExecMicros, + firstResponseExecMicros + } = metrics; + + // The tests can't predict exact timings, so just assert these three fields have been set (are + // non-zero). + assert.neq(lastExecutionMicros, NumberLong(0)); + assert.neq(firstSeenTimestamp.getTime(), 0); + assert.neq(latestSeenTimestamp.getTime(), 0); + + const distributionFields = ['sum', 'max', 'min', 'sumOfSquares']; + for (const field of distributionFields) { + assert.neq(totalExecMicros[field], NumberLong(0)); + assert.neq(firstResponseExecMicros[field], NumberLong(0)); + if (getMores) { + // If there are getMore calls, totalExecMicros fields should be greater than or equal to + // firstResponseExecMicros. + if (field == 'min' || field == 'max') { + // In the case that we've executed multiple queries with the same shape, it is + // possible for the min or max to be equal. + assert.gte(totalExecMicros[field], firstResponseExecMicros[field]); + } else { + assert.gt(totalExecMicros[field], firstResponseExecMicros[field]); + } + } else { + // If there are no getMore calls, totalExecMicros fields should be equal to + // firstResponseExecMicros. + assert.eq(totalExecMicros[field], firstResponseExecMicros[field]); + } + } +} + function asFieldPath(str) { return "$" + str; } @@ -206,3 +261,11 @@ function asFieldPath(str) { function asVarRef(str) { return "$$" + str; } + +function resetQueryStatsStore(conn, queryStatsStoreSize) { + // Set the cache size to 0MB to clear the queryStats store, and then reset to + // queryStatsStoreSize. + assert.commandWorked(conn.adminCommand({setParameter: 1, internalQueryStatsCacheSize: "0MB"})); + assert.commandWorked( + conn.adminCommand({setParameter: 1, internalQueryStatsCacheSize: queryStatsStoreSize})); +} diff --git a/jstests/noPassthrough/queryStats/application_name_find.js b/jstests/noPassthrough/queryStats/application_name_find.js index 29e887391a4..c45b6b97e9c 100644 --- a/jstests/noPassthrough/queryStats/application_name_find.js +++ b/jstests/noPassthrough/queryStats/application_name_find.js @@ -1,14 +1,12 @@ /** * Test that applicationName and namespace appear in queryStats for the find command. - * @tags: [featureFlagQueryStats] + * @tags: [featureFlagQueryStatsFindCommand] */ load("jstests/libs/query_stats_utils.js"); (function() { "use strict"; const kApplicationName = "MongoDB Shell"; -const kHashedCollName = "w6Ax20mVkbJu4wQWAMjL8Sl+DfXAr2Zqdc3kJRB7Oo0="; -const kHashedFieldName = "lU7Z0mLRPRUL+RfAD5jhYPRRpXBsZBxS/20EzDwfOG4="; // Turn on the collecting of queryStats metrics. let options = { diff --git a/jstests/noPassthrough/queryStats/clear_query_stats_store.js b/jstests/noPassthrough/queryStats/clear_query_stats_store.js index 4cdf67ebd99..a21d11c9ace 100644 --- a/jstests/noPassthrough/queryStats/clear_query_stats_store.js +++ b/jstests/noPassthrough/queryStats/clear_query_stats_store.js @@ -1,13 +1,13 @@ /** - * Test that the telemetry store can be cleared when the cache size is reset to 0. + * Test that the query stats store can be cleared when the cache size is reset to 0. * @tags: [featureFlagQueryStats] */ -load("jstests/libs/query_stats_utils.js"); // For verifyMetrics. +load("jstests/libs/query_stats_utils.js"); // For verifyMetrics and getQueryStats. (function() { "use strict"; -// Turn on the collecting of telemetry metrics. +// Turn on the collecting of query stats metrics. let options = { setParameter: {internalQueryStatsRateLimit: -1, internalQueryStatsCacheSize: "10MB"}, }; @@ -26,9 +26,10 @@ for (var j = 0; j < 10; ++j) { } // Confirm number of entries in the store and that none have been evicted. -let telemetryResults = testDB.getSiblingDB("admin").aggregate([{$queryStats: {}}]).toArray(); -assert.eq(telemetryResults.length, 10, telemetryResults); +let res = getQueryStats(conn); +assert.eq(res.length, 10, res); assert.eq(testDB.serverStatus().metrics.queryStats.numEvicted, 0); +assert.gt(testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes, 0); // Command to clear the cache. assert.commandWorked(testDB.adminCommand({setParameter: 1, internalQueryStatsCacheSize: "0MB"})); @@ -36,8 +37,9 @@ assert.commandWorked(testDB.adminCommand({setParameter: 1, internalQueryStatsCac // 10 regular queries plus the $queryStats query, means 11 entries evicted when the cache is // cleared. assert.eq(testDB.serverStatus().metrics.queryStats.numEvicted, 11); +assert.eq(testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes, 0); -// Calling $queryStats should fail when the telemetry store size is 0 bytes. +// Calling $queryStats should fail when the query stats store size is 0 bytes. assert.throwsWithCode(() => testDB.getSiblingDB("admin").aggregate([{$queryStats: {}}]), 6579000); MongoRunner.stopMongod(conn); }()); diff --git a/jstests/noPassthrough/queryStats/documentSourceQueryStats_redaction_parameters.js b/jstests/noPassthrough/queryStats/documentSourceQueryStats_redaction_parameters.js index becc4566ae7..05ffaa49433 100644 --- a/jstests/noPassthrough/queryStats/documentSourceQueryStats_redaction_parameters.js +++ b/jstests/noPassthrough/queryStats/documentSourceQueryStats_redaction_parameters.js @@ -1,10 +1,10 @@ /** * Test the $queryStats hmac properties. - * @tags: [featureFlagQueryStats] + * @tags: [featureFlagQueryStatsFindCommand] */ load("jstests/aggregation/extras/utils.js"); // For assertAdminDBErrCodeAndErrMsgContains. -load("jstests/libs/query_stats_utils.js"); +load("jstests/libs/query_stats_utils.js"); // For getQueryStatsFindCmd (function() { "use strict"; @@ -84,6 +84,14 @@ function runTest(conn) { 40414, "BSON field '$queryStats.transformIdentifiers.algorithm' is missing but a required field"); + // TransformIdentifiers with algorithm but missing hmacKey throws error. + pipeline = [{$queryStats: {transformIdentifiers: {algorithm: "hmac-sha-256"}}}]; + assertAdminDBErrCodeAndErrMsgContains( + coll, + pipeline, + ErrorCodes.FailedToParse, + "The 'hmacKey' parameter of the $queryStats stage must be specified when applying the hmac-sha-256 algorithm"); + // Parameter object with unrecognized key throws error. pipeline = [{$queryStats: {transformIdentifiers: {algorithm: "hmac-sha-256", hmacStrategy: "on"}}}]; diff --git a/jstests/noPassthrough/queryStats/feature_flag_off_sampling_rate_on.js b/jstests/noPassthrough/queryStats/feature_flag_off_sampling_rate_on.js index 26f4dd5e6a1..68e1b4447e2 100644 --- a/jstests/noPassthrough/queryStats/feature_flag_off_sampling_rate_on.js +++ b/jstests/noPassthrough/queryStats/feature_flag_off_sampling_rate_on.js @@ -1,5 +1,5 @@ /** - * Test that calls to read from telemetry store fail when feature flag is turned off and sampling + * Test that calls to read from query stats store fail when feature flag is turned off and sampling * rate > 0. */ load('jstests/libs/analyze_plan.js'); @@ -17,8 +17,10 @@ const testdb = conn.getDB('test'); // This test specifically tests error handling when the feature flag is not on. // TODO SERVER-65800 This test can be deleted when the feature is on by default. -if (!conn || FeatureFlagUtil.isEnabled(testdb, "QueryStats")) { - jsTestLog(`Skipping test since feature flag is disabled. conn: ${conn}`); +// TODO SERVER-79494 remove reference to featureFlagQueryStatsFindCommand. +if (!conn || FeatureFlagUtil.isEnabled(testdb, "QueryStats") || + FeatureFlagUtil.isEnabled(testdb, "QueryStatsFindCommand")) { + jsTestLog(`Skipping test since feature flag is enabled. conn: ${conn}`); if (conn) { MongoRunner.stopMongod(conn); } diff --git a/jstests/noPassthrough/queryStats/find_cmd_one_way_tokenization.js b/jstests/noPassthrough/queryStats/find_cmd_one_way_tokenization.js index c933a889203..62d07cba355 100644 --- a/jstests/noPassthrough/queryStats/find_cmd_one_way_tokenization.js +++ b/jstests/noPassthrough/queryStats/find_cmd_one_way_tokenization.js @@ -5,7 +5,6 @@ load("jstests/libs/query_stats_utils.js"); (function() { "use strict"; -const kHashedCollName = "w6Ax20mVkbJu4wQWAMjL8Sl+DfXAr2Zqdc3kJRB7Oo0="; const kHashedFieldName = "lU7Z0mLRPRUL+RfAD5jhYPRRpXBsZBxS/20EzDwfOG4="; function runTest(conn) { @@ -42,7 +41,7 @@ function runTest(conn) { queryStats[1].key.queryShape.filter); } -const conn = MongoRunner.runMongod({ +let conn = MongoRunner.runMongod({ setParameter: { internalQueryStatsRateLimit: -1, featureFlagQueryStats: true, @@ -51,7 +50,17 @@ const conn = MongoRunner.runMongod({ runTest(conn); MongoRunner.stopMongod(conn); -const st = new ShardingTest({ +// TODO SERVER-79494 remove second run with find-command-only feature flag +conn = MongoRunner.runMongod({ + setParameter: { + internalQueryStatsRateLimit: -1, + featureFlagQueryStatsFindCommand: true, + } +}); +runTest(conn); +MongoRunner.stopMongod(conn); + +let st = new ShardingTest({ mongos: 1, shards: 1, config: 1, @@ -65,4 +74,20 @@ const st = new ShardingTest({ }); runTest(st.s); st.stop(); + +// TODO SERVER-79494 remove second run with find-command-only feature flag +st = new ShardingTest({ + mongos: 1, + shards: 1, + config: 1, + rs: {nodes: 1}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + featureFlagQueryStatsFindCommand: true, + } + }, +}); +runTest(st.s); +st.stop(); }()); diff --git a/jstests/noPassthrough/queryStats/geometry_without_coordinates.js b/jstests/noPassthrough/queryStats/geometry_without_coordinates.js index 98ba6ec40aa..a11d6b1f38f 100644 --- a/jstests/noPassthrough/queryStats/geometry_without_coordinates.js +++ b/jstests/noPassthrough/queryStats/geometry_without_coordinates.js @@ -1,6 +1,6 @@ // This test was designed to reproduce SERVER-77430. There was a mistaken assertion in a parser that // we are interested in proving will not fail here. -// @tags: [featureFlagQueryStats] +// @tags: [featureFlagQueryStatsFindCommand] (function() { "use strict"; diff --git a/jstests/noPassthrough/queryStats/query_stats_agg_cmd_collect_on_mongos.js b/jstests/noPassthrough/queryStats/query_stats_agg_cmd_collect_on_mongos.js new file mode 100644 index 00000000000..21c24a85c56 --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_agg_cmd_collect_on_mongos.js @@ -0,0 +1,152 @@ +/** + * Test that mongos is collecting query stats metrics for agg queries. + * @tags: [featureFlagQueryStats] + */ + +load('jstests/libs/query_stats_utils.js'); + +(function() { +"use strict"; + +const setup = () => { + const st = new ShardingTest({ + mongos: 1, + shards: 1, + config: 1, + rs: {nodes: 1}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + } + }, + }); + const mongos = st.s; + const db = mongos.getDB("test"); + const coll = db.coll; + coll.insert({v: 1}); + coll.insert({v: 4}); + return st; +}; + +// Assert that, for agg queries, no query stats results are written until a cursor has reached +// exhaustion; ensure accurate results once they're written. +{ + const st = setup(); + const db = st.s.getDB("test"); + const coll = db.coll; + + const queryStatsKey = { + queryShape: { + cmdNs: {db: "test", coll: "coll"}, + command: "aggregate", + pipeline: [ + {$match: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}}, + {$project: {_id: true, hello: true}} + ] + + }, + cursor: {batchSize: "?number"}, + applicationName: "MongoDB Shell", + }; + + const cursor = coll.aggregate( + [ + {$match: {v: {$gt: 0, $lt: 5}}}, + {$project: {hello: true}}, + ], + {cursor: {batchSize: 1}}); // returns 1 doc + + // Since the cursor hasn't been exhausted yet, ensure no query stats results have been written + // yet. + let queryStats = getQueryStats(db); + assert.eq(0, queryStats.length, queryStats); + + // Run a getMore to exhaust the cursor, then ensure query stats results have been written + // accurately. batchSize must be 2 so the cursor recognizes exhaustion. + assert.commandWorked(db.runCommand({ + getMore: cursor.getId(), + collection: coll.getName(), + batchSize: 2 + })); // returns 1 doc, exhausts the cursor + queryStats = getQueryStatsAggCmd(db); + assert.eq(1, queryStats.length, queryStats); + assertExpectedResults(queryStats[0], + queryStatsKey, + /* expectedExecCount */ 1, + /* expectedDocsReturnedSum */ 2, + /* expectedDocsReturnedMax */ 2, + /* expectedDocsReturnedMin */ 2, + /* expectedDocsReturnedSumOfSq */ 4, + /* getMores */ true); + + // Run more queries (to exhaustion) with the same query shape, and ensure query stats results + // are accurate. + coll.aggregate([ + {$match: {v: {$gt: 0, $lt: 5}}}, + {$project: {hello: true}}, + ]); // returns 2 docs + coll.aggregate([ + {$match: {v: {$gt: 2, $lt: 3}}}, + {$project: {hello: true}}, + ]); // returns 0 docs + coll.aggregate([ + {$match: {v: {$gt: 0, $lt: 2}}}, + {$project: {hello: true}}, + ]); // returns 1 doc + queryStats = getQueryStatsAggCmd(db); + assert.eq(1, queryStats.length, queryStats); + assertExpectedResults(queryStats[0], + queryStatsKey, + /* expectedExecCount */ 4, + /* expectedDocsReturnedSum */ 5, + /* expectedDocsReturnedMax */ 2, + /* expectedDocsReturnedMin */ 0, + /* expectedDocsReturnedSumOfSq */ 9, + /* getMores */ true); + + st.stop(); +} + +// Assert on batchSize-limited agg queries that killCursors will write metrics with partial results +// to the query stats store. +{ + const st = setup(); + const db = st.s.getDB("test"); + const coll = db.coll; + + const queryStatsKey = { + queryShape: { + cmdNs: {db: "test", coll: "coll"}, + command: "aggregate", + pipeline: [{$match: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}}] + }, + cursor: {batchSize: "?number"}, + applicationName: "MongoDB Shell", + }; + + const cursor1 = coll.aggregate( + [ + {$match: {v: {$gt: 0, $lt: 5}}}, + ], + {cursor: {batchSize: 1}}); // returns 1 doc + const cursor2 = coll.aggregate( + [ + {$match: {v: {$gt: 0, $lt: 2}}}, + ], + {cursor: {batchSize: 1}}); // returns 1 doc + + assert.commandWorked( + db.runCommand({killCursors: coll.getName(), cursors: [cursor1.getId(), cursor2.getId()]})); + const queryStats = getQueryStats(db); + assert.eq(1, queryStats.length); + assertExpectedResults(queryStats[0], + queryStatsKey, + /* expectedExecCount */ 2, + /* expectedDocsReturnedSum */ 2, + /* expectedDocsReturnedMax */ 1, + /* expectedDocsReturnedMin */ 1, + /* expectedDocsReturnedSumOfSq */ 2, + /* getMores */ false); + st.stop(); +} +}()); diff --git a/jstests/noPassthrough/queryStats/query_stats_changeStreams.js b/jstests/noPassthrough/queryStats/query_stats_changeStreams.js new file mode 100644 index 00000000000..a279396629d --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_changeStreams.js @@ -0,0 +1,61 @@ +// Tests the collection of query stats for a change stream query. +// @tags: [ +// uses_change_streams, +// requires_replication, +// requires_sharding, +// featureFlagQueryStats +// ] +load("jstests/libs/change_stream_util.js"); // For ChangeStreamTest. +load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers +load("jstests/libs/query_stats_utils.js"); // For getQueryStatsAggCmd. +(function() { +"use strict"; +function runTest(conn) { + const db = conn.getDB("test"); + const coll = db.coll; + coll.drop(); + + // Create a changeStream collection + let cst = new ChangeStreamTest(db); + cst.startWatchingChanges({ + pipeline: [{$changeStream: {}}], + collection: coll, + }); + cst.cleanUp(); + + const queryStats = getQueryStatsAggCmd(db); + assert.eq(1, queryStats.length); + assert.eq(coll.getName(), queryStats[0].key.queryShape.cmdNs.coll); + + // TODO SERVER-76263 Support reporting 'collectionType' on a sharded cluster. + if (!FixtureHelpers.isMongos(db)) { + assert.eq("changeStream", queryStats[0].key.collectionType); + } +} + +{ + // Test the non-sharded case. + const rst = new ReplSetTest({nodes: 2}); + rst.startSet({setParameter: {internalQueryStatsRateLimit: -1}}); + rst.initiate(); + runTest(rst.getPrimary()); + rst.stopSet(); +} + +{ + // Test on a sharded cluster. + const st = new ShardingTest({ + mongos: 1, + shards: 1, + config: 1, + rs: {nodes: 1}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + } + }, + }); + runTest(st.s); + st.stop(); +} +}()); diff --git a/jstests/noPassthrough/queryStats/query_stats_collect_on_mongos.js b/jstests/noPassthrough/queryStats/query_stats_collect_on_mongos.js deleted file mode 100644 index 4a44fa2afc1..00000000000 --- a/jstests/noPassthrough/queryStats/query_stats_collect_on_mongos.js +++ /dev/null @@ -1,307 +0,0 @@ -/** - * Test that mongos is collecting query stats metrics. - * @tags: [featureFlagQueryStats] - */ - -load('jstests/libs/query_stats_utils.js'); - -(function() { -"use strict"; - -const setup = () => { - const st = new ShardingTest({ - mongos: 1, - shards: 1, - config: 1, - rs: {nodes: 1}, - mongosOptions: { - setParameter: { - internalQueryStatsRateLimit: -1, - } - }, - }); - const mongos = st.s; - const db = mongos.getDB("test"); - const coll = db.coll; - coll.insert({v: 1}); - coll.insert({v: 4}); - return st; -}; - -const assertExpectedResults = (results, - expectedQueryStatsKey, - expectedExecCount, - expectedDocsReturnedSum, - expectedDocsReturnedMax, - expectedDocsReturnedMin, - expectedDocsReturnedSumOfSq, - getMores) => { - const {key, metrics} = results; - confirmAllExpectedFieldsPresent(expectedQueryStatsKey, key); - assert.eq(expectedExecCount, metrics.execCount); - assert.docEq({ - sum: NumberLong(expectedDocsReturnedSum), - max: NumberLong(expectedDocsReturnedMax), - min: NumberLong(expectedDocsReturnedMin), - sumOfSquares: NumberLong(expectedDocsReturnedSumOfSq) - }, - metrics.docsReturned); - - const { - firstSeenTimestamp, - latestSeenTimestamp, - lastExecutionMicros, - totalExecMicros, - firstResponseExecMicros - } = metrics; - - // This test can't predict exact timings, so just assert these three fields have been set (are - // non-zero). - assert.neq(lastExecutionMicros, NumberLong(0)); - assert.neq(firstSeenTimestamp.getTime(), 0); - assert.neq(latestSeenTimestamp.getTime(), 0); - - const distributionFields = ['sum', 'max', 'min', 'sumOfSquares']; - for (const field of distributionFields) { - assert.neq(totalExecMicros[field], NumberLong(0)); - assert.neq(firstResponseExecMicros[field], NumberLong(0)); - if (getMores) { - // If there are getMore calls, totalExecMicros fields should be greater than or equal to - // firstResponseExecMicros. - if (field == 'min' || field == 'max') { - // In the case that we've executed multiple queries with the same shape, it is - // possible for the min or max to be equal. - assert.gte(totalExecMicros[field], firstResponseExecMicros[field]); - } else { - assert.gt(totalExecMicros[field], firstResponseExecMicros[field]); - } - } else { - // If there are no getMore calls, totalExecMicros fields should be equal to - // firstResponseExecMicros. - assert.eq(totalExecMicros[field], firstResponseExecMicros[field]); - } - } -}; - -// Assert that, for find queries, no query stats results are written until a cursor has reached -// exhaustion; ensure accurate results once they're written. -{ - const st = setup(); - const db = st.s.getDB("test"); - const collName = "coll"; - const coll = db[collName]; - - const queryStatsKey = { - queryShape: { - cmdNs: {db: "test", coll: "coll"}, - command: "find", - filter: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}, - }, - readConcern: {level: "local", provenance: "implicitDefault"}, - batchSize: "?number", - client: {application: {name: "MongoDB Shell"}} - }; - - const cursor = coll.find({v: {$gt: 0, $lt: 5}}).batchSize(1); // returns 1 doc - - // Since the cursor hasn't been exhausted yet, ensure no query stats results have been written - // yet. - let queryStats = getQueryStats(db); - assert.eq(0, queryStats.length, queryStats); - - // Run a getMore to exhaust the cursor, then ensure query stats results have been written - // accurately. batchSize must be 2 so the cursor recognizes exhaustion. - assert.commandWorked(db.runCommand({ - getMore: cursor.getId(), - collection: coll.getName(), - batchSize: 2 - })); // returns 1 doc, exhausts the cursor - queryStats = getQueryStatsFindCmd(db); - assert.eq(1, queryStats.length, queryStats); - assertExpectedResults(queryStats[0], - queryStatsKey, - /* expectedExecCount */ 1, - /* expectedDocsReturnedSum */ 2, - /* expectedDocsReturnedMax */ 2, - /* expectedDocsReturnedMin */ 2, - /* expectedDocsReturnedSumOfSq */ 4, - /* getMores */ true); - - // Run more queries (to exhaustion) with the same query shape, and ensure query stats results - // are accurate. - coll.find({v: {$gt: 2, $lt: 3}}).batchSize(10).toArray(); // returns 0 docs - coll.find({v: {$gt: 0, $lt: 1}}).batchSize(10).toArray(); // returns 0 docs - coll.find({v: {$gt: 0, $lt: 2}}).batchSize(10).toArray(); // return 1 doc - queryStats = getQueryStatsFindCmd(db); - assert.eq(1, queryStats.length, queryStats); - assertExpectedResults(queryStats[0], - queryStatsKey, - /* expectedExecCount */ 4, - /* expectedDocsReturnedSum */ 3, - /* expectedDocsReturnedMax */ 2, - /* expectedDocsReturnedMin */ 0, - /* expectedDocsReturnedSumOfSq */ 5, - /* getMores */ true); - - st.stop(); -} - -// Assert that, for agg queries, no query stats results are written until a cursor has reached -// exhaustion; ensure accurate results once they're written. -{ - const st = setup(); - const db = st.s.getDB("test"); - const coll = db.coll; - - const queryStatsKey = { - queryShape: { - cmdNs: {db: "test", coll: "coll"}, - command: "aggregate", - pipeline: [ - {$match: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}}, - {$project: {_id: true, hello: true}} - ] - - }, - cursor: {batchSize: "?number"}, - applicationName: "MongoDB Shell", - }; - - const cursor = coll.aggregate( - [ - {$match: {v: {$gt: 0, $lt: 5}}}, - {$project: {hello: true}}, - ], - {cursor: {batchSize: 1}}); // returns 1 doc - - // Since the cursor hasn't been exhausted yet, ensure no query stats results have been written - // yet. - let queryStats = getQueryStats(db); - assert.eq(0, queryStats.length, queryStats); - - // Run a getMore to exhaust the cursor, then ensure query stats results have been written - // accurately. batchSize must be 2 so the cursor recognizes exhaustion. - assert.commandWorked(db.runCommand({ - getMore: cursor.getId(), - collection: coll.getName(), - batchSize: 2 - })); // returns 1 doc, exhausts the cursor - queryStats = getQueryStatsAggCmd(db); - assert.eq(1, queryStats.length, queryStats); - assertExpectedResults(queryStats[0], - queryStatsKey, - /* expectedExecCount */ 1, - /* expectedDocsReturnedSum */ 2, - /* expectedDocsReturnedMax */ 2, - /* expectedDocsReturnedMin */ 2, - /* expectedDocsReturnedSumOfSq */ 4, - /* getMores */ true); - - // Run more queries (to exhaustion) with the same query shape, and ensure query stats results - // are accurate. - coll.aggregate([ - {$match: {v: {$gt: 0, $lt: 5}}}, - {$project: {hello: true}}, - ]); // returns 2 docs - coll.aggregate([ - {$match: {v: {$gt: 2, $lt: 3}}}, - {$project: {hello: true}}, - ]); // returns 0 docs - coll.aggregate([ - {$match: {v: {$gt: 0, $lt: 2}}}, - {$project: {hello: true}}, - ]); // returns 1 doc - queryStats = getQueryStatsAggCmd(db); - assert.eq(1, queryStats.length, queryStats); - assertExpectedResults(queryStats[0], - queryStatsKey, - /* expectedExecCount */ 4, - /* expectedDocsReturnedSum */ 5, - /* expectedDocsReturnedMax */ 2, - /* expectedDocsReturnedMin */ 0, - /* expectedDocsReturnedSumOfSq */ 9, - /* getMores */ true); - - st.stop(); -} - -// Assert on batchSize-limited find queries that killCursors will write metrics with partial results -// to the query stats store. -{ - const st = setup(); - const db = st.s.getDB("test"); - const collName = "coll"; - const coll = db[collName]; - - const queryStatsKey = { - queryShape: { - cmdNs: {db: "test", coll: "coll"}, - command: "find", - filter: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}, - }, - readConcern: {level: "local", provenance: "implicitDefault"}, - batchSize: "?number", - client: {application: {name: "MongoDB Shell"}} - }; - - const cursor1 = coll.find({v: {$gt: 0, $lt: 5}}).batchSize(1); // returns 1 doc - const cursor2 = coll.find({v: {$gt: 0, $lt: 2}}).batchSize(1); // returns 1 doc - - assert.commandWorked( - db.runCommand({killCursors: coll.getName(), cursors: [cursor1.getId(), cursor2.getId()]})); - const queryStats = getQueryStats(db); - assert.eq(1, queryStats.length); - assertExpectedResults(queryStats[0], - queryStatsKey, - /* expectedExecCount */ 2, - /* expectedDocsReturnedSum */ 2, - /* expectedDocsReturnedMax */ 1, - /* expectedDocsReturnedMin */ 1, - /* expectedDocsReturnedSumOfSq */ 2, - /* getMores */ false); - st.stop(); -} - -// Assert on batchSize-limited agg queries that killCursors will write metrics with partial results -// to the query stats store. -{ - const st = setup(); - const db = st.s.getDB("test"); - const coll = db.coll; - - const queryStatsKey = { - queryShape: { - cmdNs: {db: "test", coll: "coll"}, - command: "aggregate", - pipeline: [{$match: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}}] - }, - cursor: {batchSize: "?number"}, - applicationName: "MongoDB Shell", - }; - - const cursor1 = coll.aggregate( - [ - {$match: {v: {$gt: 0, $lt: 5}}}, - ], - {cursor: {batchSize: 1}}); // returns 1 doc - const cursor2 = coll.aggregate( - [ - {$match: {v: {$gt: 0, $lt: 2}}}, - ], - {cursor: {batchSize: 1}}); // returns 1 doc - - assert.commandWorked( - db.runCommand({killCursors: coll.getName(), cursors: [cursor1.getId(), cursor2.getId()]})); - const queryStats = getQueryStats(db); - assert.eq(1, queryStats.length); - assertExpectedResults(queryStats[0], - queryStatsKey, - /* expectedExecCount */ 2, - /* expectedDocsReturnedSum */ 2, - /* expectedDocsReturnedMax */ 1, - /* expectedDocsReturnedMin */ 1, - /* expectedDocsReturnedSumOfSq */ 2, - /* getMores */ false); - st.stop(); -} -}()); diff --git a/jstests/noPassthrough/queryStats/query_stats_collectionType.js b/jstests/noPassthrough/queryStats/query_stats_collectionType.js index f86e6d11022..39e2ee944b3 100644 --- a/jstests/noPassthrough/queryStats/query_stats_collectionType.js +++ b/jstests/noPassthrough/queryStats/query_stats_collectionType.js @@ -1,5 +1,7 @@ /** - * Test that collectionType is returned properly in $queryStats. + * Test that collectionType is returned properly in $queryStats. Checks for collection types + * "collection", "view", "timeseries", "nonExistent", and "virtual". Type "changeStream" is covered + * in query_stats_changeStreams.js. * @tags: [featureFlagQueryStats] */ load("jstests/libs/query_stats_utils.js"); @@ -38,15 +40,16 @@ function runTest(conn) { coll.insert({v: 3, time: ISODate("2021-05-18T02:00:00.000Z")}); coll.find({v: 6}).toArray(); coll.aggregate().toArray(); + // QueryStats should still be collected for queries run on nonexistent collections. assert.commandWorked(testDB.runCommand({find: jsTestName() + "_nonExistent", filter: {v: 6}})); assert.commandWorked( testDB.runCommand({aggregate: jsTestName() + "_nonExistent", pipeline: [], cursor: {}})); - // Verify that we have two telemetry entries for the collection type. This assumes we have + // Verify that we have two query stats entries for the collection type. This assumes we have // executed one find and one agg query for the given collection type. - function verifyTelemetryForCollectionType(collectionType) { - const telemetry = getQueryStats(conn, { + function verifyQueryStatsForCollectionType(collectionType) { + const queryStats = getQueryStats(conn, { extraMatch: { "key.collectionType": collectionType, "key.queryShape.cmdNs.coll": jsTestName() + "_" + collectionType @@ -57,13 +60,27 @@ function runTest(conn) { // that find() queries over views are rewritten to // aggregate(). Ie, the query shapes are different because the // queries are different. - assert.eq(2, telemetry.length, "Expected result for collection type " + collectionType); + assert.eq(2, queryStats.length, "Expected result for collection type " + collectionType); } - verifyTelemetryForCollectionType("collection"); - verifyTelemetryForCollectionType("view"); - verifyTelemetryForCollectionType("timeseries"); - verifyTelemetryForCollectionType("nonExistent"); + verifyQueryStatsForCollectionType("collection"); + verifyQueryStatsForCollectionType("view"); + verifyQueryStatsForCollectionType("timeseries"); + verifyQueryStatsForCollectionType("nonExistent"); + + // Run commands that should be tracked as "virtual" collection type. + assert.commandWorked( + testDB.adminCommand({aggregate: 1, pipeline: [{$currentOp: {}}], cursor: {}})); + assert.commandWorked(testDB.adminCommand( + {aggregate: 1, pipeline: [{$documents: [{a: 1}, {a: 4}]}], cursor: {}})); + assert.commandWorked( + testDB.adminCommand({aggregate: 1, pipeline: [{$listLocalSessions: {}}], cursor: {}})); + + // Verify the queries on "virtual" collection types were tracked appropriately. This includes + // the 3 queries directly run above, in addition to 1 entry for the $queryStats aggregations + // run for the test. + let queryStats = getQueryStats(conn, {extraMatch: {"key.collectionType": "virtual"}}); + assert.eq(4, queryStats.length); // Verify that, for views, we capture the original query before it's rewritten. The view would // include a $gt predicate on 'v'. @@ -101,7 +118,6 @@ if (false) { setParameter: { internalQueryStatsSamplingRate: -1, featureFlagQueryStats: true, - 'failpoint.skipClusterParameterRefresh': "{'mode':'alwaysOn'}" } }, }); diff --git a/jstests/noPassthrough/queryStats/query_stats_disable_after_initial_request.js b/jstests/noPassthrough/queryStats/query_stats_disable_after_initial_request.js index 08e87be016e..68580dda93a 100644 --- a/jstests/noPassthrough/queryStats/query_stats_disable_after_initial_request.js +++ b/jstests/noPassthrough/queryStats/query_stats_disable_after_initial_request.js @@ -4,6 +4,8 @@ * @tags: [featureFlagQueryStats] */ load("jstests/libs/query_stats_utils.js"); // For getQueryStatsFindCmd. +(function() { +"use strict"; // Test that no QueryStats entry is written when (1) dispatching an initial find query, (2) // disabling QueryStats, then (3) completing the command. Below, we run variations of this test @@ -66,4 +68,5 @@ testStatsAreNotCollectedWhenDisabledBeforeCommandCompletion({ enableQueryStatsFn: () => setQueryStatsCacheSize("10MB") }); -MongoRunner.stopMongod(conn);
\ No newline at end of file +MongoRunner.stopMongod(conn); +}());
\ No newline at end of file diff --git a/jstests/noPassthrough/queryStats/query_stats_expressions.js b/jstests/noPassthrough/queryStats/query_stats_expressions.js new file mode 100644 index 00000000000..fe7f9bc4ebb --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_expressions.js @@ -0,0 +1,76 @@ +/** + * Test that queryStats works properly for a find command that uses agg expressions and produces the + * proper query shape without issues during re-parsing. + * @tags: [featureFlagQueryStatsFindCommand] + */ +load("jstests/libs/query_stats_utils.js"); // For getQueryStats and resetQueryStatsStore. + +(function() { +"use strict"; + +// Turn on the collecting of telemetry metrics. +let options = { + setParameter: {internalQueryStatsRateLimit: -1}, +}; + +const conn = MongoRunner.runMongod(options); +const testDB = conn.getDB('test'); +var coll = testDB[jsTestName()]; +coll.drop(); + +const bulk = coll.initializeUnorderedBulkOp(); +const numDocs = 100; +for (let i = 0; i < numDocs / 2; ++i) { + bulk.insert({foo: i, bar: i, applyDiscount: true, word: "asdf"}); + bulk.insert({foo: i, bar: i, applyDiscount: false, word: "ghjk"}); +} +assert.commandWorked(bulk.execute()); +coll.createIndex({foo: 1}); + +// Tests that $meta is re-parsed correctly by ensuring the metaDataKeyword is not serialized as +// string literal. +{ + resetQueryStatsStore(conn, "1MB"); + coll.find({$expr: {idxKey: {$meta: "indexKey"}}}).itcount(); + let queryStats = getQueryStats(testDB); + assert.eq({"$expr": {"idxKey": {"$meta": "indexKey"}}}, queryStats[0].key.queryShape.filter); +} + +// Tests that $regexMatch is re-parsed correctly. The parser does not check that regex is a valid +// regex pattern, so regular serialization is okay. +{ + resetQueryStatsStore(conn, "1MB"); + coll.find({$expr: {result: {$regexMatch: {input: "$word", regex: "^a"}}}}).itcount(); + let queryStats = getQueryStats(testDB); + assert.eq({"$expr": {"result": {"$regexMatch": {"input": "$word", "regex": "?string"}}}}, + queryStats[0].key.queryShape.filter); +} + +// Tests that $let is re-parsed correctly, by ensuring the variables are not serialized as string +// literals. +{ + resetQueryStatsStore(conn, "1MB"); + coll.find({$expr: {$let: { + vars: { + total: { $add: [ '$foo', '$bar' ] }, + discounted: { $cond: { if: '$applyDiscount', then: 0.9, else: 1 } } + }, + in: { $multiply: [ "$$total", "$$discounted" ] } + }}}).itcount(); + let queryStats = getQueryStats(testDB); + assert.eq({ + "$expr": { + "$let": { + "vars": { + "total": {"$add": ["$foo", "$bar"]}, + "discounted": {"$cond": ["$applyDiscount", "?number", "?number"]} + }, + "in": {"$multiply": ["$$total", "$$discounted"]} + } + } + }, + queryStats[0].key.queryShape.filter); +} + +MongoRunner.stopMongod(conn); +}()); diff --git a/jstests/noPassthrough/queryStats/query_stats_feature_flag.js b/jstests/noPassthrough/queryStats/query_stats_feature_flag.js index bcce489d8da..c71ee1bdb93 100644 --- a/jstests/noPassthrough/queryStats/query_stats_feature_flag.js +++ b/jstests/noPassthrough/queryStats/query_stats_feature_flag.js @@ -1,5 +1,5 @@ /** - * Test that calls to read from telemetry store fail when feature flag is turned off. + * Test that calls to read from query stats store fail when feature flag is turned off. */ load('jstests/libs/analyze_plan.js'); load("jstests/libs/feature_flag_util.js"); @@ -9,9 +9,11 @@ load("jstests/libs/feature_flag_util.js"); // This test specifically tests error handling when the feature flag is not on. // TODO SERVER-65800 this test can be removed when the feature flag is removed. +// TODO SERVER-79494 remove reference to featureFlagQueryStatsFindCommand. const conn = MongoRunner.runMongod(); const testDB = conn.getDB('test'); -if (FeatureFlagUtil.isEnabled(testDB, "QueryStats")) { +if (FeatureFlagUtil.isEnabled(testDB, "QueryStats") || + FeatureFlagUtil.isEnabled(testDB, "QueryStatsFindCommand")) { jsTestLog("Skipping test since query stats are enabled."); MongoRunner.stopMongod(conn); return; diff --git a/jstests/noPassthrough/queryStats/query_stats_feature_flags_small_off_big_on.js b/jstests/noPassthrough/queryStats/query_stats_feature_flags_small_off_big_on.js new file mode 100644 index 00000000000..d5947581b48 --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_feature_flags_small_off_big_on.js @@ -0,0 +1,63 @@ +/** + * Test that query stats are collected properly when featureFlagQueryStatsFindCommand is disabled + * but featureFlagQueryStats is enabled. Metrics should be collected for find and agg queries. + * TODO SERVER-79494 remove this test once featureFlagQueryStatsFindCommand is removed. + */ +load("jstests/libs/query_stats_utils.js"); +load("jstests/libs/feature_flag_util.js"); // For 'FeatureFlagUtil' + +(function() { +"use strict"; + +function runTest(conn) { + const testDB = conn.getDB('test'); + let coll = testDB[jsTestName()]; + coll.drop(); + + coll.insert({x: 0}); + coll.insert({x: 1}); + coll.insert({x: 2}); + + let res = coll.aggregate([{$match: {x: 2}}]).toArray(); + assert.eq(1, res.length, res); + + let queryStats = getQueryStats(conn); + // Ensure metrics were collected for the agg query. + assert.eq(1, queryStats.length, queryStats); + + res = coll.find({x: 2}).toArray(); + assert.eq(1, res.length, res); + queryStats = getQueryStats(conn); + // Ensure we collected metrics for the find query, agg query, and additional agg query from + // calling $queryStats. + assert.eq(3, queryStats.length, queryStats); + verifyMetrics(queryStats); +} + +const conn = MongoRunner.runMongod({ + setParameter: {internalQueryStatsRateLimit: -1, featureFlagQueryStats: true}, +}); +const testDB = conn.getDB('test'); +if (FeatureFlagUtil.isEnabled(testDB, "QueryStatsFindCommand")) { + jsTestLog("Skipping test since featureFlagQueryStatsFindCommand is on."); + MongoRunner.stopMongod(conn); + quit(); +} +runTest(conn); +MongoRunner.stopMongod(conn); + +const st = new ShardingTest({ + mongos: 1, + shards: 1, + config: 1, + rs: {nodes: 1}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + featureFlagQueryStats: true, + } + } +}); +runTest(st.s); +st.stop(); +}());
\ No newline at end of file diff --git a/jstests/noPassthrough/queryStats/query_stats_feature_flags_small_on_big_off.js b/jstests/noPassthrough/queryStats/query_stats_feature_flags_small_on_big_off.js new file mode 100644 index 00000000000..a0eaa6a3a94 --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_feature_flags_small_on_big_off.js @@ -0,0 +1,66 @@ +/** + * Test that query stats are collected properly when featureFlagQueryStatsFindCommand is enabled but + * featureFlagQueryStats is disabled. Metrics for find commands should be collected, but metrics for + * aggregate commmands should not be collected. + * TODO SERVER-79494 remove this test once featureFlagQueryStatsFindCommand is removed. + */ +load("jstests/libs/query_stats_utils.js"); +load("jstests/libs/feature_flag_util.js"); // For 'FeatureFlagUtil' + +(function() { +"use strict"; + +function runTest(conn) { + const testDB = conn.getDB('test'); + let coll = testDB[jsTestName()]; + coll.drop(); + + coll.insert({x: 0}); + coll.insert({x: 1}); + coll.insert({x: 2}); + + let res = coll.aggregate([{$match: {x: 2}}]).toArray(); + assert.eq(1, res.length, res); + + let queryStats = getQueryStats(conn); + // Ensure no metrics have been colleted so far. + assert.eq(0, queryStats.length, queryStats); + + res = coll.find({x: 2}).toArray(); + assert.eq(1, res.length, res); + queryStats = getQueryStats(conn); + // Ensure we collected metrics for the find command. + assert.eq(1, queryStats.length, queryStats); + const entry = queryStats[0]; + assert.eq("find", entry.key.queryShape.command); + assert.eq({"x": {$eq: "?number"}}, entry.key.queryShape.filter); + verifyMetrics(queryStats); +} + +const conn = MongoRunner.runMongod({ + setParameter: {internalQueryStatsRateLimit: -1, featureFlagQueryStatsFindCommand: true}, +}); +const testDB = conn.getDB('test'); +if (FeatureFlagUtil.isEnabled(testDB, "QueryStats")) { + jsTestLog("Skipping test since full query stats are enabled."); + MongoRunner.stopMongod(conn); + quit(); +} +runTest(conn); +MongoRunner.stopMongod(conn); + +const st = new ShardingTest({ + mongos: 1, + shards: 1, + config: 1, + rs: {nodes: 1}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + featureFlagQueryStatsFindCommand: true, + } + } +}); +runTest(st.s); +st.stop(); +}());
\ No newline at end of file diff --git a/jstests/noPassthrough/queryStats/query_stats_find_cmd_collect_on_mongos.js b/jstests/noPassthrough/queryStats/query_stats_find_cmd_collect_on_mongos.js new file mode 100644 index 00000000000..e3afdda143b --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_find_cmd_collect_on_mongos.js @@ -0,0 +1,130 @@ +/** + * Test that mongos is collecting query stats metrics for find queries. + * @tags: [featureFlagQueryStatsFindCommand] + */ + +load('jstests/libs/query_stats_utils.js'); + +(function() { +"use strict"; + +const setup = () => { + const st = new ShardingTest({ + mongos: 1, + shards: 1, + config: 1, + rs: {nodes: 1}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + } + }, + }); + const mongos = st.s; + const db = mongos.getDB("test"); + const coll = db.coll; + coll.insert({v: 1}); + coll.insert({v: 4}); + return st; +}; + +// Assert that, for find queries, no query stats results are written until a cursor has reached +// exhaustion; ensure accurate results once they're written. +{ + const st = setup(); + const db = st.s.getDB("test"); + const collName = "coll"; + const coll = db[collName]; + + const queryStatsKey = { + queryShape: { + cmdNs: {db: "test", coll: "coll"}, + command: "find", + filter: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}, + }, + readConcern: {level: "local", provenance: "implicitDefault"}, + batchSize: "?number", + client: {application: {name: "MongoDB Shell"}} + }; + + const cursor = coll.find({v: {$gt: 0, $lt: 5}}).batchSize(1); // returns 1 doc + + // Since the cursor hasn't been exhausted yet, ensure no query stats results have been written + // yet. + let queryStats = getQueryStats(db); + assert.eq(0, queryStats.length, queryStats); + + // Run a getMore to exhaust the cursor, then ensure query stats results have been written + // accurately. batchSize must be 2 so the cursor recognizes exhaustion. + assert.commandWorked(db.runCommand({ + getMore: cursor.getId(), + collection: coll.getName(), + batchSize: 2 + })); // returns 1 doc, exhausts the cursor + queryStats = getQueryStatsFindCmd(db); + assert.eq(1, queryStats.length, queryStats); + assertExpectedResults(queryStats[0], + queryStatsKey, + /* expectedExecCount */ 1, + /* expectedDocsReturnedSum */ 2, + /* expectedDocsReturnedMax */ 2, + /* expectedDocsReturnedMin */ 2, + /* expectedDocsReturnedSumOfSq */ 4, + /* getMores */ true); + + // Run more queries (to exhaustion) with the same query shape, and ensure query stats results + // are accurate. + coll.find({v: {$gt: 2, $lt: 3}}).batchSize(10).toArray(); // returns 0 docs + coll.find({v: {$gt: 0, $lt: 1}}).batchSize(10).toArray(); // returns 0 docs + coll.find({v: {$gt: 0, $lt: 2}}).batchSize(10).toArray(); // return 1 doc + queryStats = getQueryStatsFindCmd(db); + assert.eq(1, queryStats.length, queryStats); + assertExpectedResults(queryStats[0], + queryStatsKey, + /* expectedExecCount */ 4, + /* expectedDocsReturnedSum */ 3, + /* expectedDocsReturnedMax */ 2, + /* expectedDocsReturnedMin */ 0, + /* expectedDocsReturnedSumOfSq */ 5, + /* getMores */ true); + + st.stop(); +} + +// Assert on batchSize-limited find queries that killCursors will write metrics with partial results +// to the query stats store. +{ + const st = setup(); + const db = st.s.getDB("test"); + const collName = "coll"; + const coll = db[collName]; + + const queryStatsKey = { + queryShape: { + cmdNs: {db: "test", coll: "coll"}, + command: "find", + filter: {$and: [{v: {$gt: "?number"}}, {v: {$lt: "?number"}}]}, + }, + readConcern: {level: "local", provenance: "implicitDefault"}, + batchSize: "?number", + client: {application: {name: "MongoDB Shell"}} + }; + + const cursor1 = coll.find({v: {$gt: 0, $lt: 5}}).batchSize(1); // returns 1 doc + const cursor2 = coll.find({v: {$gt: 0, $lt: 2}}).batchSize(1); // returns 1 doc + + assert.commandWorked( + db.runCommand({killCursors: coll.getName(), cursors: [cursor1.getId(), cursor2.getId()]})); + const queryStats = getQueryStats(db); + assert.eq(1, queryStats.length); + assertExpectedResults(queryStats[0], + queryStatsKey, + /* expectedExecCount */ 2, + /* expectedDocsReturnedSum */ 2, + /* expectedDocsReturnedMax */ 1, + /* expectedDocsReturnedMin */ 1, + /* expectedDocsReturnedSumOfSq */ 2, + /* getMores */ false); + st.stop(); +} +}());
\ No newline at end of file diff --git a/jstests/noPassthrough/queryStats/query_stats_key.js b/jstests/noPassthrough/queryStats/query_stats_key.js index f4905e00e0b..69264b6194f 100644 --- a/jstests/noPassthrough/queryStats/query_stats_key.js +++ b/jstests/noPassthrough/queryStats/query_stats_key.js @@ -1,6 +1,6 @@ /** - * This test confirms that telemetry store key fields are properly nested and none are missing. - * @tags: [featureFlagQueryStats] + * This test confirms that query stats store key fields are properly nested and none are missing. + * @tags: [featureFlagQueryStatsFindCommand] */ load("jstests/libs/query_stats_utils.js"); (function() { @@ -32,7 +32,6 @@ function confirmAllFieldsPresent(queryStatsEntries) { "filter", "sort", "projection", - "hint", "skip", "limit", "singleBatch", @@ -61,7 +60,8 @@ function confirmAllFieldsPresent(queryStatsEntries) { "apiVersion", "apiStrict", "collectionType", - "client" + "client", + "hint", ]; for (const entry of queryStatsEntries) { @@ -128,9 +128,9 @@ let commandObj = { }; assert.commandWorked(testDB.runCommand(commandObj)); -let telemetry = getQueryStats(conn); -assert.eq(1, telemetry.length); -confirmAllFieldsPresent(telemetry); +let stats = getQueryStats(conn); +assert.eq(1, stats.length); +confirmAllFieldsPresent(stats); // $hint can only be string(index name) or object (index spec). assert.throwsWithCode(() => { diff --git a/jstests/noPassthrough/queryStats/query_stats_logging.js b/jstests/noPassthrough/queryStats/query_stats_logging.js new file mode 100644 index 00000000000..cbc0e161142 --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_logging.js @@ -0,0 +1,99 @@ +/** + * Test logging of $queryStats. + * @tags: [featureFlagQueryStats] + */ + +(function() { +"use strict"; +let options = { + setParameter: {internalQueryStatsRateLimit: -1}, +}; + +function countMatching(arr, func) { + let count = 0; + let i = arr.length; + while (i--) { + if (func(arr[i])) { + count += 1; + } + } + return count; +} + +function runQueryStatsAndVerifyLogs({pipeline, transformed, logLevel}) { + const conn = MongoRunner.runMongod(options); + const db = conn.getDB('test'); + var coll = db[jsTestName()]; + coll.drop(); + + // Set the logLevel. + assert.commandWorked(db.setLogLevel(logLevel, "queryStats")); + + // Insert a document and run two find command with different shapes so that we have two entries + // in the queryStats store. + assert.commandWorked(coll.insert({foo: 1, bar: "hello"})); + assert.eq(coll.find({foo: 1}).itcount(), 1); + assert.eq(coll.find({bar: "hello"}).itcount(), 1); + + // Run $queryStats with the specified pipeline. + let result = conn.adminCommand({aggregate: 1, pipeline: [pipeline], cursor: {}}); + assert.commandWorked(result); + + if (logLevel >= 1) { + // Checking that we log the invocation of $queryStats. + let spec = transformed + ? {"transformIdentifiers": {"algorithm": "hmac-sha-256", "hmacKey": "###"}} + : {}; + assert(checkLog.checkContainsWithCountJson( + conn, 7808300, {"commandSpec": spec}, 1, null, true), + "failed to find log with id " + 7808300); + } + + // We only log the output of $queryStats when invoked with transformation, and log level at + // least 3. + if (logLevel >= 3 && transformed) { + // Checking that we are logging both entries in the query stats store. + assert(checkLog.checkContainsWithCountJson(conn, 7808301, {}, 2, null, true), + "failed to find log with id " + 7808301); + + // Checking that the output is what we expect. + const query = assert.commandWorked(db.adminCommand({getLog: "global"})); + assert(query.hasOwnProperty("log"), "no log field"); + assert.eq(countMatching(query.log, function(v) { + const has = (s) => v.indexOf(s) !== -1; + return has("thisOutput") && has("key:") && has("queryShape") && + (has("?number") || has("?string")); + }), 2); + + // Checking that we log when we are done outputting the results of $queryStats. + assert(checkLog.checkContainsWithCountJson(conn, 7808302, {}, 1, null, true), + "failed to find log with id " + 7808302); + } + + // At a log level of 0 we should not be seeing any of these logs. + if (logLevel == 0) { + assert(checkLog.checkContainsWithCountJson(conn, 7808300, {}, 0, null, true)); + assert(checkLog.checkContainsWithCountJson(conn, 7808301, {}, 0, null, true)); + assert(checkLog.checkContainsWithCountJson(conn, 7808302, {}, 0, null, true)); + } + + MongoRunner.stopMongod(conn); +} + +// Testing logging with transformation. +const hmacKey = "MjM0NTY3ODkxMDExMTIxMzE0MTUxNjE3MTgxOTIwMjE="; +let pipeline = { + $queryStats: {transformIdentifiers: {algorithm: "hmac-sha-256", hmacKey: BinData(8, hmacKey)}} +}; +runQueryStatsAndVerifyLogs({pipeline: pipeline, transformed: true, logLevel: 3}); +runQueryStatsAndVerifyLogs({pipeline: pipeline, transformed: true, logLevel: 1}); +runQueryStatsAndVerifyLogs({pipeline: pipeline, transformed: true, logLevel: 0}); + +// Testing logging without transformation. +pipeline = { + $queryStats: {} +}; +runQueryStatsAndVerifyLogs({pipeline: pipeline, transformed: false, logLevel: 3}); +runQueryStatsAndVerifyLogs({pipeline: pipeline, transformed: false, logLevel: 1}); +runQueryStatsAndVerifyLogs({pipeline: pipeline, transformed: false, logLevel: 0}); +}()); diff --git a/jstests/noPassthrough/queryStats/query_stats_metrics_across_getMore_calls.js b/jstests/noPassthrough/queryStats/query_stats_metrics_across_getMore_calls.js index 659d065a6c1..6f0dfe33373 100644 --- a/jstests/noPassthrough/queryStats/query_stats_metrics_across_getMore_calls.js +++ b/jstests/noPassthrough/queryStats/query_stats_metrics_across_getMore_calls.js @@ -1,6 +1,6 @@ /** * Test that the queryStats metrics are aggregated properly by distinct query shape over getMore - * calls. + * calls, for agg commands. * @tags: [featureFlagQueryStats] */ load("jstests/libs/query_stats_utils.js"); // For verifyMetrics and getQueryStatsAggCmd. diff --git a/jstests/noPassthrough/queryStats/query_stats_regex.js b/jstests/noPassthrough/queryStats/query_stats_regex.js index b910df94f4e..2a96b3e898e 100644 --- a/jstests/noPassthrough/queryStats/query_stats_regex.js +++ b/jstests/noPassthrough/queryStats/query_stats_regex.js @@ -1,6 +1,6 @@ /** * Test that telemetry works properly for a find command that uses regex. - * @tags: [featureFlagQueryStats] + * @tags: [featureFlagQueryStatsFindCommand] */ (function() { "use strict"; diff --git a/jstests/noPassthrough/queryStats/query_stats_sampling_rate.js b/jstests/noPassthrough/queryStats/query_stats_sampling_rate.js index 74f2cff3055..35cfe209567 100644 --- a/jstests/noPassthrough/queryStats/query_stats_sampling_rate.js +++ b/jstests/noPassthrough/queryStats/query_stats_sampling_rate.js @@ -1,9 +1,10 @@ /** - * Test that calls to read from telemetry store fail when sampling rate is not greater than 0 even + * Test that calls to read from query stats store fail when sampling rate is not greater than 0 even * if feature flag is on. - * @tags: [featureFlagQueryStats] + * @tags: [featureFlagQueryStatsFindCommand] */ load('jstests/libs/analyze_plan.js'); +load("jstests/libs/query_stats_utils.js"); (function() { "use strict"; @@ -20,19 +21,17 @@ for (var i = 0; i < 20; i++) { coll.insert({foo: 0, bar: Math.floor(Math.random() * 3)}); } -coll.aggregate([{$match: {foo: 1}}], {cursor: {batchSize: 2}}); +coll.find({foo: 1}).batchSize(2).toArray(); -// Reading telemetry store with a sampling rate of 0 should return 0 documents. -let telStore = testdb.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}}); -assert.eq(telStore.cursor.firstBatch.length, 0); +// Reading query stats store with a sampling rate of 0 should return 0 documents. +let stats = getQueryStats(testdb); +assert.eq(stats.length, 0); -// Reading telemetry store should work now with a sampling rate of greater than 0. -assert.commandWorked( - testdb.adminCommand({setParameter: 1, internalQueryStatsRateLimit: 2147483647})); -coll.aggregate([{$match: {foo: 1}}], {cursor: {batchSize: 2}}); -telStore = assert.commandWorked( - testdb.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}})); -assert.eq(telStore.cursor.firstBatch.length, 1); +// Reading query stats store should work now with a sampling rate of greater than 0. +assert.commandWorked(testdb.adminCommand({setParameter: 1, internalQueryStatsRateLimit: -1})); +coll.find({foo: 1}).batchSize(2).toArray(); +stats = getQueryStats(testdb); +assert.eq(stats.length, 1); MongoRunner.stopMongod(conn); }()); diff --git a/jstests/noPassthrough/queryStats/query_stats_server_status_metrics.js b/jstests/noPassthrough/queryStats/query_stats_server_status_metrics.js index ff470589b5f..c1070cf6ec5 100644 --- a/jstests/noPassthrough/queryStats/query_stats_server_status_metrics.js +++ b/jstests/noPassthrough/queryStats/query_stats_server_status_metrics.js @@ -161,6 +161,47 @@ function telemetryStoreWriteErrorsTest(conn, testDB, coll, testOptions) { } /** + * Test that running the $queryStats aggregation stage correctly does or does not impact + * serverStatus counters (whichever is applicable). + */ +function queryStatsAggregationStageTest(conn, testDB, coll) { + // First, ensure that the query stats store is empty. + assert.eq(testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes, 0); + assert.eq(testDB.serverStatus().metrics.queryStats.numEvicted, 0); + + // Insert some query stats data and capture "before" serverStatus metrics. + for (let i = 100; i < 200; i++) { + coll.aggregate([{$match: {["foo" + i]: "bar"}}]).itcount(); + } + + let sizeBefore = testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes; + const evictedBefore = testDB.serverStatus().metrics.queryStats.numEvicted; + + // Run a $queryStats pipeline. We should insert a new entry for this query. + assert.commandWorked( + testDB.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}})); + + assert.eq(testDB.serverStatus().metrics.queryStats.numEvicted, + evictedBefore, + "$queryStats should not have triggered evictions"); + assert.gt(testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes, + sizeBefore, + "$queryStats pipeline should have been added to the query stats store"); + sizeBefore = testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes; + + // Now run $queryStats again. The command should be fully read-only. + assert.commandWorked( + testDB.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}})); + + assert.eq(testDB.serverStatus().metrics.queryStats.numEvicted, + evictedBefore, + "$queryStats should not have triggered evictions"); + assert.eq(testDB.serverStatus().metrics.queryStats.queryStatsStoreSizeEstimateBytes, + sizeBefore, + "$queryStats pipeline should not have impacted query stats store size"); +} + +/** * In this configuration, we insert enough entries into the telemetry store to trigger LRU * eviction. */ @@ -217,4 +258,12 @@ runTestWithMongodOptions({ setParameter: {internalQueryStatsCacheSize: "0.00001MB", internalQueryStatsRateLimit: -1}, }, telemetryStoreWriteErrorsTest); + +/** + * Tests that $queryStats has expected effects (or no effect) on counters. + */ +runTestWithMongodOptions({ + setParameter: {internalQueryStatsCacheSize: "2MB", internalQueryStatsRateLimit: -1}, +}, + queryStatsAggregationStageTest); }()); diff --git a/jstests/noPassthrough/queryStats/query_stats_upgrade.js b/jstests/noPassthrough/queryStats/query_stats_upgrade.js index 35a05c52b43..86598b548c0 100644 --- a/jstests/noPassthrough/queryStats/query_stats_upgrade.js +++ b/jstests/noPassthrough/queryStats/query_stats_upgrade.js @@ -1,6 +1,6 @@ /** - * Test that telemetry doesn't work on a lower FCV version but works after an FCV upgrade. - * @tags: [featureFlagQueryStats] + * Test that query stats doesn't work on a lower FCV version but works after an FCV upgrade. + * @tags: [featureFlagQueryStatsFindCommand] */ load('jstests/libs/analyze_plan.js'); load("jstests/libs/feature_flag_util.js"); @@ -11,8 +11,6 @@ load("jstests/libs/feature_flag_util.js"); const dbpath = MongoRunner.dataPath + jsTestName(); let conn = MongoRunner.runMongod({dbpath: dbpath}); let testDB = conn.getDB(jsTestName()); -// This test should only be run with the flag enabled. -assert(FeatureFlagUtil.isEnabled(testDB, "QueryStats")); function testLower(restart = false) { let adminDB = conn.getDB("admin"); @@ -26,13 +24,14 @@ function testLower(restart = false) { } assert.commandFailedWithCode( - testDB.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}}), 6579000); + testDB.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}}), + [6579000, ErrorCodes.QueryFeatureNotAllowed]); // Upgrade FCV. assert.commandWorked(adminDB.runCommand( {setFeatureCompatibilityVersion: binVersionToFCV("latest"), confirm: true})); - // We should be able to run a telemetry pipeline now that the FCV is correct. + // We should be able to run a query stats pipeline now that the FCV is correct. assert.commandWorked( testDB.adminCommand({aggregate: 1, pipeline: [{$queryStats: {}}], cursor: {}}), ); diff --git a/jstests/noPassthrough/queryStats/redact_queries_with_nonobject_fields.js b/jstests/noPassthrough/queryStats/redact_queries_with_nonobject_fields.js index 502a3ab14a9..4543183f61c 100644 --- a/jstests/noPassthrough/queryStats/redact_queries_with_nonobject_fields.js +++ b/jstests/noPassthrough/queryStats/redact_queries_with_nonobject_fields.js @@ -1,5 +1,5 @@ /** - * Test that telemetry key generation works for queries with non-object fields. + * Test that query stats key generation works for queries with non-object fields. * @tags: [featureFlagQueryStats] */ load('jstests/libs/analyze_plan.js'); @@ -7,7 +7,7 @@ load('jstests/libs/analyze_plan.js'); (function() { "use strict"; -// Turn on the collecting of telemetry metrics. +// Turn on the collecting of query stats metrics. let options = { setParameter: {internalQueryStatsRateLimit: -1}, }; diff --git a/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_log.js b/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_log.js index 7e7599b3cd6..60651aa1ce1 100644 --- a/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_log.js +++ b/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_log.js @@ -1,9 +1,6 @@ /** * Test that the queryStats HMAC key is not logged. - * @tags: [ - * featureFlagQueryStats, - * requires_sharding, - * ] + * @tags: [featureFlagQueryStatsFindCommand] */ (function() { diff --git a/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_profile.js b/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_profile.js index b836ed74be3..2b63ced6fd0 100644 --- a/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_profile.js +++ b/jstests/noPassthrough/queryStats/redact_sensitive_fields_in_profile.js @@ -1,6 +1,6 @@ /** * Test that the queryStats HMAC key is not leaked during profiling. - * @tags: [featureFlagQueryStats] + * @tags: [featureFlagQueryStatsFindCommand] */ (function() { "use strict"; diff --git a/jstests/noPassthrough/queryStats/repl_set_query_stats_key.js b/jstests/noPassthrough/queryStats/repl_set_query_stats_key.js index 98c86f65cb1..5adb9ecfd4f 100644 --- a/jstests/noPassthrough/queryStats/repl_set_query_stats_key.js +++ b/jstests/noPassthrough/queryStats/repl_set_query_stats_key.js @@ -2,7 +2,7 @@ * This test confirms that queryStats store key fields specific to replica sets (readConcern and * readPreference) are included and correctly shapified. General command fields related to api * versioning are included for good measure. - * @tags: [featureFlagQueryStats] + * @tags: [featureFlagQueryStatsFindCommand] */ load("jstests/libs/query_stats_utils.js"); (function() { @@ -10,19 +10,16 @@ load("jstests/libs/query_stats_utils.js"); const replTest = new ReplSetTest({name: 'reindexTest', nodes: 2}); -// Turn on the collecting of telemetry metrics. +// Turn on the collecting of query stats metrics. replTest.startSet({setParameter: {internalQueryStatsRateLimit: -1}}); replTest.initiate(); const primary = replTest.getPrimary(); -const secondary = replTest.getSecondary(); const dbName = jsTestName(); const collName = "foobar"; const primaryDB = primary.getDB(dbName); const primaryColl = primaryDB.getCollection(collName); -const secondaryDB = secondary.getDB(dbName); -const secondaryColl = secondaryDB.getCollection(collName); primaryColl.drop(); @@ -53,11 +50,11 @@ let commandObj = { }; const replSetConn = new Mongo(replTest.getURL()); assert.commandWorked(replSetConn.getDB(dbName).runCommand(commandObj)); -let telemetry = getQueryStats(replSetConn, {collName: collName}); -delete telemetry[0].key["collectionType"]; -confirmCommandFieldsPresent(telemetry[0].key, commandObj); +let stats = getQueryStats(replSetConn, {collName: collName}); +delete stats[0].key["collectionType"]; +confirmCommandFieldsPresent(stats[0].key, commandObj); // check that readConcern afterClusterTime is normalized. -assert.eq(telemetry[0].key.readConcern.afterClusterTime, "?timestamp"); +assert.eq(stats[0].key.readConcern.afterClusterTime, "?timestamp"); // check that readPreference not populated and readConcern just has an afterClusterTime field. commandObj["readConcern"] = { @@ -65,11 +62,11 @@ commandObj["readConcern"] = { }; delete commandObj["$readPreference"]; assert.commandWorked(replSetConn.getDB(dbName).runCommand(commandObj)); -telemetry = getQueryStats(replSetConn, {collName}); +stats = getQueryStats(replSetConn, {collName}); // We're not concerned with this field here. -delete telemetry[0].key["collectionType"]; -confirmCommandFieldsPresent(telemetry[0].key, commandObj); -assert.eq(telemetry[0].key["readConcern"], {"afterClusterTime": "?timestamp"}); +delete stats[0].key["collectionType"]; +confirmCommandFieldsPresent(stats[0].key, commandObj); +assert.eq(stats[0].key["readConcern"], {"afterClusterTime": "?timestamp"}); // check that readConcern has no afterClusterTime and fields related to api usage are not present. commandObj["readConcern"] = { @@ -79,11 +76,11 @@ delete commandObj["apiDeprecationErrors"]; delete commandObj["apiVersion"]; delete commandObj["apiStrict"]; assert.commandWorked(replSetConn.getDB(dbName).runCommand(commandObj)); -telemetry = getQueryStats(replSetConn, {collName: collName}); -assert.eq(telemetry[1].key["readConcern"], {level: "local"}); +stats = getQueryStats(replSetConn, {collName: collName}); +assert.eq(stats[1].key["readConcern"], {level: "local"}); // We're not concerned with this field here. -delete telemetry[1].key["collectionType"]; -confirmCommandFieldsPresent(telemetry[1].key, commandObj); +delete stats[1].key["collectionType"]; +confirmCommandFieldsPresent(stats[1].key, commandObj); replTest.stopSet(); })(); diff --git a/jstests/noPassthrough/queryStats/single_batch_on_mongod.js b/jstests/noPassthrough/queryStats/single_batch_on_mongod.js deleted file mode 100644 index 9852aaec586..00000000000 --- a/jstests/noPassthrough/queryStats/single_batch_on_mongod.js +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Test that the queryStats metrics are aggregated properly for queries run on a mongod where the - * results fit into a single batch (and thus don't require a cursor). - * @tags: [featureFlagQueryStats] - */ -load("jstests/libs/query_stats_utils.js"); // For verifyMetrics and getQueryStatsAggCmd. - -(function() { -"use strict"; - -// Turn on the collecting of queryStats metrics. -let options = { - setParameter: {internalQueryStatsRateLimit: -1}, -}; - -const conn = MongoRunner.runMongod(options); -const testDB = conn.getDB('test'); -var coll = testDB[jsTestName()]; -coll.drop(); - -// Bulk insert documents to reduces roundtrips and make timeout on a slow machine less likely. -const bulk = coll.initializeUnorderedBulkOp(); -const numDocs = 100; -for (let i = 0; i < numDocs / 2; ++i) { - bulk.insert({foo: 0, bar: Math.floor(Math.random() * 3)}); - bulk.insert({foo: 1, bar: Math.floor(Math.random() * -2)}); -} -assert.commandWorked(bulk.execute()); - -// Assert that two find queries with identical structures are represented by the same key. -{ - // Note that toArray is necessary to guarantee the query finishes executing on the server (at - // which point an entry is finally written to the queryStats store). - coll.find({foo: {$eq: 0}}).toArray(); - coll.find({foo: {$eq: 1}}).toArray(); - - // This command will return all queryStats store entires. - const queryStatsResults = getQueryStatsFindCmd(testDB); - // Assert there is only one entry. - assert.eq(queryStatsResults.length, 1, queryStatsResults); - const queryStatsEntry = queryStatsResults[0]; - assert.eq(queryStatsEntry.key.queryShape.cmdNs.db, "test"); - assert.eq(queryStatsEntry.key.queryShape.cmdNs.coll, jsTestName()); - assert.eq(queryStatsEntry.key.client.application.name, "MongoDB Shell"); - - // Assert we update execution count for identically shaped queries. - assert.eq(queryStatsEntry.metrics.execCount, 2); - - // Assert queryStats values are accurate for the two above queries. - assert.eq(queryStatsEntry.metrics.docsReturned.sum, numDocs); - assert.eq(queryStatsEntry.metrics.docsReturned.min, numDocs / 2); - assert.eq(queryStatsEntry.metrics.docsReturned.max, numDocs / 2); - - // The total size of documents in the collection should ensure that the queries in this test can - // be executed without requiring multiple batches, but we verify that by looking at the - // timestamps. - assert.eq(queryStatsEntry.metrics.firstResponseExecMicros, - queryStatsEntry.metrics.totalExecMicros); - verifyMetrics(queryStatsResults); -} - -// Assert that two agg queries with identical structures are represented by the same key. -{ - // Note that toArray is necessary to guarantee the query finishes executing on the server (at - // which point an entry is finally written to the queryStats store). - coll.aggregate([{$match: {foo: 1}}]).toArray(); - coll.aggregate([{$match: {foo: 0}}]).toArray(); - - // This command will return all queryStats store entires. - const queryStatsResults = getQueryStatsAggCmd(testDB); - // Assert there is only one entry. - assert.eq(queryStatsResults.length, 1, queryStatsResults); - const queryStatsEntry = queryStatsResults[0]; - assert.eq(queryStatsEntry.key.queryShape.cmdNs.db, "test"); - assert.eq(queryStatsEntry.key.queryShape.cmdNs.coll, jsTestName()); - assert.eq(queryStatsEntry.key.client.application.name, "MongoDB Shell"); - - // Assert we update execution count for identically shaped queries. - assert.eq(queryStatsEntry.metrics.execCount, 2); - - // Assert queryStats values are accurate for the two above queries. - assert.eq(queryStatsEntry.metrics.docsReturned.sum, numDocs); - assert.eq(queryStatsEntry.metrics.docsReturned.min, numDocs / 2); - assert.eq(queryStatsEntry.metrics.docsReturned.max, numDocs / 2); - - // The total size of documents in the collection should ensure that the queries in this test can - // be executed without requiring multiple batches, but we verify that by looking at the - // timestamps. - assert.eq(queryStatsEntry.metrics.firstResponseExecMicros, - queryStatsEntry.metrics.totalExecMicros); - - verifyMetrics(queryStatsResults); -} -MongoRunner.stopMongod(conn); -}()); diff --git a/jstests/noPassthrough/queryStats/single_batch_on_mongod_agg_cmd.js b/jstests/noPassthrough/queryStats/single_batch_on_mongod_agg_cmd.js new file mode 100644 index 00000000000..9048053e89b --- /dev/null +++ b/jstests/noPassthrough/queryStats/single_batch_on_mongod_agg_cmd.js @@ -0,0 +1,61 @@ +/** + * Test that the queryStats metrics are aggregated properly for queries run on a mongod where the + * results fit into a single batch (and thus don't require a cursor), for agg commands. + * @tags: [featureFlagQueryStats] + */ +load("jstests/libs/query_stats_utils.js"); // For verifyMetrics and getQueryStatsAggCmd. + +(function() { +"use strict"; + +// Turn on the collecting of queryStats metrics. +let options = { + setParameter: {internalQueryStatsRateLimit: -1}, +}; + +const conn = MongoRunner.runMongod(options); +const testDB = conn.getDB('test'); +var coll = testDB[jsTestName()]; +coll.drop(); + +// Bulk insert documents to reduces roundtrips and make timeout on a slow machine less likely. +const bulk = coll.initializeUnorderedBulkOp(); +const numDocs = 100; +for (let i = 0; i < numDocs / 2; ++i) { + bulk.insert({foo: 0, bar: Math.floor(Math.random() * 3)}); + bulk.insert({foo: 1, bar: Math.floor(Math.random() * -2)}); +} +assert.commandWorked(bulk.execute()); + +// Assert that two agg queries with identical structures are represented by the same key. + +// Note that toArray is necessary to guarantee the query finishes executing on the server (at +// which point an entry is finally written to the queryStats store). +coll.aggregate([{$match: {foo: 1}}]).toArray(); +coll.aggregate([{$match: {foo: 0}}]).toArray(); + +// This command will return all queryStats store entires. +const queryStatsResults = getQueryStatsAggCmd(testDB); +// Assert there is only one entry. +assert.eq(queryStatsResults.length, 1, queryStatsResults); +const queryStatsEntry = queryStatsResults[0]; +assert.eq(queryStatsEntry.key.queryShape.cmdNs.db, "test"); +assert.eq(queryStatsEntry.key.queryShape.cmdNs.coll, jsTestName()); +assert.eq(queryStatsEntry.key.client.application.name, "MongoDB Shell"); + +// Assert we update execution count for identically shaped queries. +assert.eq(queryStatsEntry.metrics.execCount, 2); + +// Assert queryStats values are accurate for the two above queries. +assert.eq(queryStatsEntry.metrics.docsReturned.sum, numDocs); +assert.eq(queryStatsEntry.metrics.docsReturned.min, numDocs / 2); +assert.eq(queryStatsEntry.metrics.docsReturned.max, numDocs / 2); + +// The total size of documents in the collection should ensure that the queries in this test can +// be executed without requiring multiple batches, but we verify that by looking at the +// timestamps. +assert.eq(queryStatsEntry.metrics.firstResponseExecMicros, queryStatsEntry.metrics.totalExecMicros); + +verifyMetrics(queryStatsResults); +MongoRunner.stopMongod(conn); +}()); diff --git a/jstests/noPassthrough/queryStats/single_batch_on_mongod_find_cmd.js b/jstests/noPassthrough/queryStats/single_batch_on_mongod_find_cmd.js new file mode 100644 index 00000000000..fc8a4ab1faf --- /dev/null +++ b/jstests/noPassthrough/queryStats/single_batch_on_mongod_find_cmd.js @@ -0,0 +1,59 @@ +/** + * Test that the queryStats metrics are aggregated properly for queries run on a mongod where the + * results fit into a single batch (and thus don't require a cursor), for find commands. + * @tags: [featureFlagQueryStatsFindCommand] + */ +load("jstests/libs/query_stats_utils.js"); // For verifyMetrics and getQueryStatsAggCmd. + +(function() { +"use strict"; + +// Turn on the collecting of queryStats metrics. +let options = { + setParameter: {internalQueryStatsRateLimit: -1}, +}; + +const conn = MongoRunner.runMongod(options); +const testDB = conn.getDB('test'); +var coll = testDB[jsTestName()]; +coll.drop(); + +// Bulk insert documents to reduces roundtrips and make timeout on a slow machine less likely. +const bulk = coll.initializeUnorderedBulkOp(); +const numDocs = 100; +for (let i = 0; i < numDocs / 2; ++i) { + bulk.insert({foo: 0, bar: Math.floor(Math.random() * 3)}); + bulk.insert({foo: 1, bar: Math.floor(Math.random() * -2)}); +} +assert.commandWorked(bulk.execute()); + +// Note that toArray is necessary to guarantee the query finishes executing on the server (at +// which point an entry is finally written to the queryStats store). +coll.find({foo: {$eq: 0}}).toArray(); +coll.find({foo: {$eq: 1}}).toArray(); + +// This command will return all queryStats store entires. +const queryStatsResults = getQueryStatsFindCmd(testDB); +// Assert there is only one entry. +assert.eq(queryStatsResults.length, 1, queryStatsResults); +const queryStatsEntry = queryStatsResults[0]; +assert.eq(queryStatsEntry.key.queryShape.cmdNs.db, "test"); +assert.eq(queryStatsEntry.key.queryShape.cmdNs.coll, jsTestName()); +assert.eq(queryStatsEntry.key.client.application.name, "MongoDB Shell"); + +// Assert we update execution count for identically shaped queries. +assert.eq(queryStatsEntry.metrics.execCount, 2); + +// Assert queryStats values are accurate for the two above queries. +assert.eq(queryStatsEntry.metrics.docsReturned.sum, numDocs); +assert.eq(queryStatsEntry.metrics.docsReturned.min, numDocs / 2); +assert.eq(queryStatsEntry.metrics.docsReturned.max, numDocs / 2); + +// The total size of documents in the collection should ensure that the queries in this test can +// be executed without requiring multiple batches, but we verify that by looking at the +// timestamps. +assert.eq(queryStatsEntry.metrics.firstResponseExecMicros, queryStatsEntry.metrics.totalExecMicros); +verifyMetrics(queryStatsResults); + +MongoRunner.stopMongod(conn); +}()); diff --git a/jstests/noPassthroughWithMongod/query_stats_configuration.js b/jstests/noPassthroughWithMongod/query_stats_configuration.js index d7cc2acba53..b9a649aaca7 100644 --- a/jstests/noPassthroughWithMongod/query_stats_configuration.js +++ b/jstests/noPassthroughWithMongod/query_stats_configuration.js @@ -1,14 +1,15 @@ /** - * Tests that the telemetry store can be resized if it is configured, and cannot be resized if it is - * disabled. + * Tests that the query stats store can be resized if it is configured, and cannot be resized if it + * is disabled. */ (function() { "use strict"; load("jstests/libs/feature_flag_util.js"); -if (FeatureFlagUtil.isEnabled(db, "QueryStats")) { - function testTelemetrySetting(paramName, paramValue) { +if (FeatureFlagUtil.isEnabled(db, "QueryStats") || + FeatureFlagUtil.isEnabled(db, "QueryStatsFindCommand")) { + function testQueryStatsSetting(paramName, paramValue) { // The feature flag is enabled - make sure the telemetry store can be configured. const original = assert.commandWorked(db.adminCommand({getParameter: 1, [paramName]: 1})); assert(original.hasOwnProperty(paramName), original); @@ -21,10 +22,10 @@ if (FeatureFlagUtil.isEnabled(db, "QueryStats")) { db.adminCommand({setParameter: 1, [paramName]: originalValue})); } } - testTelemetrySetting("internalQueryStatsCacheSize", "2MB"); - testTelemetrySetting("internalQueryStatsRateLimit", 2147483647); + testQueryStatsSetting("internalQueryStatsCacheSize", "2MB"); + testQueryStatsSetting("internalQueryStatsRateLimit", 2147483647); } else { - // The feature flag is disabled - make sure the telemetry store *cannot* be configured. + // The feature flag is disabled - make sure the query stats store *cannot* be configured. assert.commandFailedWithCode( db.adminCommand({setParameter: 1, internalQueryStatsCacheSize: '2MB'}), 7373500); assert.commandFailedWithCode( diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index df9dfcc0d02..8a4f29dedfb 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -119,7 +119,6 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/clustered_collection_options', '$BUILD_DIR/mongo/db/catalog/collection_crud', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/session/logical_session_id_helpers', '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/db/transaction/transaction', @@ -196,7 +195,7 @@ env.Library( 'curop_failpoint_helpers.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/util/fail_point', ], ) @@ -561,7 +560,6 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/clustered_collection_options', '$BUILD_DIR/mongo/db/catalog/collection_crud', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/repl/storage_interface', 'change_stream_options_manager', 'change_stream_serverless_helpers', @@ -889,7 +887,7 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/collection_uuid_mismatch_info', '$BUILD_DIR/mongo/db/catalog/local_oplog_info', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', 'multitenancy', 'server_base', 'storage/capped_snapshots', @@ -1004,7 +1002,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/client/clientdriver_minimal', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', 'not_primary_error_tracker', 'ops/write_ops_parsers', ], @@ -1052,7 +1050,6 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/transport/service_entry_point', 'curop_metrics', 'rw_concern_d', @@ -1066,6 +1063,7 @@ env.Library( 'commands/fsync_locked', 'concurrency/lock_manager', 'not_primary_error_tracker', + 'query/query_stats/query_stats', 'read_concern_d_impl', 's/sharding_runtime_d', 'service_entry_point_common', @@ -1146,7 +1144,6 @@ env.Library( 'catalog/collection_catalog', 'catalog/index_build_entry_idl', 'index_build_entry_helpers', - 'query/op_metrics', 'repl/tenant_migration_access_blocker', 'resumable_index_builds_idl', 's/forwardable_operation_metadata', @@ -1239,7 +1236,7 @@ env.Library( "curop_metrics.cpp", ], LIBDEPS_PRIVATE=[ - "$BUILD_DIR/mongo/db/query/op_metrics", + "$BUILD_DIR/mongo/db/query/query_stats/query_stats", "commands/server_status_core", ], ) @@ -1265,7 +1262,7 @@ env.Library( 'write_concern.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', 'read_write_concern_defaults', 'repl/read_concern_args', 'repl/speculative_majority_read_info', @@ -1289,7 +1286,7 @@ env.Library( 'read_concern_mongod.idl', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/s/grid', 'concurrency/exception_util', 'repl/repl_coordinator_interface', @@ -1503,7 +1500,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/auth/auth_checks', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/scripting/scripting', '$BUILD_DIR/mongo/util/background_job', '$BUILD_DIR/mongo/util/elapsed_tracker', @@ -1661,7 +1658,7 @@ env.Library( 'query/collation/collator_interface', 'query/datetime/date_time_support', 'query/query_knobs', - 'serialization_options', + 'query_shape_common', 'stats/counters', 'update/pattern_cmp', ], @@ -1674,17 +1671,17 @@ env.Library( ) env.Library( - target='serialization_options', - source=[ + target='query_shape_common', source=[ + 'query/query_shape.idl', + 'query/shape_helpers.cpp', 'query/serialization_options.cpp', - ], - LIBDEPS=[ + ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/exec/document_value/document_value', '$BUILD_DIR/mongo/db/pipeline/field_path', - ], - LIBDEPS_PRIVATE=[], -) + ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/util/namespace_string_database_name_util', + ]) env.Library( target='startup_recovery', @@ -1801,7 +1798,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/pipeline/lite_parsed_document_source', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/session/logical_session_id', 'commands', ], @@ -2276,7 +2273,7 @@ env.Library( LIBDEPS_PRIVATE=[ # NOTE: If you need to add a static or mongo initializer to mongod startup, # please add that library here, as a private library dependency. - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/executor/async_rpc_error_info', '$BUILD_DIR/mongo/executor/network_interface_factory', '$BUILD_DIR/mongo/rpc/rpc', @@ -2407,7 +2404,6 @@ env.Library( '$BUILD_DIR/mongo/db/change_stream_options_manager', '$BUILD_DIR/mongo/db/change_streams_cluster_parameter', '$BUILD_DIR/mongo/db/pipeline/change_stream_expired_pre_image_remover', - '$BUILD_DIR/mongo/db/query/stats/query_stats', '$BUILD_DIR/mongo/db/s/query_analysis_writer', '$BUILD_DIR/mongo/db/set_change_stream_state_coordinator', '$BUILD_DIR/mongo/idl/cluster_server_parameter', @@ -2453,6 +2449,7 @@ env.Library( 'op_observer/user_write_block_mode_op_observer', 'periodic_runner_job_abort_expired_transactions', 'pipeline/process_interface/mongod_process_interface_factory', + 'query/stats/stats', 'repl/drop_pending_collection_reaper', 'repl/initial_syncer', 'repl/repl_coordinator_impl', @@ -2672,7 +2669,6 @@ if wiredtiger: '$BUILD_DIR/mongo/db/ops/write_ops', '$BUILD_DIR/mongo/db/pipeline/change_stream_expired_pre_image_remover', '$BUILD_DIR/mongo/db/query/common_query_enums_and_helpers', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_test_service_context', '$BUILD_DIR/mongo/db/repl/image_collection_entry', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', @@ -2715,6 +2711,7 @@ if wiredtiger: 'mirror_maestro', 'multitenancy', 'operation_time_tracker', + 'query/query_stats/query_stats', 'query_exec', 'read_write_concern_defaults_mock', 'record_id_helpers', diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index f8abf79f273..653f38a192d 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -202,7 +202,7 @@ env.Library( '$BUILD_DIR/mongo/db/commands/authentication_commands', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/db/global_settings', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/util/concurrency/thread_pool', '$BUILD_DIR/mongo/util/icu', '$BUILD_DIR/mongo/util/net/ssl_manager', diff --git a/src/mongo/db/auth/action_type.idl b/src/mongo/db/auth/action_type.idl index 650dadeee57..a111236ac9e 100644 --- a/src/mongo/db/auth/action_type.idl +++ b/src/mongo/db/auth/action_type.idl @@ -149,7 +149,8 @@ enums: planCacheIndexFilter : "planCacheIndexFilter" # view/update index filters planCacheRead : "planCacheRead" # view contents of plan cache planCacheWrite : "planCacheWrite" # clear cache, drop cache entry, pin/unpin/shun plans - queryStatsRead: "queryStatsRead" # view contents of queryStats store + queryStatsRead: "queryStatsRead" # view untransformed contents of queryStats store + queryStatsReadTransformed: "queryStatsReadTransformed" # view transformed contents of queryStats store refineCollectionShardKey : "refineCollectionShardKey" reIndex : "reIndex" remove : "remove" diff --git a/src/mongo/db/auth/builtin_roles.yml b/src/mongo/db/auth/builtin_roles.yml index 19ee41b876a..626f10935c6 100644 --- a/src/mongo/db/auth/builtin_roles.yml +++ b/src/mongo/db/auth/builtin_roles.yml @@ -352,6 +352,7 @@ roles: - setChangeStreamState - getChangeStreamState - queryStatsRead + - queryStatsReadTransformed - checkMetadataConsistency - transitionFromDedicatedConfigServer - transitionToDedicatedConfigServer diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 1b0adf9370a..9b44fbb8e74 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -125,7 +125,7 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/index/index_access_method', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/shard_role', '$BUILD_DIR/mongo/db/storage/duplicate_key_error_info', '$BUILD_DIR/mongo/db/storage/key_string', @@ -142,8 +142,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/shard_role', '$BUILD_DIR/mongo/db/storage/record_store_base', '$BUILD_DIR/mongo/db/storage/storage_repair_observer', @@ -232,7 +232,7 @@ env.Library( 'collection_write_path.cpp', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/shard_role_api', @@ -270,7 +270,7 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/index/index_access_method', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/resumable_index_builds_idl', @@ -307,7 +307,7 @@ env.Library( '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/profile_filter', '$BUILD_DIR/mongo/db/query/collation/collator_factory_interface', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/storage/bson_collection_catalog_entry', @@ -392,7 +392,7 @@ env.Library( '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/op_observer/op_observer', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper', '$BUILD_DIR/mongo/db/repl/oplog', @@ -465,7 +465,7 @@ env.Library( '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/multi_key_path_tracker', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', @@ -493,7 +493,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/index/index_access_method', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/util/fail_point', 'validate_idl', ], @@ -599,8 +599,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/collection_index_usage_tracker', '$BUILD_DIR/mongo/db/fts/base_fts', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_planner', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/shard_role_api', diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index 23fbaa9e538..8db9f882933 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -48,7 +48,7 @@ #include "mongo/db/cursor_server_params.h" #include "mongo/db/jsobj.h" #include "mongo/db/query/explain.h" -#include "mongo/db/query/query_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/util/background.h" diff --git a/src/mongo/db/collection_type.h b/src/mongo/db/collection_type.h index e30056c8146..0c58804bf2a 100644 --- a/src/mongo/db/collection_type.h +++ b/src/mongo/db/collection_type.h @@ -29,18 +29,42 @@ #pragma once -#include "mongo/db/query/util/named_enum.h" +#include "mongo/util/assert_util.h" namespace mongo::query_shape { -#define COLLECTION_TYPE(F) \ - F(unknown) \ - F(timeseries) \ - F(view) \ - F(nonExistent) \ - F(collection) \ - F(end) -QUERY_UTIL_NAMED_ENUM_DEFINE(CollectionType, COLLECTION_TYPE) -#undef COLLECTION_TYPE +// This enum is not compatible with the QUERY_UTIL_NAMED_ENUM_DEFINE util since the "virtual" type +// conflicts with the C++ keyword "virtual". Instead, we manually define the enum and the +// toStringData function below. +enum class CollectionType { + kUnknown, + kCollection, + kView, + kTimeseries, + kChangeStream, + kVirtual, + kNonExistent, +}; + +static StringData toStringData(CollectionType type) { + switch (type) { + case CollectionType::kUnknown: + return "unknown"_sd; + case CollectionType::kCollection: + return "collection"_sd; + case CollectionType::kView: + return "view"_sd; + case CollectionType::kTimeseries: + return "timeseries"_sd; + case CollectionType::kChangeStream: + return "changeStream"_sd; + case CollectionType::kVirtual: + return "virtual"_sd; + case CollectionType::kNonExistent: + return "nonExistent"_sd; + default: + MONGO_UNREACHABLE_TASSERT(7804900); + } +} } // namespace mongo::query_shape diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript index 5c8f2de0365..4d0ce84193b 100644 --- a/src/mongo/db/commands/SConscript +++ b/src/mongo/db/commands/SConscript @@ -200,7 +200,7 @@ env.Library( '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/dbdirectclient', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/backup_cursor_hooks', 'fsync_locked', ], @@ -411,9 +411,9 @@ env.Library( '$BUILD_DIR/mongo/db/pipeline/process_interface/mongo_process_interface', '$BUILD_DIR/mongo/db/query/command_request_response', '$BUILD_DIR/mongo/db/query/cursor_response_idl', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_shape', - '$BUILD_DIR/mongo/db/query/stats/query_stats', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', + '$BUILD_DIR/mongo/db/query/stats/stats', '$BUILD_DIR/mongo/db/query/stats/stats_histograms', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/repl/replica_set_messages', @@ -721,7 +721,7 @@ env.Library( '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/auth/authprivilege', '$BUILD_DIR/mongo/db/catalog/collection_catalog', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_expressions', '$BUILD_DIR/mongo/db/server_base', ], diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index c78bbcaefc4..1b7d3973a2d 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -55,8 +55,9 @@ #include "mongo/db/query/get_executor.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/query/query_shape.h" -#include "mongo/db/query/query_stats.h" -#include "mongo/db/query/query_stats_find_key_generator.h" +#include "mongo/db/query/query_stats/find_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/s/query_analysis_writer.h" #include "mongo/db/service_context.h" @@ -155,14 +156,20 @@ std::unique_ptr<CanonicalQuery> parseQueryAndBeginOperation( // It is important to do this before canonicalizing and optimizing the query, each of which // would alter the query shape. if (!(collection && collection.get()->getCollectionOptions().encryptedFieldConfig)) { - query_stats::registerRequest(opCtx, nss, [&]() { - BSONObj queryShape = query_shape::extractQueryShape( - *parsedRequest, - SerializationOptions::kRepresentativeQueryShapeSerializeOptions, - expCtx); - return std::make_unique<query_stats::FindKeyGenerator>( - expCtx, *parsedRequest, std::move(queryShape), ctx.getCollectionType()); - }); + query_stats::registerRequest( + opCtx, + nss, + [&]() { + // This callback is either never invoked or invoked immediately within + // registerRequest, so use-after-move of parsedRequest isn't an issue. + BSONObj queryShape = query_shape::extractQueryShape( + *parsedRequest, + SerializationOptions::kRepresentativeQueryShapeSerializeOptions, + expCtx); + return std::make_unique<query_stats::FindKeyGenerator>( + expCtx, *parsedRequest, std::move(queryShape), ctx.getCollectionType()); + }, + /*requiresFullQueryStatsFeatureFlag*/ false); } return uassertStatusOK( diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index 8220d3f7015..996f043c713 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -73,8 +73,9 @@ #include "mongo/db/query/plan_summary_stats.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/query/query_planner_common.h" -#include "mongo/db/query/query_stats.h" -#include "mongo/db/query/query_stats_aggregate_key_generator.h" +#include "mongo/db/query/query_stats/aggregate_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/read_concern.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/read_concern_args.h" @@ -981,6 +982,14 @@ Status _runAggregate(OperationContext* opCtx, return std::make_pair(expCtx, std::move(pipeline)); }; + auto collectionType = + ctx ? ctx->getCollectionType() : query_shape::CollectionType::kUnknown; + if (liteParsedPipeline.hasChangeStream()) { + collectionType = query_shape::CollectionType::kChangeStream; + } else if (nss.isCollectionlessAggregateNS()) { + collectionType = query_shape::CollectionType::kVirtual; + } + // If this is a view, resolve it by finding the underlying collection and stitching view // pipelines and this request's pipeline together. We then release our locks before // recursively calling runAggregate(), which will re-acquire locks on the underlying @@ -994,7 +1003,7 @@ Status _runAggregate(OperationContext* opCtx, (!liteParsedPipeline.startsWithCollStats() || ctx->getView()->timeseries())) { try { invariant(collatorToUse.has_value()); - query_stats::registerRequest(opCtx, nss, [&]() { + query_stats::registerRequest(opCtx, request.getNamespace(), [&]() { // In this path we haven't yet parsed the pipeline, but we need to do so for // query shape stats - which should track the queries before views are resolved. // Inside this callback we know we have already checked that query stats are @@ -1011,7 +1020,7 @@ Status _runAggregate(OperationContext* opCtx, expCtx, pipelineInvolvedNamespaces, origNss, - ctx->getCollectionType()); + collectionType); }); } catch (const DBException& ex) { if (ex.code() == 6347902) { @@ -1053,14 +1062,14 @@ Status _runAggregate(OperationContext* opCtx, // with encrypted fields. We still collect query stats on collection-less aggregations. if (!(ctx && ctx->getCollection() && ctx->getCollection()->getCollectionOptions().encryptedFieldConfig)) { - query_stats::registerRequest(opCtx, nss, [&]() { + query_stats::registerRequest(opCtx, request.getNamespace(), [&]() { return std::make_unique<query_stats::AggregateKeyGenerator>( request, *pipeline, expCtx, pipelineInvolvedNamespaces, - nss, - ctx ? ctx->getCollectionType() : query_shape::CollectionType::unknown); + request.getNamespace(), + collectionType); }); } diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript index 1a94a2cc627..458259a11b7 100644 --- a/src/mongo/db/concurrency/SConscript +++ b/src/mongo/db/concurrency/SConscript @@ -27,7 +27,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/commands/server_status_core', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/server_options_servers', '$BUILD_DIR/mongo/db/storage/recovery_unit_base', @@ -104,7 +104,7 @@ env.CppUnitTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authmocks', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/service_context_d_test_fixture', '$BUILD_DIR/mongo/transport/transport_layer_common', '$BUILD_DIR/mongo/transport/transport_layer_mock', diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h index cae355b5704..3dedf0a84e9 100644 --- a/src/mongo/db/curop.h +++ b/src/mongo/db/curop.h @@ -40,7 +40,7 @@ #include "mongo/db/cursor_id.h" #include "mongo/db/operation_context.h" #include "mongo/db/profile_filter.h" -#include "mongo/db/query/query_stats_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" #include "mongo/db/server_options.h" #include "mongo/db/stats/resource_consumption_metrics.h" #include "mongo/db/write_concern_options.h" diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp index b55b118c966..6e2de95fa9e 100644 --- a/src/mongo/db/db_raii.cpp +++ b/src/mongo/db/db_raii.cpp @@ -1586,12 +1586,12 @@ const NamespaceString& AutoGetCollectionForReadCommandMaybeLockFree::getNss() co query_shape::CollectionType AutoGetCollectionForReadCommandMaybeLockFree::getCollectionType() const { if (auto&& view = getView()) { - return view->timeseries() ? query_shape::CollectionType::timeseries - : query_shape::CollectionType::view; + return view->timeseries() ? query_shape::CollectionType::kTimeseries + : query_shape::CollectionType::kView; } auto&& collection = getCollection(); - return collection ? query_shape::CollectionType::collection - : query_shape::CollectionType::nonExistent; + return collection ? query_shape::CollectionType::kCollection + : query_shape::CollectionType::kNonExistent; } diff --git a/src/mongo/db/index/SConscript b/src/mongo/db/index/SConscript index 9f7366cbab5..81562c6abfe 100644 --- a/src/mongo/db/index/SConscript +++ b/src/mongo/db/index/SConscript @@ -72,8 +72,8 @@ iamEnv.Library( '$BUILD_DIR/mongo/db/pipeline/document_path_support', '$BUILD_DIR/mongo/db/query/collation/collator_factory_interface', '$BUILD_DIR/mongo/db/query/collation/collator_interface', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/projection_ast', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query/sort_pattern', '$BUILD_DIR/mongo/db/query_expressions', '$BUILD_DIR/mongo/db/record_id_helpers', diff --git a/src/mongo/db/matcher/expression_path.h b/src/mongo/db/matcher/expression_path.h index 296dc2dfbf6..611acbcb3e8 100644 --- a/src/mongo/db/matcher/expression_path.h +++ b/src/mongo/db/matcher/expression_path.h @@ -196,11 +196,12 @@ public: } void serialize(BSONObjBuilder* out, const SerializationOptions& opts) const override { - auto&& rhs = getSerializedRightHandSide(opts); if (opts.includePath) { - out->append(opts.serializeFieldPathFromString(path()), rhs); + BSONObjBuilder subObj(out->subobjStart(opts.serializeFieldPathFromString(path()))); + appendSerializedRightHandSide(&subObj, opts); + subObj.doneFast(); } else { - out->appendElements(rhs); + appendSerializedRightHandSide(out, opts); } } diff --git a/src/mongo/db/ops/SConscript b/src/mongo/db/ops/SConscript index 7569e7382bf..9118a4854bd 100644 --- a/src/mongo/db/ops/SConscript +++ b/src/mongo/db/ops/SConscript @@ -21,7 +21,7 @@ env.Library( '$BUILD_DIR/mongo/db/matcher/expressions_mongod_only', '$BUILD_DIR/mongo/db/mongod_options', '$BUILD_DIR/mongo/db/query/command_request_response', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/repl/image_collection_entry', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', @@ -60,7 +60,7 @@ env.Library( '$BUILD_DIR/mongo/db/curop_metrics', '$BUILD_DIR/mongo/db/dbhelpers', '$BUILD_DIR/mongo/db/introspect', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/oplog', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 8b20e26017a..c849e850110 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -136,7 +136,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/exec/document_value/document_value', '$BUILD_DIR/mongo/db/query/query_knobs', - '$BUILD_DIR/mongo/db/query/stats/query_stats', + '$BUILD_DIR/mongo/db/query/stats/stats', '$BUILD_DIR/mongo/db/query_expressions', '$BUILD_DIR/mongo/scripting/scripting_common', '$BUILD_DIR/mongo/util/summation', @@ -504,8 +504,8 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/exec/document_value/document_value', - '$BUILD_DIR/mongo/db/query/query_stats_parse', - '$BUILD_DIR/mongo/db/serialization_options', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats_parse', + '$BUILD_DIR/mongo/db/query_shape_common', '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/idl/idl_parser', diff --git a/src/mongo/db/pipeline/aggregate_command.idl b/src/mongo/db/pipeline/aggregate_command.idl index bb6cad08b1b..3aa63c40a76 100644 --- a/src/mongo/db/pipeline/aggregate_command.idl +++ b/src/mongo/db/pipeline/aggregate_command.idl @@ -100,6 +100,10 @@ commands: agg_stage: queryStats resource_pattern: cluster action_type: queryStatsRead + - privilege: # $queryStats + agg_stage: queryStats + resource_pattern: cluster + action_type: queryStatsReadTransformed - privilege: # $changeStream resource_pattern: exact_namespace action_type: changeStream diff --git a/src/mongo/db/pipeline/document_source_query_stats.cpp b/src/mongo/db/pipeline/document_source_query_stats.cpp index 0b53f72448d..01bd1a87d84 100644 --- a/src/mongo/db/pipeline/document_source_query_stats.cpp +++ b/src/mongo/db/pipeline/document_source_query_stats.cpp @@ -35,18 +35,18 @@ #include "mongo/util/assert_util.h" #include "mongo/util/debug_util.h" -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats namespace mongo { namespace { CounterMetric queryStatsHmacApplicationErrors("queryStats.numHmacApplicationErrors"); } -REGISTER_DOCUMENT_SOURCE_WITH_FEATURE_FLAG(queryStats, - DocumentSourceQueryStats::LiteParsed::parse, - DocumentSourceQueryStats::createFromBson, - AllowedWithApiStrict::kNeverInVersion1, - feature_flags::gFeatureFlagQueryStats); +// TODO SERVER-79494 Use REGISTER_DOCUMENT_SOURCE_WITH_FEATURE_FLAG +REGISTER_DOCUMENT_SOURCE(queryStats, + DocumentSourceQueryStats::LiteParsed::parse, + DocumentSourceQueryStats::createFromBson, + AllowedWithApiStrict::kNeverInVersion1); namespace { @@ -71,9 +71,12 @@ auto parseSpec(const BSONElement& spec, const Ctor& ctor) { if (transformIdentifiers) { algorithm = transformIdentifiers->getAlgorithm(); boost::optional<ConstDataRange> hmacKeyContainer = transformIdentifiers->getHmacKey(); - if (hmacKeyContainer) { - hmacKey = std::string(hmacKeyContainer->data(), (size_t)hmacKeyContainer->length()); - } + uassert(ErrorCodes::FailedToParse, + str::stream() << "The 'hmacKey' parameter of the $queryStats stage must be " + "specified when applying the hmac-sha-256 algorithm", + algorithm != TransformAlgorithmEnum::kHmacSha256 || + hmacKeyContainer != boost::none); + hmacKey = std::string(hmacKeyContainer->data(), (size_t)hmacKeyContainer->length()); } return ctor(algorithm, hmacKey); } @@ -97,6 +100,13 @@ std::vector<std::pair<size_t, QueryStatsEntry>> copyPartition( std::unique_ptr<DocumentSourceQueryStats::LiteParsed> DocumentSourceQueryStats::LiteParsed::parse( const NamespaceString& nss, const BSONElement& spec) { + // TODO SERVER-79494 Remove this manual feature flag check once we're registering doc source + // with REGISTER_DOCUMENT_SOURCE_WITH_FEATURE_FLAG + uassert(ErrorCodes::QueryFeatureNotAllowed, + "$queryStats is not allowed in the current configuration. You may need to enable the " + "correponding feature flag", + query_stats::isQueryStatsFeatureEnabled(/*requiresFullQueryStatsFeatureFlag*/ false)); + return parseSpec(spec, [&](TransformAlgorithmEnum algorithm, std::string hmacKey) { return std::make_unique<DocumentSourceQueryStats::LiteParsed>( spec.fieldName(), algorithm, hmacKey); @@ -105,12 +115,25 @@ std::unique_ptr<DocumentSourceQueryStats::LiteParsed> DocumentSourceQueryStats:: boost::intrusive_ptr<DocumentSource> DocumentSourceQueryStats::createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& pExpCtx) { + // TODO SERVER-79494 Remove this manual feature flag check once we're registering doc source + // with REGISTER_DOCUMENT_SOURCE_WITH_FEATURE_FLAG + uassert(ErrorCodes::QueryFeatureNotAllowed, + "$queryStats is not allowed in the current configuration. You may need to enable the " + "correponding feature flag", + query_stats::isQueryStatsFeatureEnabled(/*requiresFullQueryStatsFeatureFlag*/ false)); + const NamespaceString& nss = pExpCtx->ns; uassert(ErrorCodes::InvalidNamespace, "$queryStats must be run against the 'admin' database with {aggregate: 1}", nss.db() == DatabaseName::kAdmin.db() && nss.isCollectionlessAggregateNS()); + LOGV2_DEBUG_OPTIONS(7808300, + 1, + {logv2::LogTruncation::Disabled}, + "Logging invocation $queryStats", + "commandSpec"_attr = + spec.Obj().redact(BSONObj::RedactLevel::sensitiveOnly)); return parseSpec(spec, [&](TransformAlgorithmEnum algorithm, std::string hmacKey) { return new DocumentSourceQueryStats(pExpCtx, algorithm, hmacKey); }); @@ -145,12 +168,20 @@ DocumentSource::GetNextResult DocumentSourceQueryStats::doGetNext() { * The inner iterator iterates over a materialized container of all entries in the partition. * This is done to reduce the time under which the partition lock is held. */ + bool shouldLog = _algorithm != TransformAlgorithmEnum::kNone; while (true) { // First, attempt to exhaust all elements in the materialized partition. if (!_materializedPartition.empty()) { // Move out of the container reference. auto doc = std::move(_materializedPartition.front()); _materializedPartition.pop_front(); + if (shouldLog) { + LOGV2_DEBUG_OPTIONS(7808301, + 3, + {logv2::LogTruncation::Disabled}, + "Logging all outputs of $queryStats", + "thisOutput"_attr = doc); + } return {std::move(doc)}; } @@ -159,6 +190,12 @@ DocumentSource::GetNextResult DocumentSourceQueryStats::doGetNext() { // Materialized partition is exhausted, move to the next. _currentPartition++; if (_currentPartition >= _queryStatsStore.numPartitions()) { + if (shouldLog) { + LOGV2_DEBUG_OPTIONS(7808302, + 3, + {logv2::LogTruncation::Disabled}, + "Finished logging outout of $queryStats"); + } return DocumentSource::GetNextResult::makeEOF(); } diff --git a/src/mongo/db/pipeline/document_source_query_stats.h b/src/mongo/db/pipeline/document_source_query_stats.h index 5118a293cd2..da290d7c3e6 100644 --- a/src/mongo/db/pipeline/document_source_query_stats.h +++ b/src/mongo/db/pipeline/document_source_query_stats.h @@ -32,7 +32,7 @@ #include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/document_source_query_stats_gen.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" -#include "mongo/db/query/query_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/util/producer_consumer_queue.h" namespace mongo { @@ -59,8 +59,14 @@ public: PrivilegeVector requiredPrivileges(bool isMongos, bool bypassDocumentValidation) const override { - return {Privilege(ResourcePattern::forClusterResource(), ActionType::queryStatsRead)}; - ; + return _algorithm == TransformAlgorithmEnum::kNone + + ? PrivilegeVector{Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsReadTransformed), + Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsRead)} + : PrivilegeVector{Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsReadTransformed)}; } bool allowedToPassthroughFromMongos() const final { diff --git a/src/mongo/db/pipeline/document_source_query_stats.idl b/src/mongo/db/pipeline/document_source_query_stats.idl index 025e3e956d2..e7d18e578ad 100644 --- a/src/mongo/db/pipeline/document_source_query_stats.idl +++ b/src/mongo/db/pipeline/document_source_query_stats.idl @@ -31,7 +31,7 @@ global: - "mongo/db/pipeline/document_source_query_stats_validators.h" imports: - "mongo/db/basic_types.idl" - - "mongo/db/query/query_stats_transform_algorithm.idl" + - "mongo/db/query/query_stats/transform_algorithm.idl" structs: TransformIdentifiersSpec: diff --git a/src/mongo/db/pipeline/document_source_query_stats_test.cpp b/src/mongo/db/pipeline/document_source_query_stats_test.cpp index 94c5dddee35..0fc946f6a1b 100644 --- a/src/mongo/db/pipeline/document_source_query_stats_test.cpp +++ b/src/mongo/db/pipeline/document_source_query_stats_test.cpp @@ -34,6 +34,7 @@ #include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/document_source_query_stats.h" +#include "mongo/idl/server_parameter_test_util.h" #include "mongo/unittest/unittest.h" #include "mongo/util/str.h" @@ -54,6 +55,7 @@ public: }; TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfSpecIsNotObject) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( fromjson("{$queryStats: 1}").firstElement(), getExpCtx()), AssertionException, @@ -61,6 +63,7 @@ TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfSpecIsNotObject) { } TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunOnAdmin) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; getExpCtx()->ns = NamespaceString::makeCollectionlessAggregateNSS(DatabaseName(boost::none, "foo")); ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( @@ -70,6 +73,7 @@ TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunOnAdmin) { } TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunWithAggregateOne) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; getExpCtx()->ns = NamespaceString::createNamespaceString_forTest("admin.foo"); ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( fromjson("{$queryStats: {}}").firstElement(), getExpCtx()), @@ -78,6 +82,7 @@ TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunWithAggregateOne) } TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfUnrecognisedParameterSpecified) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( fromjson("{$queryStats: {foo: true}}").firstElement(), getExpCtx()), AssertionException, @@ -85,6 +90,7 @@ TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfUnrecognisedParameterSpe } TEST_F(DocumentSourceQueryStatsTest, ParseAndSerialize) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; const auto obj = fromjson("{$queryStats: {}}"); const auto doc = DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()); const auto queryStatsOp = static_cast<DocumentSourceQueryStats*>(doc.get()); @@ -99,6 +105,7 @@ TEST_F(DocumentSourceQueryStatsTest, ParseAndSerialize) { } TEST_F(DocumentSourceQueryStatsTest, ParseAndSerializeShouldIncludeHmacKey) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; const auto obj = fromjson(R"({ $queryStats: { transformIdentifiers: { @@ -130,6 +137,7 @@ TEST_F(DocumentSourceQueryStatsTest, ParseAndSerializeShouldIncludeHmacKey) { } TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfAlgorithmIsNotSupported) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; auto obj = fromjson(R"({ $queryStats: { transformIdentifiers: { @@ -144,6 +152,7 @@ TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfAlgorithmIsNotSupported) TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfTransformIdentifiersSpecifiedButEmptyAlgorithm) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; auto obj = fromjson(R"({ $queryStats: { transformIdentifiers: { @@ -158,6 +167,7 @@ TEST_F(DocumentSourceQueryStatsTest, TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfTransformIdentifiersSpecifiedButNoAlgorithm) { + RAIIServerParameterControllerForTest queryStatsFeatureFlag{"featureFlagQueryStats", true}; auto obj = fromjson(R"({ $queryStats: { transformIdentifiers: { diff --git a/src/mongo/db/pipeline/document_source_query_stats_validators.cpp b/src/mongo/db/pipeline/document_source_query_stats_validators.cpp index 8a4980c6297..8a3f234e325 100644 --- a/src/mongo/db/pipeline/document_source_query_stats_validators.cpp +++ b/src/mongo/db/pipeline/document_source_query_stats_validators.cpp @@ -30,7 +30,7 @@ #include "mongo/platform/basic.h" #include "mongo/db/pipeline/document_source_query_stats_validators.h" -#include "mongo/db/query/query_stats_transform_algorithm_gen.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" #include "mongo/util/str.h" #include <set> diff --git a/src/mongo/db/pipeline/document_source_query_stats_validators.h b/src/mongo/db/pipeline/document_source_query_stats_validators.h index 412eff10cee..077552da0b3 100644 --- a/src/mongo/db/pipeline/document_source_query_stats_validators.h +++ b/src/mongo/db/pipeline/document_source_query_stats_validators.h @@ -30,7 +30,7 @@ #pragma once #include "mongo/base/status.h" -#include "mongo/db/query/query_stats_transform_algorithm_gen.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" namespace mongo { /** diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index a66dcde06a3..920fcb5dd34 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -14,6 +14,7 @@ env.SConscript( 'cost_model', 'datetime', 'optimizer', + 'query_stats', 'stats', ], exports=[ @@ -275,8 +276,8 @@ env.Library( 'framework_control.cpp', 'query_feature_flags.idl', 'query_knobs.idl', + 'query_stats/util.cpp', 'sbe_plan_cache_on_parameter_change.cpp', - 'query_stats_util.cpp', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/commands/test_commands_enabled', @@ -364,23 +365,11 @@ env.Library( ) env.Library( - target='rate_limiting', - source=[ - 'rate_limiting.cpp', - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/util/clock_sources', - ], -) - -env.Library( target='query_shape', source=[ 'query_shape.cpp', - 'query_shape.idl', - 'query_stats_find_key_generator.cpp', - 'query_stats_aggregate_key_generator.cpp', + 'query_stats/find_key_generator.cpp', + 'query_stats/aggregate_key_generator.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', @@ -390,44 +379,12 @@ env.Library( 'canonical_query', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/query_shape_common', 'projection_ast', 'sort_pattern', ], ) -env.Library(target='query_stats_parse', source=['query_stats_transform_algorithm.idl'], LIBDEPS=[ - '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/idl/idl_parser', -]) - -env.Library( - target='op_metrics', - source=['query_stats.cpp'], - LIBDEPS=[ - '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/mutable/mutable_bson', - '$BUILD_DIR/mongo/db/commands', - '$BUILD_DIR/mongo/db/profile_filter', - '$BUILD_DIR/mongo/db/server_options', - '$BUILD_DIR/mongo/db/service_context', - '$BUILD_DIR/mongo/db/stats/counters', - '$BUILD_DIR/mongo/db/storage/storage_engine_parameters', - '$BUILD_DIR/mongo/rpc/client_metadata', - '$BUILD_DIR/mongo/util/fail_point', - '$BUILD_DIR/mongo/util/net/network', - '$BUILD_DIR/mongo/util/processinfo', - 'command_request_response', - 'memory_util', - 'query_knobs', - 'query_stats_parse', - 'rate_limiting', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/auth/auth', - '$BUILD_DIR/mongo/util/namespace_string_database_name_util', - ], -) - env.CppUnitTest( target="db_query_test", source=[ @@ -483,9 +440,7 @@ env.CppUnitTest( "query_settings_test.cpp", "query_shape_test.cpp", "query_shape_test.idl", - "query_stats_store_test.cpp", "query_solution_test.cpp", - "rate_limiting_test.cpp", "sbe_and_hash_test.cpp", "sbe_and_sorted_test.cpp", "sbe_shard_filter_test.cpp", @@ -529,7 +484,6 @@ env.CppUnitTest( "query_request", "query_shape", "query_test_service_context", - "rate_limiting", ], ) diff --git a/src/mongo/db/query/lru_key_value.h b/src/mongo/db/query/lru_key_value.h index 3bb26e95f20..2786e1c40ab 100644 --- a/src/mongo/db/query/lru_key_value.h +++ b/src/mongo/db/query/lru_key_value.h @@ -39,18 +39,42 @@ namespace mongo { /** + * 'InsertionEvictionListener' class to use with 'LRUBudgetTracker' that will always noop. + */ +class NoopInsertionEvictionListener { +public: + // Called when a key-value pair is being inserted. Parameters are the key-value pair and its + // estimated size. + template <class K, class V> + void onInsert(const K&, const V&, size_t) {} + + // Called when a key-value pair is being evicted. Parameters are the key-value pair and its + // estimated size. + template <class K, class V> + void onEvict(const K&, const V&, size_t) {} + + // Called when the cache is being cleared. Parameter is the estimated size of the key-value + // pairs in the cache before it was cleared. + void onClear(size_t) {} +}; + +/** * This class tracks a size of entries in 'LRUKeyValue'. * The size can be understood as a number of the entries, an amount of memory they occupied, * or any other value defined by the template parameter 'Estimator'. * The 'Estimator' must be deterministic and always return the same value for the same entry. + * The 'InsertionEvictionListener' will be called on every insertion and eviction as well as when + * the cache is cleared. */ -template <class K, class V, typename Estimator> +template <class K, class V, typename Estimator, typename InsertionEvictionListener> class LRUBudgetTracker { public: LRUBudgetTracker(size_t maxBudget) : _max(maxBudget), _current(0) {} void onAdd(const K& k, const V& v) { - _current += _estimator(k, v); + size_t budget = _estimator(k, v); + _current += budget; + _listener.onInsert(k, v, budget); } void onRemove(const K& k, const V& v) { @@ -60,9 +84,11 @@ public: "LRU budget underflow: current={}, budget={} "_format(_current, budget), _current >= budget); _current -= budget; + _listener.onEvict(k, v, budget); } void onClear() { + _listener.onClear(_current); _current = 0; } @@ -83,6 +109,7 @@ private: size_t _max; size_t _current; Estimator _estimator; + InsertionEvictionListener _listener; }; /** @@ -90,6 +117,9 @@ private: * policy. The size allowed in the kv-store is controlled by 'LRUBudgetTracker' * set in the constructor. * + * An 'InsertionEvictionListener' may optionally be specified to track the insertion and eviction of + * each key-value pair. + * * Caveat: * This kv-store is NOT thread safe! The client to this utility is responsible * for protecting concurrent access to the LRU store if used in a threaded @@ -104,6 +134,7 @@ private: template <class K, class V, class KeyValueBudgetEstimator, + class InsertionEvictionListener = NoopInsertionEvictionListener, class KeyHasher = std::hash<K>, class Eq = std::equal_to<K>> class LRUKeyValue { @@ -273,7 +304,7 @@ private: return nEvicted; } - LRUBudgetTracker<K, V, KeyValueBudgetEstimator> _budgetTracker; + LRUBudgetTracker<K, V, KeyValueBudgetEstimator, InsertionEvictionListener> _budgetTracker; // (K, V) pairs are stored in this std::list. They are sorted in order of use, where the front // is the most recently used and the back is the least recently used. diff --git a/src/mongo/db/query/lru_key_value_test.cpp b/src/mongo/db/query/lru_key_value_test.cpp index 8d767021543..6dcdfc5a4ea 100644 --- a/src/mongo/db/query/lru_key_value_test.cpp +++ b/src/mongo/db/query/lru_key_value_test.cpp @@ -95,8 +95,40 @@ struct NonTrivialBudgetEstimator { using NonTrivialTestSharedPtrValue = LRUKeyValue<size_t, std::shared_ptr<NonTrivialEntry>, NonTrivialBudgetEstimator>; -template <typename Key, typename Value, typename Estimator> -void assertInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key, Value value) { +class NonTrivialInsertionEvictionListener { +public: + NonTrivialInsertionEvictionListener() { + keyTotal = 0; + valueTotal = 0; + budgetTotal = 0; + } + + void onInsert(const int& k, const ValueType& v, size_t budget) { + keyTotal += k; + valueTotal += v.val; + budgetTotal += budget; + } + + void onEvict(const int& k, const ValueType& v, size_t budget) { + keyTotal -= k; + valueTotal -= v.val; + budgetTotal -= budget; + } + + void onClear(size_t budget) { + budgetTotal -= budget; + } + + static size_t keyTotal; + static size_t valueTotal; + static size_t budgetTotal; +}; +size_t NonTrivialInsertionEvictionListener::keyTotal; +size_t NonTrivialInsertionEvictionListener::valueTotal; +size_t NonTrivialInsertionEvictionListener::budgetTotal; + +template <typename Key, typename Value, typename Estimator, typename Listener> +void assertInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key key, Value value) { ASSERT_TRUE(cache.hasKey(key)); auto s = cache.get(key); ASSERT(s.isOK()); @@ -105,8 +137,8 @@ void assertInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key, Value v ASSERT_EQUALS(*(kvItr->second), *value); } -template <typename Key, typename Value, typename Estimator> -void assertNotInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key) { +template <typename Key, typename Value, typename Estimator, typename Listener> +void assertNotInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key key) { ASSERT_FALSE(cache.hasKey(key)); auto s = cache.get(key); ASSERT(!s.isOK()); @@ -356,7 +388,8 @@ TEST(LRUKeyValueTest, UniquePtrKeyValue) { assertNotInKVStore(cacheForEviction, 1); // The entry with key '1' has been Evicted. } -using TestScalarValue = LRUKeyValue<int, ValueType, TrivialBudgetEstimator>; +using TestScalarValue = + LRUKeyValue<int, ValueType, TrivialBudgetEstimator, NonTrivialInsertionEvictionListener>; void assertValueInKVStore(TestScalarValue& cache, int key, ValueType value) { ASSERT_TRUE(cache.hasKey(key)); @@ -373,9 +406,17 @@ TEST(LRUKeyValueTest, ScalarKeyValue) { assertValueInKVStore(cache, 1, ValueType{2}); assertNotInKVStore(cache, 3); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 1); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 2); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 1); + cache.add(1, ValueType{3}); assertValueInKVStore(cache, 1, ValueType{3}); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 1); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 3); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 1); + // Test eviction. TestScalarValue cacheForEviction{2}; cacheForEviction.add(1, ValueType{1}); @@ -384,6 +425,18 @@ TEST(LRUKeyValueTest, ScalarKeyValue) { ASSERT_EQUALS(cacheForEviction.size(), static_cast<size_t>(2)); assertNotInKVStore(cacheForEviction, 1); // The entry with key '1' has been Evicted. + + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 5); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 5); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 2); + + // Clear the remaining values. + cacheForEviction.clear(); + + assertNotInKVStore(cacheForEviction, 2); // The entry with key '2' has been Evicted. + assertNotInKVStore(cacheForEviction, 3); // The entry with key '3' has been Evicted. + + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 0); } } // namespace diff --git a/src/mongo/db/query/partitioned_cache.h b/src/mongo/db/query/partitioned_cache.h index 87f622373d2..ca10f731b71 100644 --- a/src/mongo/db/query/partitioned_cache.h +++ b/src/mongo/db/query/partitioned_cache.h @@ -45,6 +45,7 @@ template <class KeyType, class ValueType, class KeyBudgetEstimator, class Partitioner, + class InsertionEvictionListener, class KeyHasher = std::hash<KeyType>, class Eq = std::equal_to<KeyType>> class PartitionedCache { @@ -53,12 +54,49 @@ private: PartitionedCache& operator=(const PartitionedCache&) = delete; public: - using Lru = LRUKeyValue<KeyType, ValueType, KeyBudgetEstimator, KeyHasher, Eq>; + using Lru = LRUKeyValue<KeyType, + ValueType, + KeyBudgetEstimator, + InsertionEvictionListener, + KeyHasher, + Eq>; using Partition = typename Partitioned<Lru, Partitioner>::OnePartition; using PartitionId = typename Partitioned<Lru, Partitioner>::PartitionId; /** * Initialize plan cache with the total cache size in bytes and number of partitions. + * + * Important edge cases to consider include: + * + * 1. Adding an entry that is larger than the max partition size to a non-empty partition. + * + * This will evict both entries. This is because entries are evicted from the partition in + * order of least recently used. Thus, the oldest, small entry will be evicted first but the + * partition will still be over budget with the new, too-large entry so it will be evicted as + * well. + * + * 2. Adding a queryStats store entry that is smaller than the overall cache size but larger + * than single partition max size. + * + * It is not possible to write entries to the cache that are larger than a single + * partition's max size, even if it is smaller than the entire cache max size. This is because + * the cache's budget is configured/regulated on the partition level (cacheSize / + * numPartitions). This makes sense as each entry is written to a specific partition, but might + * not be immediately obvious so worthy to highlight. + * + * 3. Too few partitions can cause unnecessary evictions + * + * Every class that implements the PartitionedCache template provides a partitioner() that + * returns the id of the partition to which to write the entry. In existing implementations, + * partitioner() returns the remainder after dividing the entry's key hash by numPartitions. In + * the case where we have only two partitions, every odd key hash will be written to the first + * partition (and vice versa). In this way, it can quickly be the case that one partition + * fills up completely but the partitioner() call keeps returning the already full partition and + * the cache evict old entries from it to put the new one in. At the end of all the write + * operations, the cache is below it's budget (as the second partition is only partially full) + * but we don't have all the entries we expect. It is therefore important to have sufficient + * enough number of partitions so the entries can be more equally dispersed to avoid unnecessary + * evictions. */ explicit PartitionedCache(size_t cacheSize, size_t numPartitions) : _numPartitions(numPartitions) { diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h index 04b2cf118f9..6f7ca15145a 100644 --- a/src/mongo/db/query/plan_cache.h +++ b/src/mongo/db/query/plan_cache.h @@ -327,6 +327,7 @@ class PlanCacheBase std::shared_ptr<const PlanCacheEntryBase<CachedPlanType, DebugInfoType>>, KeyBudgetEstimator, Partitioner, + NoopInsertionEvictionListener, KeyHasher> { private: PlanCacheBase(const PlanCacheBase&) = delete; @@ -338,6 +339,7 @@ public: std::shared_ptr<const PlanCacheEntryBase<CachedPlanType, DebugInfoType>>, KeyBudgetEstimator, Partitioner, + NoopInsertionEvictionListener, KeyHasher>; using Entry = PlanCacheEntryBase<CachedPlanType, DebugInfoType>; diff --git a/src/mongo/db/query/query_feature_flags.idl b/src/mongo/db/query/query_feature_flags.idl index f726b0bcf4a..0b701e5a311 100644 --- a/src/mongo/db/query/query_feature_flags.idl +++ b/src/mongo/db/query/query_feature_flags.idl @@ -82,7 +82,7 @@ feature_flags: default: false featureFlagQueryStats: - description: "Feature flag for enabling the queryStats store." + description: "Feature flag for enabling full queryStats collection." cpp_varname: gFeatureFlagQueryStats default: false @@ -117,6 +117,11 @@ feature_flags: default: true version: 7.0 + featureFlagQueryStatsFindCommand: + description: "Feature flag for enabling queryStats collection for the find command." + cpp_varname: gFeatureFlagQueryStatsFindCommand + default: false + featureFlagSearchBatchSizeLimit: description: "Feature flag to enable the search batchsize and limit optimization." cpp_varname: gFeatureFlagSearchBatchSizeLimit diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index 4cc34456274..330462550e5 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -36,7 +36,7 @@ global: - "mongo/db/query/ce_mode_parameter.h" - "mongo/db/query/explain_version_validator.h" - "mongo/db/query/sbe_plan_cache_on_parameter_change.h" - - "mongo/db/query/query_stats_util.h" + - "mongo/db/query/query_stats/util.h" - "mongo/platform/atomic_proxy.h" - "mongo/platform/atomic_word.h" diff --git a/src/mongo/db/query/query_shape.cpp b/src/mongo/db/query/query_shape.cpp index cb94a23490f..90860f3eb23 100644 --- a/src/mongo/db/query/query_shape.cpp +++ b/src/mongo/db/query/query_shape.cpp @@ -34,6 +34,7 @@ #include "mongo/db/query/projection_ast_util.h" #include "mongo/db/query/query_request_helper.h" #include "mongo/db/query/query_shape_gen.h" +#include "mongo/db/query/shape_helpers.h" #include "mongo/db/query/sort_pattern.h" namespace mongo::query_shape { @@ -100,7 +101,6 @@ BSONObj extractSortShape(const BSONObj& sortSpec, return bob.obj(); } -static std::string hintSpecialField = "$hint"; void addShapeLiterals(BSONObjBuilder* bob, const FindCommandRequest& findCommand, const SerializationOptions& opts) { @@ -144,34 +144,6 @@ void addRemainingFindCommandFields(BSONObjBuilder* bob, } } -BSONObj extractHintShape(BSONObj obj, const SerializationOptions& opts, bool preserveValue) { - BSONObjBuilder bob; - for (BSONElement elem : obj) { - if (hintSpecialField.compare(elem.fieldName()) == 0) { - if (elem.type() == BSONType::String) { - bob.append(hintSpecialField, opts.serializeFieldPathFromString(elem.String())); - } else if (elem.type() == BSONType::Object) { - opts.appendLiteral(&bob, hintSpecialField, elem.Obj()); - } else { - uasserted(ErrorCodes::FailedToParse, "$hint must be a string or an object"); - } - continue; - } - - // $natural doesn't need to be redacted. - if (elem.fieldNameStringData().compare(query_request_helper::kNaturalSortField) == 0) { - bob.append(elem); - continue; - } - - if (preserveValue) { - bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldName())); - } else { - opts.appendLiteral(&bob, opts.serializeFieldPathFromString(elem.fieldName()), elem); - } - } - return bob.obj(); -} /** * In a let specification all field names are variable names, and all values are either expressions @@ -195,20 +167,10 @@ void appendCmdNs(BSONObjBuilder& bob, const NamespaceString& nss, const SerializationOptions& opts) { BSONObjBuilder nsObj = bob.subobjStart("cmdNs"); - appendNamespaceShape(nsObj, nss, opts); + shape_helpers::appendNamespaceShape(nsObj, nss, opts); nsObj.doneFast(); } -void appendNamespaceShape(BSONObjBuilder& bob, - const NamespaceString& nss, - const SerializationOptions& opts) { - if (nss.tenantId()) { - bob.append("tenantId", opts.serializeIdentifier(nss.tenantId().value().toString())); - } - bob.append("db", opts.serializeIdentifier(nss.db())); - bob.append("coll", opts.serializeIdentifier(nss.coll())); -} - BSONObj extractQueryShape(const ParsedFindCommand& findRequest, const SerializationOptions& opts, const boost::intrusive_ptr<ExpressionContext>& expCtx) { @@ -231,7 +193,6 @@ BSONObj extractQueryShape(const ParsedFindCommand& findRequest, std::unique_ptr<MatchExpression> filterExpr; // Filter. bob.append(FindCommandRequest::kFilterFieldName, findRequest.filter->serialize(opts)); - // Let Spec. if (auto letSpec = findCmd.getLet()) { auto redactedObj = extractLetSpecShape(letSpec.get(), opts, expCtx); @@ -244,21 +205,13 @@ BSONObj extractQueryShape(const ParsedFindCommand& findRequest, projection_ast::serialize(*findRequest.proj->root(), opts)); } - // Assume the hint is correct and contains field names. It is possible that this hint - // doesn't actually represent an index, but we can't detect that here. - // Hint, max, and min won't serialize if the object is empty. - if (!findCmd.getHint().isEmpty()) { - bob.append(FindCommandRequest::kHintFieldName, - extractHintShape(findCmd.getHint(), opts, true)); - // Max/Min aren't valid without hint. - if (!findCmd.getMax().isEmpty()) { - bob.append(FindCommandRequest::kMaxFieldName, - extractHintShape(findCmd.getMax(), opts, false)); - } - if (!findCmd.getMin().isEmpty()) { - bob.append(FindCommandRequest::kMinFieldName, - extractHintShape(findCmd.getMin(), opts, false)); - } + if (!findCmd.getMax().isEmpty()) { + bob.append(FindCommandRequest::kMaxFieldName, + shape_helpers::extractMinOrMaxShape(findCmd.getMax(), opts)); + } + if (!findCmd.getMin().isEmpty()) { + bob.append(FindCommandRequest::kMinFieldName, + shape_helpers::extractMinOrMaxShape(findCmd.getMin(), opts)); } // Sort. @@ -316,12 +269,6 @@ BSONObj extractQueryShape(const AggregateCommandRequest& aggregateCommand, bob.append(AggregateCommandRequest::kCollationFieldName, param.get()); } - // hint - if (auto hint = aggregateCommand.getHint()) { - bob.append(AggregateCommandRequest::kHintFieldName, - extractHintShape(hint.get(), opts, true)); - } - // let if (auto letSpec = aggregateCommand.getLet()) { auto redactedObj = extractLetSpecShape(letSpec.get(), opts, expCtx); @@ -331,26 +278,6 @@ BSONObj extractQueryShape(const AggregateCommandRequest& aggregateCommand, return bob.obj(); } -NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt) { - tassert(7632900, "cmdNs must be an object.", cmdNsElt.type() == BSONType::Object); - auto cmdNs = CommandNamespace::parse(IDLParserContext("cmdNs"), cmdNsElt.embeddedObject()); - - boost::optional<TenantId> tenantId = cmdNs.getTenantId().map(TenantId::parseFromString); - - if (cmdNs.getColl().has_value()) { - tassert(7632903, - "Exactly one of 'uuid' and 'coll' can be defined.", - !cmdNs.getUuid().has_value()); - return NamespaceString(cmdNs.getDb(), cmdNs.getColl().value()); - } else { - tassert(7632904, - "Exactly one of 'uuid' and 'coll' can be defined.", - !cmdNs.getColl().has_value()); - UUID uuid = uassertStatusOK(UUID::parse(cmdNs.getUuid().value().toString())); - return NamespaceStringOrUUID(cmdNs.getDb().toString(), uuid, tenantId); - } -} - QueryShapeHash hash(const BSONObj& queryShape) { return QueryShapeHash::computeHash(reinterpret_cast<const uint8_t*>(queryShape.objdata()), queryShape.objsize()); diff --git a/src/mongo/db/query/query_shape.h b/src/mongo/db/query/query_shape.h index 9415b7387ce..ca699019523 100644 --- a/src/mongo/db/query/query_shape.h +++ b/src/mongo/db/query/query_shape.h @@ -37,6 +37,7 @@ namespace mongo::query_shape { using QueryShapeHash = SHA256Block; + /** * Computes a BSONObj that is meant to be used to classify queries according to their shape, for the * purposes of collecting queryStats. @@ -73,10 +74,5 @@ BSONObj extractQueryShape(const AggregateCommandRequest& aggregateCommand, const boost::intrusive_ptr<ExpressionContext>& expCtx, const NamespaceString& nss); -NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt); -void appendNamespaceShape(BSONObjBuilder& bob, - const NamespaceString& nss, - const SerializationOptions& opts); - QueryShapeHash hash(const BSONObj& queryShape); } // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape_test.cpp b/src/mongo/db/query/query_shape_test.cpp index 059495e9797..b6b5097f3e6 100644 --- a/src/mongo/db/query/query_shape_test.cpp +++ b/src/mongo/db/query/query_shape_test.cpp @@ -35,6 +35,7 @@ #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/query_shape.h" #include "mongo/db/query/query_shape_test_gen.h" +#include "mongo/db/query/shape_helpers.h" #include "mongo/unittest/bson_test_util.h" #include "mongo/unittest/unittest.h" @@ -204,13 +205,16 @@ TEST(QueryPredicateShape, Exists) { } TEST(QueryPredicateShape, In) { - // Any number of children is always the same shape + // Any number of children in any order is always the same shape ASSERT_SHAPE_EQ_AUTO( // NOLINT R"({"a":{"$in":"?array<?number>"}})", "{a: {$in: [1]}}"); ASSERT_SHAPE_EQ_AUTO( // NOLINT R"({"a":{"$in":"?array<>"}})", "{a: {$in: [1, 4, 'str', /regex/]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<>"}})", + "{a: {$in: ['str', /regex/, 1, 4]}}"); } TEST(QueryPredicateShape, BitTestOperators) { diff --git a/src/mongo/db/query/query_stats/README.md b/src/mongo/db/query/query_stats/README.md new file mode 100644 index 00000000000..c6e9a164f6c --- /dev/null +++ b/src/mongo/db/query/query_stats/README.md @@ -0,0 +1,28 @@ +# Query Stats +This directory is the home of the infrastructure related to recording query statistics for the +database. It is not to be confused with src/mongo/db/query/stats/ which is the home of the logic for +computing and maintaining statistics about a collection or index's data distribution - for use by +the query planner. + +At the center of everything here is the QueryStatsStore, which essentially maps query shapes to some +metrics about how often each one occurs. For example, if the client does this: +```js +db.example.findOne({x: 24}); +db.example.findOne({x: 53}); +``` +then the QueryStatsStore should contain an entry for a single query shape which would record 2 +executions and some related statistics (see the QueryStatsEntry for details). + +The query stats store actually uses _more_ dimensions (i.e. more granularity) to group incoming +queries than just the query shape. For example, these queries would all three have the same shape +but the first would have a different query stats store entry from the other two: +```js +db.example.find({x: 55}); +db.example.find({x: 55}).batchSize(2); +db.example.find({x: 55}).batchSize(3); +``` +There are two distinct query stats store entries here - both the examples which include the batch +size will be treated separately from the example which does not specify a batch size. + +The dimensions considered will depend on the command, but can generally be found in the KeyGenerator +iterface, which will generate the QueryStatsStore keys by which we accumulate statistics. diff --git a/src/mongo/db/query/query_stats/SConscript b/src/mongo/db/query/query_stats/SConscript new file mode 100644 index 00000000000..0a478733d7b --- /dev/null +++ b/src/mongo/db/query/query_stats/SConscript @@ -0,0 +1,102 @@ +# -*- mode: python -*- + +Import([ + "env", + "get_option", +]) + +env = env.Clone() + +env.Library( + target='rate_limiting', + source=[ + 'rate_limiting.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/util/clock_sources', + ], +) + +env.Library(target='query_stats_parse', source=['transform_algorithm.idl'], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/idl/idl_parser', +]) + +env.Library( + target='query_stats', + source=['query_stats.cpp', 'query_stats_entry.cpp'], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/mutable/mutable_bson', + '$BUILD_DIR/mongo/db/commands', + '$BUILD_DIR/mongo/db/profile_filter', + '$BUILD_DIR/mongo/db/query/command_request_response', + '$BUILD_DIR/mongo/db/query/memory_util', + '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/server_options', + '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/stats/counters', + '$BUILD_DIR/mongo/db/storage/storage_engine_parameters', + '$BUILD_DIR/mongo/rpc/client_metadata', + '$BUILD_DIR/mongo/util/fail_point', + '$BUILD_DIR/mongo/util/net/network', + '$BUILD_DIR/mongo/util/processinfo', + 'query_stats_parse', + 'rate_limiting', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/auth', + '$BUILD_DIR/mongo/util/namespace_string_database_name_util', + ], +) + +env.CppUnitTest( + target="db_query_query_stats_test", + source=[ + "query_stats_store_test.cpp", + "rate_limiting_test.cpp", + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/db/auth/authmocks", + "$BUILD_DIR/mongo/db/query/query_shape", + "$BUILD_DIR/mongo/db/query/query_test_service_context", + "$BUILD_DIR/mongo/db/service_context_d_test_fixture", + "query_stats", + "rate_limiting", + ], +) + +env.Benchmark( + target='rate_limiting_bm', + source=[ + 'rate_limiting_bm.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/unittest/unittest', + '$BUILD_DIR/mongo/util/processinfo', + 'rate_limiting', + ], +) + +env.Benchmark( + target='shapifying_bm', + source=[ + 'shapifying_bm.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/auth/auth', + '$BUILD_DIR/mongo/db/pipeline/pipeline', + '$BUILD_DIR/mongo/db/query/canonical_query', + '$BUILD_DIR/mongo/db/query/query_shape', + '$BUILD_DIR/mongo/db/query/query_test_service_context', + '$BUILD_DIR/mongo/db/server_base', + '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/rpc/client_metadata', + '$BUILD_DIR/mongo/unittest/unittest', + '$BUILD_DIR/mongo/util/processinfo', + 'query_stats', + ], +) diff --git a/src/mongo/db/query/query_stats_aggregate_key_generator.cpp b/src/mongo/db/query/query_stats/aggregate_key_generator.cpp index cd340991406..f2b501b39bb 100644 --- a/src/mongo/db/query/query_stats_aggregate_key_generator.cpp +++ b/src/mongo/db/query/query_stats/aggregate_key_generator.cpp @@ -27,11 +27,12 @@ * it in the license file. */ -#include "mongo/db/query/query_stats_aggregate_key_generator.h" +#include "mongo/db/query/query_stats/aggregate_key_generator.h" #include "mongo/db/pipeline/pipeline.h" #include "mongo/db/query/query_shape.h" #include "mongo/db/query/serialization_options.h" +#include "mongo/db/query/shape_helpers.h" namespace mongo::query_stats { @@ -81,7 +82,7 @@ void AggregateKeyGenerator::appendCommandSpecificComponents( BSONArrayBuilder otherNss = bob.subarrayStart(kOtherNssFieldName); for (const auto& nss : _involvedNamespaces) { BSONObjBuilder otherNsEntryBob = otherNss.subobjStart(); - query_shape::appendNamespaceShape(otherNsEntryBob, nss, opts); + shape_helpers::appendNamespaceShape(otherNsEntryBob, nss, opts); otherNsEntryBob.doneFast(); } otherNss.doneFast(); diff --git a/src/mongo/db/query/query_stats_aggregate_key_generator.h b/src/mongo/db/query/query_stats/aggregate_key_generator.h index 619f8a126a3..ab6aa0ef688 100644 --- a/src/mongo/db/query/query_stats_aggregate_key_generator.h +++ b/src/mongo/db/query/query_stats/aggregate_key_generator.h @@ -32,7 +32,7 @@ #include "mongo/db/collection_type.h" #include "mongo/db/pipeline/aggregate_command_gen.h" #include "mongo/db/pipeline/pipeline.h" -#include "mongo/db/query/query_stats_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" namespace mongo::query_stats { @@ -51,15 +51,17 @@ public: const boost::intrusive_ptr<ExpressionContext>& expCtx, stdx::unordered_set<NamespaceString> involvedNamespaces, const NamespaceString& origNss, - query_shape::CollectionType collectionType = query_shape::CollectionType::unknown) + query_shape::CollectionType collectionType = query_shape::CollectionType ::kUnknown) : KeyGenerator( expCtx->opCtx, // TODO: SERVER-76330 Store representative agg query shape in telemetry store. BSONObj(), + request.getHint(), collectionType), _request(std::move(request)), _involvedNamespaces(std::move(involvedNamespaces)), _origNss(origNss), + _inMongos(expCtx->inMongos), _initialQueryStatsKey(_makeQueryStatsKeyHelper( SerializationOptions::kDebugQueryShapeSerializeOptions, expCtx, pipeline)) { _queryShapeHash = query_shape::hash(*_initialQueryStatsKey); @@ -104,6 +106,7 @@ private: // per a given query. expCtx->stopExpressionCounters(); expCtx->addResolvedNamespaces(_involvedNamespaces); + expCtx->inMongos = _inMongos; return expCtx; } @@ -118,6 +121,10 @@ private: // The original NSS of the request before views are resolved. const NamespaceString _origNss; + // Flag to denote if the query was run on mongos. Needed to rebuild the "dummy" expression + // context for re-parsing. + bool _inMongos; + // This is computed and cached upon construction until asked for once - at which point this // transitions to boost::none. This both a performance and a memory optimization. // diff --git a/src/mongo/db/query/query_stats/aggregated_metric.h b/src/mongo/db/query/query_stats/aggregated_metric.h new file mode 100644 index 00000000000..fca94c3183e --- /dev/null +++ b/src/mongo/db/query/query_stats/aggregated_metric.h @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <algorithm> +#include <cstdint> + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" + +namespace mongo::query_stats { + +/** + * An aggregated metric stores a compressed view of data. It balances the loss of information + * with the reduction in required storage. + */ +struct AggregatedMetric { + + /** + * Aggregate an observed value into the metric. + */ + void aggregate(uint64_t val) { + sum += val; + max = std::max(val, max); + min = std::min(val, min); + sumOfSquares += val * val; + } + + void appendTo(BSONObjBuilder& builder, const StringData& fieldName) const { + BSONObjBuilder metricsBuilder = builder.subobjStart(fieldName); + metricsBuilder.append("sum", (long long)sum); + metricsBuilder.append("max", (long long)max); + metricsBuilder.append("min", (long long)min); + metricsBuilder.append("sumOfSquares", (long long)sumOfSquares); + metricsBuilder.done(); + } + + uint64_t sum = 0; + // Default to the _signed_ maximum (which fits in unsigned range) because we cast to + // BSONNumeric when serializing. + uint64_t min = (uint64_t)std::numeric_limits<int64_t>::max; + uint64_t max = 0; + + /** + * The sum of squares along with (an externally stored) count will allow us to compute the + * variance/stddev. + */ + uint64_t sumOfSquares = 0; +}; + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats_find_key_generator.cpp b/src/mongo/db/query/query_stats/find_key_generator.cpp index 86c9555c59a..0d4fc25ac36 100644 --- a/src/mongo/db/query/query_stats_find_key_generator.cpp +++ b/src/mongo/db/query/query_stats/find_key_generator.cpp @@ -27,7 +27,7 @@ * it in the license file. */ -#include "mongo/db/query/query_stats_find_key_generator.h" +#include "mongo/db/query/query_stats/find_key_generator.h" #include "mongo/db/matcher/extensions_callback_real.h" #include "mongo/db/namespace_string.h" @@ -45,7 +45,7 @@ std::unique_ptr<FindCommandRequest> FindKeyGenerator::reparse(OperationContext* // TODO: SERVER-76330 factor out building the parseable cmdObj into a helper function in // query_shape.h. BSONObjBuilder cmdBuilder; - NamespaceStringOrUUID nss = query_shape::parseNamespaceShape(_parseableQueryShape["cmdNs"]); + NamespaceStringOrUUID nss = shape_helpers::parseNamespaceShape(_parseableQueryShape["cmdNs"]); nss.serialize(&cmdBuilder, FindCommandRequest::kCommandName); cmdBuilder.append("$db", nss.dbname()); diff --git a/src/mongo/db/query/query_stats_find_key_generator.h b/src/mongo/db/query/query_stats/find_key_generator.h index 6175e67ae52..ca7207e95fb 100644 --- a/src/mongo/db/query/query_stats_find_key_generator.h +++ b/src/mongo/db/query/query_stats/find_key_generator.h @@ -35,7 +35,7 @@ #include "mongo/db/query/find_command.h" #include "mongo/db/query/find_command_gen.h" #include "mongo/db/query/parsed_find_command.h" -#include "mongo/db/query/query_stats_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" namespace mongo::query_stats { @@ -45,8 +45,11 @@ public: const boost::intrusive_ptr<ExpressionContext>& expCtx, const ParsedFindCommand& request, BSONObj parseableQueryShape, - query_shape::CollectionType collectionType = query_shape::CollectionType::unknown) - : KeyGenerator(expCtx->opCtx, parseableQueryShape, collectionType), + query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown) + : KeyGenerator(expCtx->opCtx, + parseableQueryShape, + request.findCommandRequest->getHint(), + collectionType), _readConcern(request.findCommandRequest->getReadConcern().has_value() ? request.findCommandRequest->getReadConcern()->copy() : BSONObj()), diff --git a/src/mongo/db/query/query_stats_key_generator.h b/src/mongo/db/query/query_stats/key_generator.h index c3e25fafce7..6b109e89d01 100644 --- a/src/mongo/db/query/query_stats_key_generator.h +++ b/src/mongo/db/query/query_stats/key_generator.h @@ -37,6 +37,7 @@ #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/query/query_shape.h" #include "mongo/db/query/serialization_options.h" +#include "mongo/db/query/shape_helpers.h" #include "mongo/rpc/metadata/client_metadata.h" namespace mongo { @@ -89,7 +90,7 @@ public: _parseableQueryShape.objsize() + /* _collectionType is not owned here */ (_apiParams ? sizeof(*_apiParams) + optionalSize(_apiParams->getAPIVersion()) : 0) + (_hasField.clientMetaData ? _clientMetaData.objsize() : 0) + _commentObj.objsize() + - (_hasField.readPreference ? _readPreference.objsize() : 0); + (_hasField.readPreference ? _readPreference.objsize() : 0) + _hintObj.objsize(); } BSONObj getRepresentativeQueryShapeForDebug() const { @@ -99,9 +100,11 @@ public: protected: KeyGenerator(OperationContext* opCtx, BSONObj parseableQueryShape, - query_shape::CollectionType collectionType = query_shape::CollectionType::unknown, + boost::optional<BSONObj> hint, + query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown, boost::optional<query_shape::QueryShapeHash> queryShapeHash = boost::none) : _parseableQueryShape(parseableQueryShape.getOwned()), + _hintObj(hint.value_or(BSONObj())), _queryShapeHash(queryShapeHash.value_or(query_shape::hash(parseableQueryShape))), _collectionType(collectionType) { if (auto metadata = ClientMetadata::get(opCtx->getClient())) { @@ -179,10 +182,13 @@ protected: if (_hasField.clientMetaData) { bob.append("client", _clientMetaData); } - if (_collectionType > query_shape::CollectionType::unknown && - _collectionType < query_shape::CollectionType::end) { + if (_collectionType != query_shape::CollectionType::kUnknown) { bob.append("collectionType", toStringData(_collectionType)); } + + if (!_hintObj.isEmpty()) { + bob.append("hint", shape_helpers::extractHintShape(_hintObj, opts)); + } } /** @@ -199,12 +205,17 @@ protected: // minimize the struct's size as much as possible. BSONObj _parseableQueryShape; - // Preserve this value in the query shape. + // Preserve this value. BSONObj _clientMetaData; // Shapify this value. BSONObj _commentObj; - // Preserve this value in the query shape. + // Preserve this value. BSONObj _readPreference; + // Preserve this value. Possibly empty. + // In the future a hint may not be part of every single type of request, but it is possibly + // set on a find, aggregate, distinct, and update, so this is going to be a "common" element + // for a while. It may not make sense on an insert request. + BSONObj _hintObj; // Separate the possibly-enormous BSONObj from the remaining members @@ -230,7 +241,7 @@ protected: // This static assert checks to ensure that the struct's size is changed thoughtfully. If adding // or otherwise changing the members, this assert may be updated with care. static_assert( - sizeof(KeyGenerator) <= 4 * sizeof(BSONObj) + sizeof(BSONElement) + + sizeof(KeyGenerator) <= 5 * sizeof(BSONObj) + sizeof(BSONElement) + 2 * sizeof(std::unique_ptr<APIParameters>) + sizeof(query_shape::CollectionType) + sizeof(query_shape::QueryShapeHash) + sizeof(int64_t), "Size of KeyGenerator is too large! " diff --git a/src/mongo/db/query/query_stats.cpp b/src/mongo/db/query/query_stats/query_stats.cpp index 3e94d415cf7..f36edce4b6c 100644 --- a/src/mongo/db/query/query_stats.cpp +++ b/src/mongo/db/query/query_stats/query_stats.cpp @@ -27,10 +27,10 @@ * it in the license file. */ -#include "mongo/db/query/query_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" + #include "mongo/crypto/hash_block.h" -#include "mongo/crypto/sha256_block.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/locker.h" #include "mongo/db/curop.h" @@ -45,8 +45,7 @@ #include "mongo/db/query/query_feature_flags_gen.h" #include "mongo/db/query/query_planner_params.h" #include "mongo/db/query/query_request_helper.h" -#include "mongo/db/query/query_stats_util.h" -#include "mongo/db/query/rate_limiting.h" +#include "mongo/db/query/query_stats/util.h" #include "mongo/db/query/serialization_options.h" #include "mongo/db/query/sort_pattern.h" #include "mongo/logv2/log.h" @@ -59,11 +58,7 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery -namespace mongo { - -namespace query_stats { - -CounterMetric queryStatsStoreSizeEstimateBytesMetric("queryStats.queryStatsStoreSizeEstimateBytes"); +namespace mongo::query_stats { namespace { @@ -98,59 +93,6 @@ size_t getQueryStatsStoreSize() { return capQueryStatsStoreSize(requestedSize); } -/** - * A manager for the queryStats store allows a "pointer swap" on the queryStats store itself. The - * usage patterns are as follows: - * - * - Updating the queryStats store uses the `getQueryStatsStore()` method. The queryStats store - * instance is obtained, entries are looked up and mutated, or created anew. - * - The queryStats store is "reset". This involves atomically allocating a new instance, once - * there are no more updaters (readers of the store "pointer"), and returning the existing - * instance. - */ -class QueryStatsStoreManager { -public: - template <typename... QueryStatsStoreArgs> - QueryStatsStoreManager(size_t cacheSize, size_t numPartitions) - : _queryStatsStore(std::make_unique<QueryStatsStore>(cacheSize, numPartitions)), - _maxSize(cacheSize) {} - - /** - * Acquire the instance of the queryStats store. - */ - QueryStatsStore& getQueryStatsStore() { - return *_queryStatsStore; - } - - size_t getMaxSize() { - return _maxSize; - } - - /** - * Resize the queryStats store and return the number of evicted - * entries. - */ - size_t resetSize(size_t cacheSize) { - _maxSize = cacheSize; - return _queryStatsStore->reset(cacheSize); - } - -private: - std::unique_ptr<QueryStatsStore> _queryStatsStore; - - /** - * Max size of the queryStats store. Tracked here to avoid having to recompute after it's - * divided up into partitions. - */ - size_t _maxSize; -}; - -const auto queryStatsStoreDecoration = - ServiceContext::declareDecoration<std::unique_ptr<QueryStatsStoreManager>>(); - -const auto queryStatsRateLimiter = - ServiceContext::declareDecoration<std::unique_ptr<RateLimiting>>(); - class TelemetryOnParamChangeUpdaterImpl final : public query_stats_util::OnParamChangeUpdater { public: void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final { @@ -171,7 +113,9 @@ ServiceContext::ConstructorActionRegisterer queryStatsStoreManagerRegisterer{ // It is possible that this is called before FCV is properly set up. Setting up the store if // the flag is enabled but FCV is incorrect is safe, and guards against the FCV being // changed to a supported version later. - if (!feature_flags::gFeatureFlagQueryStats.isEnabledAndIgnoreFCVUnsafeAtStartup()) { + if (!feature_flags::gFeatureFlagQueryStats.isEnabledAndIgnoreFCVUnsafeAtStartup() && + !feature_flags::gFeatureFlagQueryStatsFindCommand + .isEnabledAndIgnoreFCVUnsafeAtStartup()) { // featureFlags are not allowed to be changed at runtime. Therefore it's not an issue // to not create a queryStats store in ConstructorActionRegisterer at start up with the // flag off - because the flag can not be turned on at any point afterwards. @@ -184,7 +128,6 @@ ServiceContext::ConstructorActionRegisterer queryStatsStoreManagerRegisterer{ std::make_unique<TelemetryOnParamChangeUpdaterImpl>(); size_t size = getQueryStatsStoreSize(); auto&& globalQueryStatsStoreManager = queryStatsStoreDecoration(serviceCtx); - // Initially the queryStats store used the same number of partitions as the plan cache, that // is the number of cpu cores. However, with performance investigation we found that when // the size of the partitions was too large, it took too long to copy out and read one @@ -207,21 +150,20 @@ ServiceContext::ConstructorActionRegisterer queryStatsStoreManagerRegisterer{ std::make_unique<QueryStatsStoreManager>(size, numPartitions); auto configuredSamplingRate = internalQueryStatsRateLimit.load(); queryStatsRateLimiter(serviceCtx) = std::make_unique<RateLimiting>( - configuredSamplingRate < 0 ? INT_MAX : configuredSamplingRate); + configuredSamplingRate < 0 ? INT_MAX : configuredSamplingRate, Seconds{1}); }}; /** * Top-level checks for whether queryStats collection is enabled. If this returns false, we must go * no further. + * TODO SERVER-79494 Remove requiresFullQueryStatsFeatureFlag parameter. */ -bool isQueryStatsEnabled(const ServiceContext* serviceCtx) { - // During initialization FCV may not yet be setup but queries could be run. We can't +bool isQueryStatsEnabled(const ServiceContext* serviceCtx, bool requiresFullQueryStatsFeatureFlag) { + // During initialization, FCV may not yet be setup but queries could be run. We can't // check whether queryStats should be enabled without FCV, so default to not recording // those queries. // TODO SERVER-75935 Remove FCV Check. - const auto fcvSnapshot = serverGlobalParams.featureCompatibility.acquireFCVSnapshot(); - return fcvSnapshot.isVersionInitialized() && - feature_flags::gFeatureFlagQueryStats.isEnabled(fcvSnapshot) && + return isQueryStatsFeatureEnabled(requiresFullQueryStatsFeatureFlag) && queryStatsStoreDecoration(serviceCtx)->getMaxSize() > 0; } @@ -230,10 +172,6 @@ bool isQueryStatsEnabled(const ServiceContext* serviceCtx) { * configuration for a global on/off decision and, if enabled, delegates to the rate limiter. */ bool shouldCollect(const ServiceContext* serviceCtx) { - // Quick escape if queryStats is turned off. - if (!isQueryStatsEnabled(serviceCtx)) { - return false; - } // Cannot collect queryStats if sampling rate is not greater than 0. Note that we do not // increment queryStatsRateLimitedRequestsMetric here since queryStats is entirely disabled. auto samplingRate = queryStatsRateLimiter(serviceCtx)->getSamplingRate(); @@ -249,33 +187,29 @@ bool shouldCollect(const ServiceContext* serviceCtx) { return true; } -std::string sha256HmacStringDataHasher(std::string key, const StringData& sd) { - auto hashed = SHA256Block::computeHmac( - (const uint8_t*)key.data(), key.size(), (const uint8_t*)sd.rawData(), sd.size()); - return hashed.toString(); -} - -std::size_t hash(const BSONObj& obj) { - return absl::hash_internal::CityHash64(obj.objdata(), obj.objsize()); -} - } // namespace -BSONObj QueryStatsEntry::computeQueryStatsKey(OperationContext* opCtx, - TransformAlgorithmEnum algorithm, - std::string hmacKey) const { - return keyGenerator->generate( - opCtx, - algorithm == TransformAlgorithmEnum::kHmacSha256 - ? boost::optional<SerializationOptions::TokenizeIdentifierFunc>( - [&](StringData sd) { return sha256HmacStringDataHasher(hmacKey, sd); }) - : boost::none); +/** + * Indicates whether or not query stats is enabled via the feature flags. If + * requiresFullQueryStatsFeatureFlag is true, it will only return true if featureFlagQueryStats is + * enabled. Otherwise, it will return true if either featureFlagQueryStats or + * featureFlagQueryStatsFindCommand is enabled. + * + * TODO SERVER-79494 Remove this function and collapse feature flag check into isQueryStatsEnabled. + */ +bool isQueryStatsFeatureEnabled(bool requiresFullQueryStatsFeatureFlag) { + const auto fcvSnapshot = serverGlobalParams.featureCompatibility.acquireFCVSnapshot(); + return fcvSnapshot.isVersionInitialized() && + (feature_flags::gFeatureFlagQueryStats.isEnabled(fcvSnapshot) || + (!requiresFullQueryStatsFeatureFlag && + feature_flags::gFeatureFlagQueryStatsFindCommand.isEnabled(fcvSnapshot))); } void registerRequest(OperationContext* opCtx, const NamespaceString& collection, - std::function<std::unique_ptr<KeyGenerator>(void)> makeKeyGenerator) { - if (!isQueryStatsEnabled(opCtx->getServiceContext())) { + std::function<std::unique_ptr<KeyGenerator>(void)> makeKeyGenerator, + bool requiresFullQueryStatsFeatureFlag) { + if (!isQueryStatsEnabled(opCtx->getServiceContext(), requiresFullQueryStatsFeatureFlag)) { return; } @@ -298,16 +232,30 @@ void registerRequest(OperationContext* opCtx, "collection"_attr = collection); return; } - - opDebug.queryStatsKeyGenerator = makeKeyGenerator(); + // There are a few cases where a query shape can be larger than the original query. For example, + // {$exists: false} in the input query serializes to {$not: {$exists: true}. In rare cases where + // an input query has thousands of clauses, the cumulative bloat that shapification adds results + // in a BSON object that exceeds the 16 MB memory limit. In these cases, we want to exclude the + // original query from queryStats metrics collection and let it execute normally. + try { + opDebug.queryStatsKeyGenerator = makeKeyGenerator(); + } catch (ExceptionFor<ErrorCodes::BSONObjectTooLarge>&) { + LOGV2_DEBUG(7979400, + 1, + "Query Stats shapification has exceeded the 16 MB memory limit. Metrics will " + "not be collected "); + queryStatsStoreWriteErrorsMetric.increment(); + return; + } opDebug.queryStatsStoreKeyHash = opDebug.queryStatsKeyGenerator->hash(); } QueryStatsStore& getQueryStatsStore(OperationContext* opCtx) { uassert(6579000, - "Telemetry is not enabled without the feature flag on and a cache size greater than 0 " - "bytes", - isQueryStatsEnabled(opCtx->getServiceContext())); + "Query stats is not enabled without the feature flag on and a cache size greater than " + "0 bytes", + isQueryStatsEnabled(opCtx->getServiceContext(), + /*requiresFullQueryStatsFeatureFlag*/ false)); return queryStatsStoreDecoration(opCtx->getServiceContext())->getQueryStatsStore(); } @@ -321,11 +269,12 @@ void writeQueryStats(OperationContext* opCtx, return; } - // It's possible that telemetry was enabled in registerRequest but has been disabled since + // It's possible that query stats was enabled in registerRequest but has been disabled since // (e.g., by FCV downgrade or setting the store size to 0). Rather than calling - // getTelemetryStore (which would trigger a uassert if telemetry is disabled), we return and - // log a message if telemetry is disabled, and otherwise grab the telemetry store directly. - if (!isQueryStatsEnabled(opCtx->getServiceContext())) { + // getQueryStatsStore (which would trigger a uassert if telemetry is disabled), we return and + // log a message if query stats is disabled, and otherwise grab the query stats store directly. + if (!isQueryStatsEnabled(opCtx->getServiceContext(), + /*requiresFullQueryStatsFeatureFlag*/ false)) { LOGV2_DEBUG(8456700, 2, "Query stats was enabled when the command started but is now disabled. " @@ -371,5 +320,4 @@ void writeQueryStats(OperationContext* opCtx, metrics->firstResponseExecMicros.aggregate(firstResponseExecMicros); metrics->docsReturned.aggregate(docsReturned); } -} // namespace query_stats -} // namespace mongo +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats.h b/src/mongo/db/query/query_stats/query_stats.h index fc4a6448ca0..90c942d9b3e 100644 --- a/src/mongo/db/query/query_stats.h +++ b/src/mongo/db/query/query_stats/query_stats.h @@ -36,168 +36,115 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/query/partitioned_cache.h" #include "mongo/db/query/plan_explainer.h" -#include "mongo/db/query/query_stats_key_generator.h" -#include "mongo/db/query/query_stats_transform_algorithm_gen.h" -#include "mongo/db/query/util/memory_util.h" +#include "mongo/db/query/query_stats/key_generator.h" +#include "mongo/db/query/query_stats/query_stats_entry.h" +#include "mongo/db/query/query_stats/rate_limiting.h" #include "mongo/db/service_context.h" #include "mongo/db/views/view.h" #include <cstdint> #include <memory> -namespace mongo { +namespace mongo::query_stats { -namespace { -/** - * Type we use to render values to BSON. - */ -using BSONNumeric = long long; -} // namespace - -namespace query_stats { -/** - * An aggregated metric stores a compressed view of data. It balances the loss of information - * with the reduction in required storage. - */ -struct AggregatedMetric { - - /** - * Aggregate an observed value into the metric. - */ - void aggregate(uint64_t val) { - sum += val; - max = std::max(val, max); - min = std::min(val, min); - sumOfSquares += val * val; +struct QueryStatsPartitioner { + // The partitioning function for use with the 'Partitioned' utility. + std::size_t operator()(const std::size_t k, const std::size_t nPartitions) const { + return k % nPartitions; } +}; - void appendTo(BSONObjBuilder& builder, const StringData& fieldName) const { - BSONObjBuilder metricsBuilder = builder.subobjStart(fieldName); - metricsBuilder.append("sum", (BSONNumeric)sum); - metricsBuilder.append("max", (BSONNumeric)max); - metricsBuilder.append("min", (BSONNumeric)min); - metricsBuilder.append("sumOfSquares", (BSONNumeric)sumOfSquares); - metricsBuilder.done(); +struct QueryStatsStoreEntryBudgetor { + size_t operator()(const std::size_t key, const std::shared_ptr<QueryStatsEntry>& value) { + return sizeof(decltype(key)) + value->size(); } - - uint64_t sum = 0; - // Default to the _signed_ maximum (which fits in unsigned range) because we cast to - // BSONNumeric when serializing. - uint64_t min = (uint64_t)std::numeric_limits<int64_t>::max; - uint64_t max = 0; - - /** - * The sum of squares along with (an externally stored) count will allow us to compute the - * variance/stddev. - */ - uint64_t sumOfSquares = 0; }; -extern CounterMetric queryStatsStoreSizeEstimateBytesMetric; -const auto kKeySize = sizeof(std::size_t); -// Used to aggregate the metrics for one query stats key over all its executions. -class QueryStatsEntry { -public: - QueryStatsEntry(std::unique_ptr<KeyGenerator> keyGenerator) - : firstSeenTimestamp(Date_t::now()), keyGenerator(std::move(keyGenerator)) { - // Increment by size of query stats store key (hash returns size_t) and value - // (QueryStatsEntry) - queryStatsStoreSizeEstimateBytesMetric.increment(kKeySize + size()); +/* + * 'QueryStatsStore insertion and eviction listener implementation. This class adjusts the + * 'queryStatsStoreSize' serverStatus metric when entries are inserted or evicted. + */ +struct QueryStatsStoreInsertionEvictionListener { + void onInsert(const std::size_t&, + const std::shared_ptr<QueryStatsEntry>&, + size_t estimatedSize) { + queryStatsStoreSizeEstimateBytesMetric.increment(estimatedSize); } - ~QueryStatsEntry() { - // Decrement by size of query stats store key (hash returns size_t) and value - // (QueryStatsEntry) - queryStatsStoreSizeEstimateBytesMetric.decrement(kKeySize + size()); + void onEvict(const std::size_t&, + const std::shared_ptr<QueryStatsEntry>&, + size_t estimatedSize) { + queryStatsStoreSizeEstimateBytesMetric.decrement(estimatedSize); } - BSONObj toBSON() const { - BSONObjBuilder builder{sizeof(QueryStatsEntry) + 100}; - builder.append("lastExecutionMicros", (BSONNumeric)lastExecutionMicros); - builder.append("execCount", (BSONNumeric)execCount); - totalExecMicros.appendTo(builder, "totalExecMicros"); - firstResponseExecMicros.appendTo(builder, "firstResponseExecMicros"); - docsReturned.appendTo(builder, "docsReturned"); - builder.append("firstSeenTimestamp", firstSeenTimestamp); - builder.append("latestSeenTimestamp", latestSeenTimestamp); - return builder.obj(); + void onClear(size_t estimatedSize) { + queryStatsStoreSizeEstimateBytesMetric.decrement(estimatedSize); } +}; +using QueryStatsStore = PartitionedCache<std::size_t, + std::shared_ptr<QueryStatsEntry>, + QueryStatsStoreEntryBudgetor, + QueryStatsPartitioner, + QueryStatsStoreInsertionEvictionListener>; - int64_t size() { - return sizeof(*this) + (keyGenerator ? keyGenerator->size() : 0); - } +/** + * A manager for the queryStats store allows a "pointer swap" on the queryStats store itself. The + * usage patterns are as follows: + * + * - Updating the queryStats store uses the `getQueryStatsStore()` method. The queryStats store + * instance is obtained, entries are looked up and mutated, or created anew. + * - The queryStats store is "reset". This involves atomically allocating a new instance, once + * there are no more updaters (readers of the store "pointer"), and returning the existing + * instance. + */ +class QueryStatsStoreManager { +public: + template <typename... QueryStatsStoreArgs> + QueryStatsStoreManager(size_t cacheSize, size_t numPartitions) + : _queryStatsStore(std::make_unique<QueryStatsStore>(cacheSize, numPartitions)), + _maxSize(cacheSize) {} /** - * Generate the queryStats key for this entry's request. If algorithm is not - * TransformAlgorithm::kNone, any identifying information (field names, namespace) will be - * anonymized. + * Acquire the instance of the queryStats store. */ - BSONObj computeQueryStatsKey(OperationContext* opCtx, - TransformAlgorithmEnum algorithm, - std::string hmacKey) const; - - BSONObj getRepresentativeQueryShapeForDebug() const { - return keyGenerator->getRepresentativeQueryShapeForDebug(); + QueryStatsStore& getQueryStatsStore() { + return *_queryStatsStore; } - /** - * Timestamp for when this query shape was added to the store. Set on construction. - */ - const Date_t firstSeenTimestamp; - - /** - * Timestamp for when the latest time this query shape was seen. - */ - Date_t latestSeenTimestamp; - - /** - * Last execution time in microseconds. - */ - uint64_t lastExecutionMicros = 0; - - /** - * Number of query executions. - */ - uint64_t execCount = 0; - - /** - * Aggregates the total time for execution including getMore requests. - */ - AggregatedMetric totalExecMicros; + size_t getMaxSize() { + return _maxSize; + } /** - * Aggregates the time for execution for first batch only. + * Resize the queryStats store and return the number of evicted + * entries. */ - AggregatedMetric firstResponseExecMicros; + size_t resetSize(size_t cacheSize) { + _maxSize = cacheSize; + return _queryStatsStore->reset(cacheSize); + } - AggregatedMetric docsReturned; +private: + std::unique_ptr<QueryStatsStore> _queryStatsStore; /** - * The KeyGenerator that can generate the query stats key for this request. + * Max size of the queryStats store. Tracked here to avoid having to recompute after it's + * divided up into partitions. */ - const std::shared_ptr<const KeyGenerator> keyGenerator; -}; -struct TelemetryPartitioner { - // The partitioning function for use with the 'Partitioned' utility. - std::size_t operator()(const std::size_t k, const std::size_t nPartitions) const { - return k % nPartitions; - } + size_t _maxSize; }; -struct QueryStatsStoreEntryBudgetor { - size_t operator()(const std::size_t key, const std::shared_ptr<QueryStatsEntry>& value) { - return sizeof(decltype(key)) + value->size(); - } -}; -using QueryStatsStore = PartitionedCache<std::size_t, - std::shared_ptr<QueryStatsEntry>, - QueryStatsStoreEntryBudgetor, - TelemetryPartitioner>; +const auto queryStatsStoreDecoration = + ServiceContext::declareDecoration<std::unique_ptr<QueryStatsStoreManager>>(); +const auto queryStatsRateLimiter = + ServiceContext::declareDecoration<std::unique_ptr<RateLimiting>>(); /** * Acquire a reference to the global queryStats store. */ QueryStatsStore& getQueryStatsStore(OperationContext* opCtx); +bool isQueryStatsFeatureEnabled(bool requiresFullQueryStatsFeatureFlag); + /** * Registers a request for query stats collection. The function may decide not to collect anything, * so this should be called for all requests. The decision is made based on the feature flag and @@ -230,10 +177,15 @@ QueryStatsStore& getQueryStatsStore(OperationContext* opCtx); * deferred construction callback to ensure that this feature does not impact performance if * collecting stats is not needed due to the feature being disabled or the request being rate * limited. + * - Since we currently have 2 feature flags (one for full query stats, and one for + * find-command-only query stats), we use the requiresFullQueryStatsFeatureFlag parameter to + * denote which requests should only be registered when the full feature flag is enabled. TODO + * SERVER-79494 Remove requiresFullQueryStatsFeatureFlag parameter. */ void registerRequest(OperationContext* opCtx, const NamespaceString& collection, - std::function<std::unique_ptr<KeyGenerator>(void)> makeKeyGenerator); + std::function<std::unique_ptr<KeyGenerator>(void)> makeKeyGenerator, + bool requiresFullQueryStatsFeatureFlag = true); /** * Writes query stats to the query stats store for the operation identified by `queryStatsKeyHash`. @@ -251,5 +203,4 @@ void writeQueryStats(OperationContext* opCtx, uint64_t queryExecMicros, uint64_t firstResponseExecMicros, uint64_t docsReturned); -} // namespace query_stats -} // namespace mongo +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_entry.cpp b/src/mongo/db/query/query_stats/query_stats_entry.cpp new file mode 100644 index 00000000000..922d0572f10 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_entry.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/query_stats_entry.h" + +#include <boost/optional.hpp> + +#include "mongo/crypto/hash_block.h" +#include "mongo/crypto/sha256_block.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +namespace mongo::query_stats { + +CounterMetric queryStatsStoreSizeEstimateBytesMetric("queryStats.queryStatsStoreSizeEstimateBytes"); + +namespace { + +std::string sha256HmacStringDataHasher(std::string key, const StringData& sd) { + auto hashed = SHA256Block::computeHmac( + (const uint8_t*)key.data(), key.size(), (const uint8_t*)sd.rawData(), sd.size()); + return hashed.toString(); +} + +} // namespace + +BSONObj QueryStatsEntry::computeQueryStatsKey(OperationContext* opCtx, + TransformAlgorithmEnum algorithm, + std::string hmacKey) const { + return keyGenerator->generate( + opCtx, + algorithm == TransformAlgorithmEnum::kHmacSha256 + ? boost::optional<SerializationOptions::TokenizeIdentifierFunc>( + [&](StringData sd) { return sha256HmacStringDataHasher(hmacKey, sd); }) + : boost::none); +} + +BSONObj QueryStatsEntry::toBSON() const { + BSONObjBuilder builder{sizeof(QueryStatsEntry) + 100}; + builder.append("lastExecutionMicros", (long long)lastExecutionMicros); + builder.append("execCount", (long long)execCount); + totalExecMicros.appendTo(builder, "totalExecMicros"); + firstResponseExecMicros.appendTo(builder, "firstResponseExecMicros"); + docsReturned.appendTo(builder, "docsReturned"); + builder.append("firstSeenTimestamp", firstSeenTimestamp); + builder.append("latestSeenTimestamp", latestSeenTimestamp); + return builder.obj(); +} + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_entry.h b/src/mongo/db/query/query_stats/query_stats_entry.h new file mode 100644 index 00000000000..714ed864070 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_entry.h @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <algorithm> +#include <cstdint> +#include <memory> + +#include "mongo/db/commands/server_status_metric.h" +#include "mongo/db/query/query_stats/aggregated_metric.h" +#include "mongo/db/query/query_stats/key_generator.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" +#include "mongo/util/time_support.h" + +namespace mongo::query_stats { + +extern CounterMetric queryStatsStoreSizeEstimateBytesMetric; + +const auto kKeySize = sizeof(std::size_t); + +/** + * The value stored in the query stats store. It contains a KeyGenerator representing this "kind" of + * query, and some metrics about that shape. This class is responsible for knowing its size and + * updating our server status metrics about the size of the query stats store accordingly. At the + * time of this writing, the LRUCache utility does not easily expose its size in a way we could use + * as server status metrics. + */ +class QueryStatsEntry { +public: + QueryStatsEntry(std::unique_ptr<KeyGenerator> keyGenerator) + : firstSeenTimestamp(Date_t::now()), keyGenerator(std::move(keyGenerator)) {} + + BSONObj toBSON() const; + + int64_t size() { + return sizeof(*this) + (keyGenerator ? keyGenerator->size() : 0); + } + + /** + * Generate the queryStats key for this entry's request. If algorithm is not + * TransformAlgorithm::kNone, any identifying information (field names, namespace) will be + * anonymized. + */ + BSONObj computeQueryStatsKey(OperationContext* opCtx, + TransformAlgorithmEnum algorithm, + std::string hmacKey) const; + + BSONObj getRepresentativeQueryShapeForDebug() const { + return keyGenerator->getRepresentativeQueryShapeForDebug(); + } + + /** + * Timestamp for when this query shape was added to the store. Set on construction. + */ + const Date_t firstSeenTimestamp; + + /** + * Timestamp for when the latest time this query shape was seen. + */ + Date_t latestSeenTimestamp; + + /** + * Last execution time in microseconds. + */ + uint64_t lastExecutionMicros = 0; + + /** + * Number of query executions. + */ + uint64_t execCount = 0; + + /** + * Aggregates the total time for execution including getMore requests. + */ + AggregatedMetric totalExecMicros; + + /** + * Aggregates the time for execution for first batch only. + */ + AggregatedMetric firstResponseExecMicros; + + AggregatedMetric docsReturned; + + /** + * The KeyGenerator that can generate the query stats key for this request. + */ + const std::shared_ptr<const KeyGenerator> keyGenerator; +}; + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats_store_test.cpp b/src/mongo/db/query/query_stats/query_stats_store_test.cpp index cbc92445950..43d791efac8 100644 --- a/src/mongo/db/query/query_stats_store_test.cpp +++ b/src/mongo/db/query/query_stats/query_stats_store_test.cpp @@ -33,9 +33,10 @@ #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/query_feature_flags_gen.h" #include "mongo/db/query/query_shape.h" -#include "mongo/db/query/query_stats.h" -#include "mongo/db/query/query_stats_aggregate_key_generator.h" -#include "mongo/db/query/query_stats_find_key_generator.h" +#include "mongo/db/query/query_stats/aggregate_key_generator.h" +#include "mongo/db/query/query_stats/find_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/idl/server_parameter_test_util.h" #include "mongo/unittest/inline_auto_update.h" @@ -55,7 +56,7 @@ std::size_t hash(const BSONObj& obj) { class QueryStatsStoreTest : public ServiceContextTest { public: - static constexpr auto collectionType = query_shape::CollectionType::collection; + static constexpr auto collectionType = query_shape::CollectionType::kCollection; BSONObj makeQueryStatsKeyFindRequest(const FindCommandRequest& fcr, const boost::intrusive_ptr<ExpressionContext>& expCtx, bool applyHmac) { @@ -94,19 +95,19 @@ public: }; TEST_F(QueryStatsStoreTest, BasicUsage) { - QueryStatsStore telStore{5000000, 1000}; + QueryStatsStore queryStatsStore{5000000, 1000}; auto getMetrics = [&](const BSONObj& key) { - auto lookupResult = telStore.lookup(hash(key)); + auto lookupResult = queryStatsStore.lookup(hash(key)); return *lookupResult.getValue(); }; auto collectMetrics = [&](BSONObj& key) { std::shared_ptr<QueryStatsEntry> metrics; - auto lookupResult = telStore.lookup(hash(key)); + auto lookupResult = queryStatsStore.lookup(hash(key)); if (!lookupResult.isOK()) { - telStore.put(hash(key), std::make_shared<QueryStatsEntry>(nullptr)); - lookupResult = telStore.lookup(hash(key)); + queryStatsStore.put(hash(key), std::make_shared<QueryStatsEntry>(nullptr)); + lookupResult = queryStatsStore.lookup(hash(key)); } metrics = *lookupResult.getValue(); metrics->execCount += 1; @@ -128,7 +129,7 @@ TEST_F(QueryStatsStoreTest, BasicUsage) { ASSERT_EQ(getMetrics(query2)->execCount, 1); auto collectMetricsWithLock = [&](BSONObj& key) { - auto [lookupResult, lock] = telStore.getWithPartitionLock(hash(key)); + auto [lookupResult, lock] = queryStatsStore.getWithPartitionLock(hash(key)); auto metrics = *lookupResult.getValue(); metrics->execCount += 1; metrics->lastExecutionMicros += 123456; @@ -143,31 +144,154 @@ TEST_F(QueryStatsStoreTest, BasicUsage) { int numKeys = 0; - telStore.forEach( + queryStatsStore.forEach( [&](std::size_t key, const std::shared_ptr<QueryStatsEntry>& entry) { numKeys++; }); ASSERT_EQ(numKeys, 2); } +TEST_F(QueryStatsStoreTest, EvictionTest) { + // This creates a queryStats store with a single partition to specifically test the eviction + // behavior with very large queries. + const auto cacheSize = 500; + const auto numPartitions = 1; + QueryStatsStore queryStatsStore{cacheSize, numPartitions}; -TEST_F(QueryStatsStoreTest, EvictEntries) { - // This creates a queryStats store with 2 partitions, each with a size of 1200 bytes. - const auto cacheSize = 2400; - const auto numPartitions = 2; - QueryStatsStore telStore{cacheSize, numPartitions}; - - for (int i = 0; i < 30; i++) { - auto query = BSON("query" + std::to_string(i) << 1 << "xEquals" << 42); - telStore.put(hash(query), std::make_shared<QueryStatsEntry>(nullptr)); - } + // Add an entry that is smaller than the max partition size. + auto query = BSON("query" << 1 << "xEquals" << 42); + queryStatsStore.put(hash(query), std::make_shared<QueryStatsEntry>(nullptr)); int numKeys = 0; - telStore.forEach( + queryStatsStore.forEach( [&](std::size_t key, const std::shared_ptr<QueryStatsEntry>& entry) { numKeys++; }); + ASSERT_EQ(numKeys, 1); + // Add an entry that is larger than the max partition size to the non-empty partition. This + // should evict both entries, the first small entry written to the partition and the current too + // large entry we wish to write to the partition. The reason is because entries are evicted from + // the partition in order of least recently used. Thus, the small entry will be evicted first + // but the partition will still be over budget so the final, too large entry will also be + // evicted. + auto opCtx = makeOperationContext(); + auto fcr = std::make_unique<FindCommandRequest>( + NamespaceStringOrUUID(NamespaceString::createNamespaceString_forTest("testDB.testColl"))); + fcr->setLet(BSON("var" << 2)); + fcr->setFilter(fromjson("{$expr: [{$eq: ['$a', '$$var']}]}")); + fcr->setProjection(fromjson("{varIs: '$$var'}")); + fcr->setLimit(5); + fcr->setSkip(2); + fcr->setBatchSize(25); + fcr->setMaxTimeMS(1000); + fcr->setNoCursorTimeout(false); + opCtx->setComment(BSON("comment" + << " foo")); + fcr->setSingleBatch(false); + fcr->setAllowDiskUse(false); + fcr->setAllowPartialResults(true); + fcr->setAllowDiskUse(false); + fcr->setShowRecordId(true); + fcr->setMirrored(true); + fcr->setHint(BSON("z" << 1 << "c" << 1)); + fcr->setMax(BSON("z" << 25)); + fcr->setMin(BSON("z" << 80)); + fcr->setSort(BSON("sortVal" << 1 << "otherSort" << -1)); - int entriesPerPartition = - (cacheSize / numPartitions) / (sizeof(std::size_t) + sizeof(QueryStatsEntry)); + auto&& [expCtx, parsedFind] = + uassertStatusOK(parsed_find_command::parse(opCtx.get(), std::move(fcr))); + auto queryShape = query_shape::extractQueryShape( + *parsedFind, SerializationOptions::kRepresentativeQueryShapeSerializeOptions, expCtx); + QueryStatsEntry testMetrics{std::make_unique<query_stats::FindKeyGenerator>( + expCtx, *parsedFind, queryShape, collectionType)}; + queryStatsStore.put(hash(testMetrics.computeQueryStatsKey( + opCtx.get(), TransformAlgorithmEnum::kNone, std::string{})), + std::make_shared<QueryStatsEntry>(testMetrics)); + numKeys = 0; + queryStatsStore.forEach( + [&](std::size_t key, const std::shared_ptr<QueryStatsEntry>& entry) { numKeys++; }); + ASSERT_EQ(numKeys, 0); - ASSERT_EQ(numKeys, entriesPerPartition * numPartitions); + // This creates a queryStats store where each partition has a max size of 500 bytes. + QueryStatsStore queryStatsStoreTwo{/*cacheSize*/ 1500, /*numPartitions*/ 3}; + // Adding a queryStats store entry that is smaller than the overal cache size but larger than a + // single partition max size, will cause an eviction. testMetrics is larger than 500 bytes and + // thus over budget for the partitions of this cache. + queryStatsStoreTwo.put(hash(testMetrics.computeQueryStatsKey( + opCtx.get(), TransformAlgorithmEnum::kNone, std::string{})), + std::make_shared<QueryStatsEntry>(testMetrics)); + queryStatsStoreTwo.forEach( + [&](std::size_t key, const std::shared_ptr<QueryStatsEntry>& entry) { numKeys++; }); + ASSERT_EQ(numKeys, 0); +} + +TEST_F(QueryStatsStoreTest, GenerateMaxBsonSizeQueryShape) { + const NamespaceString nss = NamespaceString::createNamespaceString_forTest("testDB.testColl"); + FindCommandRequest fcr((NamespaceStringOrUUID(nss))); + // This creates a query that is just below the 16 MB memory limit. + int limit = 225500; + BSONObjBuilder bob; + BSONArrayBuilder andBob(bob.subarrayStart("$and")); + for (int i = 1; i <= limit; i++) { + BSONObjBuilder childrenBob; + childrenBob.append("x", BSON("$lt" << i << "$gte" << i)); + andBob.append(childrenBob.obj()); + } + andBob.doneFast(); + fcr.setFilter(bob.obj()); + auto fcrCopy = std::make_unique<FindCommandRequest>(fcr); + auto opCtx = makeOperationContext(); + auto parsedFindPair = + uassertStatusOK(parsed_find_command::parse(opCtx.get(), std::move(fcrCopy))); + + // TODO SERVER-84011 backport: re-enable SERVER-79794 test. This test is currently + // broken since the queryStatsfeatureFlags are off by default. Query stats assumes that the + // feature flag can never change without a restart, so enabling the flag in the test still + // doesn't allow for query stats functionality. The following replicates what queryStats would + // have done. + + auto& opDebug = CurOp::get(*opCtx)->debug(); + ASSERT_DOES_NOT_THROW( + try { + BSONObj queryShape = query_shape::extractQueryShape( + *parsedFindPair.second, + SerializationOptions::kRepresentativeQueryShapeSerializeOptions, + parsedFindPair.first); + opDebug.queryStatsKeyGenerator = std::make_unique<query_stats::FindKeyGenerator>( + parsedFindPair.first, + *parsedFindPair.second, + std::move(queryShape), + query_shape::CollectionType::kCollection); + } catch (ExceptionFor<ErrorCodes::BSONObjectTooLarge>&) { return; }) + + opDebug.queryStatsStoreKeyHash = opDebug.queryStatsKeyGenerator->hash(); + ASSERT_EQ(opDebug.queryStatsStoreKeyHash, boost::none); + + // TODO SERVER-84011 backport. Below is the proper test using queryStats. + // RAIIServerParameterControllerForTest controller("featureFlagQueryStats", true); + // RAIIServerParameterControllerForTest queryKnobController{"internalQueryStatsRateLimit", -1}; + + // auto&& globalQueryStatsStoreManager = queryStatsStoreDecoration(opCtx->getServiceContext()); + // globalQueryStatsStoreManager = std::make_unique<QueryStatsStoreManager>(500000, 1000); + + // // The shapification process will bloat the input query over the 16 MB memory limit. Assert + // that + // // calling registerRequest() doesn't throw and that the opDebug isn't registered with a key + // hash + // // (thus metrics won't be tracked for this query). + // ASSERT_DOES_NOT_THROW(query_stats::registerRequest( + // opCtx.get(), + // nss, + // [&]() { + // BSONObj queryShape = query_shape::extractQueryShape( + // *parsedFindPair.second, + // SerializationOptions::kRepresentativeQueryShapeSerializeOptions, + // parsedFindPair.first); + // return std::make_unique<query_stats::FindKeyGenerator>( + // parsedFindPair.first, + // *parsedFindPair.second, + // std::move(queryShape), + // query_shape::CollectionType::kCollection); + // }, + // /*requiresFullQueryStatsFeatureFlag*/ false)); + // auto& opDebug = CurOp::get(*opCtx)->debug(); + // ASSERT_EQ(opDebug.queryStatsStoreKeyHash, boost::none); } TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { @@ -313,10 +437,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "HASH<f>": true, "HASH<_id>": true }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "max": { "HASH<z>": "?number" }, @@ -328,7 +448,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "HASH<otherSort>": -1 } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", key); @@ -363,10 +487,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "HASH<f>": true, "HASH<_id>": true }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "max": { "HASH<z>": "?number" }, @@ -382,7 +502,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { }, "maxTimeMS": "?number", "batchSize": "?number", - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", key); @@ -417,10 +541,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "HASH<f>": true, "HASH<_id>": true }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "max": { "HASH<z>": "?number" }, @@ -441,7 +561,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "allowPartialResults": true, "maxTimeMS": "?number", "batchSize": "?number", - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", key); @@ -470,10 +594,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "HASH<f>": true, "HASH<_id>": true }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "max": { "HASH<z>": "?number" }, @@ -494,7 +614,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { "allowPartialResults": false, "maxTimeMS": "?number", "batchSize": "?number", - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", key); @@ -512,13 +636,13 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { }, "command": "find", "filter": {}, - "hint": { - "$natural": 1 - }, "tailable": true, "awaitData": true }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "$natural": 1 + } })", key); } @@ -570,10 +694,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "$eq": "?number" } }, - "hint": { - "z": 1, - "c": 1 - }, "max": { "z": "?number" }, @@ -581,7 +701,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "z": "?number" } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "z": 1, + "c": 1 + } })", key); // Test with a string hint. Note that this is the internal representation of the string hint @@ -603,9 +727,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "$eq": "?number" } }, - "hint": { - "$hint": "z" - }, "max": { "z": "?number" }, @@ -613,7 +734,10 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "z": "?number" } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "$hint": "z" + } })", key); @@ -632,10 +756,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "$eq": "?number" } }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "max": { "HASH<z>": "?number" }, @@ -643,7 +763,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "HASH<z>": "?number" } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", key); @@ -663,9 +787,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "$eq": "?number" } }, - "hint": { - "$natural": -1 - }, "max": { "HASH<z>": "?number" }, @@ -673,7 +794,10 @@ TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { "HASH<z>": "?number" } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "$natural": -1 + } })", key); } @@ -758,7 +882,8 @@ TEST_F(QueryStatsStoreTest, DefinesLetVariables) { "adaJc6H3zDirh5/52MLv5yvnb6nXNP15Z4HzGfumvx8=": "?number" }, "projection": { - "BL649QER7lTs0+8ozTMVNAa6JNjbhf57YT8YQ4EkT1E=": "$$adaJc6H3zDirh5/52MLv5yvnb6nXNP15Z4HzGfumvx8=", + "BL649QER7lTs0+8ozTMVNAa6JNjbhf57YT8YQ4EkT1E=": + "$$adaJc6H3zDirh5/52MLv5yvnb6nXNP15Z4HzGfumvx8=", "ljovqLSfuj6o2syO1SynOzHQK1YVij6+Wlx1fL8frUo=": true } }, @@ -967,13 +1092,13 @@ TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestAllFieldsSi "allowDiskUse": false, "collation": { "locale": "simple" - }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", shapified); @@ -1040,16 +1165,16 @@ TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestAllFieldsSi "collation": { "locale": "simple" }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "let": { "HASH<var1>": "$HASH<foo>", "HASH<var2>": "?string" } }, - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", shapified); @@ -1119,10 +1244,6 @@ TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestAllFieldsSi "collation": { "locale": "simple" }, - "hint": { - "HASH<z>": 1, - "HASH<c>": 1 - }, "let": { "HASH<var1>": "$HASH<foo>", "HASH<var2>": "?string" @@ -1134,7 +1255,11 @@ TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestAllFieldsSi "maxTimeMS": "?number", "bypassDocumentValidation": "?bool", "comment": "?string", - "collectionType": "collection" + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } })", shapified); } diff --git a/src/mongo/db/query/rate_limiting.cpp b/src/mongo/db/query/query_stats/rate_limiting.cpp index 3aee39facae..029a42b9bdb 100644 --- a/src/mongo/db/query/rate_limiting.cpp +++ b/src/mongo/db/query/query_stats/rate_limiting.cpp @@ -28,10 +28,14 @@ */ #include "rate_limiting.h" +#include "mongo/stdx/mutex.h" +#include "mongo/util/clock_source.h" namespace mongo { -RateLimiting::RateLimiting(RequestCount samplingRate, Milliseconds timePeriod) - : _clockSource(SystemClockSource::get()), +RateLimiting::RateLimiting(RequestCount samplingRate, + Milliseconds timePeriod, + ClockSource* clockSource) + : _clockSource(clockSource != nullptr ? clockSource : SystemClockSource::get()), _samplingRate(samplingRate), _timePeriod(timePeriod), _windowStart(_clockSource->now()), @@ -62,6 +66,7 @@ bool RateLimiting::handleRequestFixedWindow() { } bool RateLimiting::handleRequestSlidingWindow() { + // TODO SERVER-80006: Determine the best RAII type to hold the lock. stdx::unique_lock windowLock{_windowMutex}; Date_t currentTime = tickWindow(); diff --git a/src/mongo/db/query/rate_limiting.h b/src/mongo/db/query/query_stats/rate_limiting.h index 67b8a7fc106..66e38d7119b 100644 --- a/src/mongo/db/query/rate_limiting.h +++ b/src/mongo/db/query/query_stats/rate_limiting.h @@ -30,6 +30,7 @@ #pragma once #include "mongo/util/clock_source.h" +#include "mongo/util/concurrency/mutex.h" #include "mongo/util/system_clock_source.h" namespace mongo { @@ -47,7 +48,9 @@ public: * Constructor for a rate limiter. Specify the number of requests you want to take place, as * well as the time period in milliseconds. */ - RateLimiting(RequestCount samplingRate, Milliseconds timePeriod = Seconds{1}); + RateLimiting(RequestCount samplingRate, + Milliseconds timePeriod = Seconds{1}, + ClockSource* clockSource = nullptr); /* * Getter for the sampling rate. @@ -118,6 +121,6 @@ private: /* * Mutex used when reading/writing the window. */ - stdx::recursive_mutex _windowMutex; + SimpleMutex _windowMutex; }; } // namespace mongo diff --git a/src/mongo/db/query/query_stats/rate_limiting_bm.cpp b/src/mongo/db/query/query_stats/rate_limiting_bm.cpp new file mode 100644 index 00000000000..540c3c63486 --- /dev/null +++ b/src/mongo/db/query/query_stats/rate_limiting_bm.cpp @@ -0,0 +1,144 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include <benchmark/benchmark.h> +#include <climits> +#include <memory> + +#include "mongo/bson/json.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/query_shape.h" +#include "mongo/db/query/query_stats/rate_limiting.h" +#include "mongo/util/duration.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/time_support.h" + +namespace mongo { +namespace { + +// Local testing determined that these parameter values drove the most lock contention, which is +// what we want to capture in this benchmark. +constexpr long long rateLimitedWorkTimeMicros = 5; +constexpr long long consistentWorkTimeMicros = 10; + +constexpr long long numThreads = 256; + +// Rate limit some fraction of the overall work for a request with a sliding window. +int requestWithSlidingWindow(RateLimiting& limit) { + if (limit.handleRequestSlidingWindow()) { + sleepmicros(rateLimitedWorkTimeMicros); + } + sleepmicros(consistentWorkTimeMicros); + return 0; +} + +// Represent a request that bypasses the rate limiter. +int requestUnlimited() { + constexpr long long totalTime = rateLimitedWorkTimeMicros + consistentWorkTimeMicros; + sleepmicros(totalTime); + return 0; +} + +// Represent a request without the rate limited work. +int requestDeactivated() { + sleepmicros(consistentWorkTimeMicros); + return 0; +} + +// Benchmark sliding window rate limiting. +void BM_SlidingWindow(benchmark::State& state) { + // The rate limiter needs a clock source passed in. + static std::unique_ptr<ClockSource> clockSource; + static std::unique_ptr<RateLimiting> rateLimit; + + // Initialize the rate limiter only on the first thread to start up. + if (state.thread_index == 0) { + clockSource = std::make_unique<SystemClockSource>(); + rateLimit = + std::make_unique<RateLimiting>(state.range(0), Milliseconds(1), clockSource.get()); + } + + // Run the benchmark. + for (auto keepRunning : state) { + benchmark::DoNotOptimize(requestWithSlidingWindow(*rateLimit)); + } + + // Clean up the rate limiter when the benchmark is done. + if (state.thread_index == 0) { + rateLimit.reset(); + clockSource.reset(); + } +} + +// "Control" benchmark that does not rate limit requests. In other words, the extra work is always +// done for every request. This benchmark can be thought of as the "goal" performance for the peak, +// or the highest rate limit in BM_SlidingWindow, to compare against. +void BM_Unlimited(benchmark::State& state) { + for (auto keepRunning : state) { + benchmark::DoNotOptimize(requestUnlimited()); + } +} +// Another control benchmark, where the extra work is never done for any request. This can be +// thought of as the goal performance for when rate limit equals 0. +void BM_Deactivated(benchmark::State& state) { + for (auto keepRunning : state) { + benchmark::DoNotOptimize(requestDeactivated()); + } +} + +// Google microbenchmarks report time T (in nanoseconds) spent per operation. But at Mongo we are +// interested in total opereations performed per second. The former can easily be converted to the +// latter by diving 10^6 by T. Use this benchmark to determine the natural throughput of the +// operation. This can be compared to the rate limited benchmarks (BM_SlidingWindow) to determine +// the overhead of rate limiting. Looking at the percentage change in throughput between the control +// benchmarks and the rate limited benchmark, will indicate how much overhead is due to lock +// contention. +BENCHMARK(BM_Unlimited)->Threads(numThreads); + +BENCHMARK(BM_Deactivated)->Threads(numThreads); + +// Local testing has confirmed that the higher the rate limit, the worse the throughput. This makes +// sense as putting a higher upper bound on number of requests allowed in a given time period, means +// longer wait times for the lock. +BENCHMARK(BM_SlidingWindow) + ->ArgName("rate limit") + ->Arg(0) + ->Arg(64) + ->Arg(128) + ->Arg(256) + ->Arg(512) + ->Arg(1024) + ->Arg(2048) + ->Arg(4816) + ->Threads(numThreads); + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/query/rate_limiting_test.cpp b/src/mongo/db/query/query_stats/rate_limiting_test.cpp index 2d9ef35647a..380636a2a20 100644 --- a/src/mongo/db/query/rate_limiting_test.cpp +++ b/src/mongo/db/query/query_stats/rate_limiting_test.cpp @@ -27,7 +27,7 @@ * it in the license file. */ -#include "mongo/db/query/rate_limiting.h" +#include "mongo/db/query/query_stats/rate_limiting.h" #include "mongo/unittest/unittest.h" #include "mongo/util/time_support.h" diff --git a/src/mongo/db/query/query_stats/shapifying_bm.cpp b/src/mongo/db/query/query_stats/shapifying_bm.cpp new file mode 100644 index 00000000000..b7f5715233a --- /dev/null +++ b/src/mongo/db/query/query_stats/shapifying_bm.cpp @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include <benchmark/benchmark.h> +#include <climits> +#include <memory> + +#include "mongo/bson/json.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/find_command.h" +#include "mongo/db/query/query_shape.h" +#include "mongo/db/query/query_stats/find_key_generator.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/db/query/query_stats/rate_limiting.h" +#include "mongo/idl/server_parameter_test_util.h" +#include "mongo/rpc/metadata/client_metadata.h" +#include "mongo/util/duration.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/testing_proctor.h" +#include "mongo/util/time_support.h" + +namespace mongo { +namespace { + +static const NamespaceStringOrUUID kDefaultTestNss = + NamespaceStringOrUUID{NamespaceString::createNamespaceString_forTest("testDB.testColl")}; + +static constexpr auto kCollectionType = query_shape::CollectionType::kCollection; + +// This is a snapshot of the client metadata generated from our IDHACK genny workload. The +// specifics aren't so important, but it chosen in an attempt to be indicative of the size/shape +// of this kind of thing "in the wild". +const auto kMetadataWrapper = fromjson(R"({metadata: { + "application" : { + "name" : "Genny" + }, + "driver" : { + "name" : "mongoc / mongocxx", + "version" : "1.23.2 / 3.7.0" + }, + "os" : { + "type" : "Linux", + "name" : "Ubuntu", + "version" : "22.04", + "architecture" : "aarch64" + }, + "platform" : "cfg=0x03215e88e9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"-fPIC\" LDFLAGS=\"\"" + }})"); +auto kMockClientMetadataElem = kMetadataWrapper["metadata"]; + +auto makeFindKeyGenerator(const boost::intrusive_ptr<ExpressionContext>& expCtx, + const ParsedFindCommand& parsedFind) { + return std::make_unique<query_stats::FindKeyGenerator>( + expCtx, + parsedFind, + query_shape::extractQueryShape( + parsedFind, SerializationOptions::kRepresentativeQueryShapeSerializeOptions, expCtx), + kCollectionType); +} + +int shapifyAndHashRequest(const boost::intrusive_ptr<ExpressionContext>& expCtx, + const ParsedFindCommand& parsedFind) { + auto keyGenerator = makeFindKeyGenerator(expCtx, parsedFind); + [[maybe_unused]] auto hash = keyGenerator->hash(); + return 0; +} + +// Benchmark the performance of computing and hashing the query stats key for an IDHACK query. +void BM_ShapfiyIDHack(benchmark::State& state) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("query_test"); + + auto opCtx = client->makeOperationContext(); + auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get()); + auto fcr = std::make_unique<FindCommandRequest>(expCtx->ns); + fcr->setFilter(fromjson("{_id: 4}")); + ClientMetadata::setFromMetadata(opCtx->getClient(), kMockClientMetadataElem, false); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr))); + + // Run the benchmark. + for (auto keepRunning : state) { + benchmark::DoNotOptimize(shapifyAndHashRequest(expCtx, *parsedFind)); + } +} + +// Benchmark computing the query stats key and its hash for a mildly complex query predicate. +void BM_ShapfiyMildlyComplex(benchmark::State& state) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("query_test"); + + auto opCtx = client->makeOperationContext(); + auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get()); + auto fcr = std::make_unique<FindCommandRequest>(expCtx->ns); + fcr->setFilter(fromjson(R"({ + clientId: {$nin: ["432345", "4386945", "111111"]}, + nEmployees: {$gte: 4, $lt: 20}, + deactivated: false, + region: "US", + yearlySpend: {$lte: 1000} + })")); + ClientMetadata::setFromMetadata(opCtx->getClient(), kMockClientMetadataElem, false); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr))); + + // Run the benchmark. + for (auto keepRunning : state) { + benchmark::DoNotOptimize(shapifyAndHashRequest(expCtx, *parsedFind)); + } +} + +BENCHMARK(BM_ShapfiyIDHack)->Threads(1); +BENCHMARK(BM_ShapfiyMildlyComplex)->Threads(1); + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/query/query_stats_transform_algorithm.idl b/src/mongo/db/query/query_stats/transform_algorithm.idl index cd0a5ba43db..cd0a5ba43db 100644 --- a/src/mongo/db/query/query_stats_transform_algorithm.idl +++ b/src/mongo/db/query/query_stats/transform_algorithm.idl diff --git a/src/mongo/db/query/query_stats_util.cpp b/src/mongo/db/query/query_stats/util.cpp index 4c102d983dc..c93da189ff6 100644 --- a/src/mongo/db/query/query_stats_util.cpp +++ b/src/mongo/db/query/query_stats/util.cpp @@ -27,7 +27,7 @@ * it in the license file. */ -#include "mongo/db/query/query_stats_util.h" +#include "mongo/db/query/query_stats/util.h" #include "mongo/base/status.h" #include "mongo/db/concurrency/d_concurrency.h" diff --git a/src/mongo/db/query/query_stats_util.h b/src/mongo/db/query/query_stats/util.h index ebd8f1e2fbd..ebd8f1e2fbd 100644 --- a/src/mongo/db/query/query_stats_util.h +++ b/src/mongo/db/query/query_stats/util.h diff --git a/src/mongo/db/query/serialization_options.cpp b/src/mongo/db/query/serialization_options.cpp index 2a6ba72def4..271d6cf2d49 100644 --- a/src/mongo/db/query/serialization_options.cpp +++ b/src/mongo/db/query/serialization_options.cpp @@ -87,6 +87,21 @@ static const StringMap<StringData> kArrayTypeStringConstants{ {kMaxKeyTypeString.rawData(), "?array<?maxKey>"_sd}, }; +static constexpr auto kRepresentativeString = "?"_sd; +static constexpr auto kRepresentativeNumber = 1; +static const auto kRepresentativeObject = BSON("?" + << "?"); +static const auto kRepresentativeArray = BSONArray(); +static constexpr auto kRepresentativeBinData = BSONBinData(); +static const auto kRepresentativeObjectId = OID::max(); +static constexpr auto kRepresentativeBool = true; +static const auto kRepresentativeDate = Date_t::fromMillisSinceEpoch(0); +static const auto kRepresentativeRegex = BSONRegEx("/\?/"); +static const auto kRepresentativeDbPointer = BSONDBRef("?.?", OID::max()); +static const auto kRepresentativeJavascript = BSONCode("return ?;"); +static const auto kRepresentativeJavascriptWithScope = BSONCodeWScope("return ?;", BSONObj()); +static const auto kRepresentativeTimestamp = Timestamp::min(); + /** * Computes a debug string meant to represent "any value of type t", where "t" is the type of the * provided argument. For example "?number" for any number (int, double, etc.). @@ -154,39 +169,39 @@ ImplicitValue defaultLiteralOfType(BSONType t) { return BSONUndefined; case Symbol: case String: - return "?"_sd; + return kRepresentativeString; case NumberInt: case NumberLong: case NumberDouble: case NumberDecimal: - return 1; + return kRepresentativeNumber; case MinKey: return MINKEY; case Object: - return Document{{"?"_sd, "?"_sd}}; + return kRepresentativeObject; case Array: // This case should only happen if we have an array within an array. - return BSONArray(); + return kRepresentativeArray; case BinData: - return BSONBinData(); + return kRepresentativeBinData; case jstOID: - return OID::max(); + return kRepresentativeObjectId; case Bool: - return true; + return kRepresentativeBool; case Date: - return Date_t::fromMillisSinceEpoch(0); + return kRepresentativeDate; case jstNULL: return BSONNULL; case RegEx: - return BSONRegEx("/\?/"); + return kRepresentativeRegex; case DBRef: - return BSONDBRef("?.?", OID::max()); + return kRepresentativeDbPointer; case Code: - return BSONCode("return ?;"); + return kRepresentativeJavascript; case CodeWScope: - return BSONCodeWScope("return ?;", BSONObj()); + return kRepresentativeJavascriptWithScope; case bsonTimestamp: - return Timestamp::min(); + return kRepresentativeTimestamp; case MaxKey: return MAXKEY; default: @@ -306,6 +321,68 @@ ArraySubtypeInfo getSubTypeFromValueArray(const Value& arrayVal) { return determineArraySubType(arrayVal.getArray()); } +void appendDefaultOfNonArrayType(BSONObjBuilder* bob, StringData name, const BSONElement& e) { + switch (e.type()) { + case EOO: + case Undefined: + bob->appendUndefined(name); + return; + case Symbol: + case String: + bob->append(name, kRepresentativeString); + return; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + bob->append(name, kRepresentativeNumber); + return; + case MinKey: + bob->appendMinKey(name); + return; + case Object: + bob->append(name, kRepresentativeObject); + return; + case Array: + // This case is more complicated and callers should use a more generic helper. + MONGO_UNREACHABLE_TASSERT(8094100); + case BinData: + bob->append(name, kRepresentativeBinData); + return; + case jstOID: + bob->append(name, kRepresentativeObjectId); + return; + case Bool: + bob->append(name, kRepresentativeBool); + return; + case Date: + bob->append(name, kRepresentativeDate); + return; + case jstNULL: + bob->appendNull(name); + return; + case RegEx: + bob->append(name, kRepresentativeRegex); + return; + case DBRef: + bob->append(name, kRepresentativeDbPointer); + return; + case Code: + bob->append(name, kRepresentativeJavascript); + return; + case CodeWScope: + bob->append(name, kRepresentativeJavascriptWithScope); + return; + case bsonTimestamp: + bob->append(name, kRepresentativeTimestamp); + return; + case MaxKey: + bob->appendMaxKey(name); + return; + default: + MONGO_UNREACHABLE_TASSERT(8094101); + }; +} } // namespace const SerializationOptions SerializationOptions::kRepresentativeQueryShapeSerializeOptions = @@ -332,7 +409,34 @@ ImplicitValue defaultLiteralOfType(BSONElement e) { } void SerializationOptions::appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const { - serializeLiteral(e).addToBsonObj(bob, e.fieldNameStringData()); + appendLiteral(bob, e.fieldNameStringData(), e); +} +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, + StringData name, + const BSONElement& e) const { + // The first two cases are particularly performance sensitive. We could answer everything here + // with the code inside the 'kToDebugTypeString' branch, but there are some relatively easy ways + // to accomplish the first two policy cases (in the common cases), so we'll special case those + // in order to avoid constructing a temporary Value. + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + bob->appendAs(e, name); + return; + case LiteralSerializationPolicy::kToRepresentativeParseableValue: { + if (e.type() != BSONType::Array) { + appendDefaultOfNonArrayType(bob, name, e); + return; + } + // If it's an array we'll default to the slow but general codepath below. + [[fallthrough]]; + } + case LiteralSerializationPolicy::kToDebugTypeString: { + // Performance isn't as sensitive here. + return serializeLiteral(e).addToBsonObj(bob, name); + } + default: + MONGO_UNREACHABLE_TASSERT(8094102); + } } void SerializationOptions::appendLiteral(BSONObjBuilder* bob, diff --git a/src/mongo/db/query/serialization_options.h b/src/mongo/db/query/serialization_options.h index 1661bc88819..f6bbc210a34 100644 --- a/src/mongo/db/query/serialization_options.h +++ b/src/mongo/db/query/serialization_options.h @@ -173,6 +173,7 @@ struct SerializationOptions { * using the same name as 'e'. */ void appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const; + void appendLiteral(BSONObjBuilder* bob, StringData name, const BSONElement& e) const; /** * Helper method to call 'serializeLiteral()' on 'v' and append the result to 'bob' using field * name 'fieldName'. diff --git a/src/mongo/db/query/shape_helpers.cpp b/src/mongo/db/query/shape_helpers.cpp new file mode 100644 index 00000000000..6a5601d2362 --- /dev/null +++ b/src/mongo/db/query/shape_helpers.cpp @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/shape_helpers.h" + +#include "mongo/db/query/query_shape_gen.h" + +namespace mongo::shape_helpers { + +static constexpr StringData hintSpecialField = "$hint"_sd; +// A "Flat" object is one with only top-level fields. We won't descend recursively to shapify any +// sub-objects. +BSONObj shapifyFlatObj(BSONObj obj, const SerializationOptions& opts, bool valuesAreLiterals) { + if (obj.isEmpty()) { + // fast-path for the common case. + return obj; + } + + BSONObjBuilder bob; + for (BSONElement elem : obj) { + if (hintSpecialField.compare(elem.fieldNameStringData()) == 0) { + if (elem.type() == BSONType::String) { + bob.append(hintSpecialField, opts.serializeFieldPathFromString(elem.String())); + } else if (elem.type() == BSONType::Object) { + opts.appendLiteral(&bob, hintSpecialField, elem.Obj()); + } else { + uasserted(ErrorCodes::FailedToParse, "$hint must be a string or an object"); + } + continue; + } + + // $natural doesn't need to be redacted. + if (elem.fieldNameStringData().compare(query_request_helper::kNaturalSortField) == 0) { + bob.append(elem); + continue; + } + + if (valuesAreLiterals) { + opts.appendLiteral(&bob, opts.serializeFieldPathFromString(elem.fieldName()), elem); + } else { + bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldName())); + } + } + return bob.obj(); +} + +BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts) { + return shapifyFlatObj(hintObj, opts, /* valuesAreLiterals = */ false); +} + +BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts) { + return shapifyFlatObj(obj, opts, /* valuesAreLiterals = */ true); +} + +void appendNamespaceShape(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) { + if (nss.tenantId()) { + bob.append("tenantId", opts.serializeIdentifier(nss.tenantId().value().toString())); + } + bob.append("db", opts.serializeIdentifier(nss.db())); + bob.append("coll", opts.serializeIdentifier(nss.coll())); +} + +NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt) { + tassert(7632900, "cmdNs must be an object.", cmdNsElt.type() == BSONType::Object); + auto cmdNs = + query_shape::CommandNamespace::parse(IDLParserContext("cmdNs"), cmdNsElt.embeddedObject()); + + boost::optional<TenantId> tenantId = cmdNs.getTenantId().map(TenantId::parseFromString); + + if (cmdNs.getColl().has_value()) { + tassert(7632903, + "Exactly one of 'uuid' and 'coll' can be defined.", + !cmdNs.getUuid().has_value()); + return NamespaceString(cmdNs.getDb(), cmdNs.getColl().value()); + } else { + tassert(7632904, + "Exactly one of 'uuid' and 'coll' can be defined.", + !cmdNs.getColl().has_value()); + UUID uuid = uassertStatusOK(UUID::parse(cmdNs.getUuid().value().toString())); + return NamespaceStringOrUUID(cmdNs.getDb().toString(), uuid, tenantId); + } +} + +} // namespace mongo::shape_helpers diff --git a/src/mongo/db/query/shape_helpers.h b/src/mongo/db/query/shape_helpers.h new file mode 100644 index 00000000000..1f0e8d72389 --- /dev/null +++ b/src/mongo/db/query/shape_helpers.h @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/query_shape.h" + +namespace mongo::shape_helpers { + +/** + * Serializes the given 'hintObj' in accordance with the options. Assumes the hint is correct and + * contains field names. It is possible that this hint doesn't actually represent an index, but we + * can't detect that here. + */ +BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts); +BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts); + +NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt); +void appendNamespaceShape(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts); + +} // namespace mongo::shape_helpers diff --git a/src/mongo/db/query/stats/SConscript b/src/mongo/db/query/stats/SConscript index 13457cccaea..c9c3e3a59ec 100644 --- a/src/mongo/db/query/stats/SConscript +++ b/src/mongo/db/query/stats/SConscript @@ -5,7 +5,7 @@ Import("env") env = env.Clone() env.Library( - target="query_stats", + target="stats", source=[ 'collection_statistics_impl.cpp', 'stats_catalog.cpp', diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index ac3f69518f4..a54d6d0dfe8 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -631,7 +631,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authorization_manager_global', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/stats/timer_stats', '$BUILD_DIR/mongo/db/storage/storage_options', @@ -1578,7 +1578,7 @@ env.Library( '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/not_primary_error_tracker', '$BUILD_DIR/mongo/db/op_observer/op_observer', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index 6921f8424bd..8e7ed846248 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -29,7 +29,7 @@ env.Library( '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/dbhelpers', '$BUILD_DIR/mongo/db/ops/write_ops', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/shard_role', '$BUILD_DIR/mongo/db/shard_role_api', '$BUILD_DIR/mongo/s/common_s', diff --git a/src/mongo/db/service_context.cpp b/src/mongo/db/service_context.cpp index c2b7f11898c..62f6a32a92c 100644 --- a/src/mongo/db/service_context.cpp +++ b/src/mongo/db/service_context.cpp @@ -256,7 +256,10 @@ ServiceContext::UniqueOperationContext ServiceContext::makeOperationContext(Clie onCreate(opCtx.get(), _clientObservers); ScopeGuard onCreateGuard([&] { onDestroy(opCtx.get(), _clientObservers); }); - invariant(opCtx->lockState(), ProcessInfo().getProcessName()); + invariant( + opCtx->lockState(), + str::stream() << "No lock state configured. This could be a missing build dependency. " + << ProcessInfo().getProcessName()); if (!opCtx->recoveryUnit()) { opCtx->setRecoveryUnit(std::make_unique<RecoveryUnitNoop>(), diff --git a/src/mongo/db/stats/SConscript b/src/mongo/db/stats/SConscript index ce98fc6ee13..ec7bc1490d0 100644 --- a/src/mongo/db/stats/SConscript +++ b/src/mongo/db/stats/SConscript @@ -21,7 +21,7 @@ env.Library( 'operation_latency_histogram.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/util/namespace_string_database_name_util', diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript index 3c3b9237b97..8b2ec215195 100644 --- a/src/mongo/db/storage/kv/SConscript +++ b/src/mongo/db/storage/kv/SConscript @@ -11,7 +11,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/write_unit_of_work', ], ) diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index f752ee0dd60..56345c5b40b 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -56,7 +56,7 @@ wtEnv.Library( '$BUILD_DIR/mongo/db/global_settings', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/prepare_conflict_tracker', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/repl_settings', '$BUILD_DIR/mongo/db/server_options_servers', diff --git a/src/mongo/db/timeseries/SConscript b/src/mongo/db/timeseries/SConscript index b57794e8f8f..8575d3c2410 100644 --- a/src/mongo/db/timeseries/SConscript +++ b/src/mongo/db/timeseries/SConscript @@ -21,7 +21,7 @@ env.Library( 'timeseries_options.cpp', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/serialization_options', + '$BUILD_DIR/mongo/db/query_shape_common', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/storage/storage_options', '$BUILD_DIR/mongo/util/processinfo', @@ -113,7 +113,7 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/collection_query_info', '$BUILD_DIR/mongo/db/catalog/document_validation', '$BUILD_DIR/mongo/db/ops/write_ops_exec_util', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/tenant_migration_decoration', '$BUILD_DIR/mongo/db/shard_role', diff --git a/src/mongo/logv2/log_component.h b/src/mongo/logv2/log_component.h index 65647b00771..e0da86ac55f 100644 --- a/src/mongo/logv2/log_component.h +++ b/src/mongo/logv2/log_component.h @@ -62,6 +62,7 @@ namespace mongo::logv2 { X(kNetwork, , "network" , "NETWORK" , kDefault) \ X(kProcessHealth, , "processHealth" , "HEALTH" , kDefault) \ X(kQuery, , "query" , "QUERY" , kDefault) \ + X(kQueryStats, , "queryStats" , "QRYSTATS", kDefault) \ X(kReplication, , "replication" , "REPL" , kDefault) \ X(kReplicationElection, , "election" , "ELECTION", kReplication) \ X(kReplicationHeartbeats, , "heartbeats" , "REPL_HB" , kReplication) \ diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index a13c87641ab..ce9582fd785 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -104,8 +104,8 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/collection_uuid_mismatch_info', '$BUILD_DIR/mongo/db/internal_transactions_feature_flag', '$BUILD_DIR/mongo/db/mongohasher', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_planner', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/session/sessions_collection', ], ) @@ -287,7 +287,7 @@ env.Library( '$BUILD_DIR/mongo/db/commands/set_user_write_block_mode_idl', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/db/index_commands_idl', - '$BUILD_DIR/mongo/db/serialization_options', + '$BUILD_DIR/mongo/db/query_shape_common', '$BUILD_DIR/mongo/rpc/message', '$BUILD_DIR/mongo/util/caching', 'analyze_shard_key_common', @@ -468,7 +468,7 @@ env.Library( '$BUILD_DIR/mongo/db/logical_time_metadata_hook', '$BUILD_DIR/mongo/db/pipeline/process_interface/mongos_process_interface_factory', '$BUILD_DIR/mongo/db/process_health/fault_manager', - '$BUILD_DIR/mongo/db/query/op_metrics', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/read_write_concern_defaults', '$BUILD_DIR/mongo/db/server_options', '$BUILD_DIR/mongo/db/server_options_base', diff --git a/src/mongo/s/commands/cluster_find_cmd.h b/src/mongo/s/commands/cluster_find_cmd.h index ee246c26870..adc429e1163 100644 --- a/src/mongo/s/commands/cluster_find_cmd.h +++ b/src/mongo/s/commands/cluster_find_cmd.h @@ -40,8 +40,8 @@ #include "mongo/db/pipeline/query_request_conversion.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/query/query_shape.h" -#include "mongo/db/query/query_stats.h" -#include "mongo/db/query/query_stats_find_key_generator.h" +#include "mongo/db/query/query_stats/find_key_generator.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/stats/counters.h" #include "mongo/db/views/resolved_view.h" #include "mongo/rpc/get_status_from_command_result.h" @@ -215,14 +215,20 @@ public: auto& parsedFind = parsedFindResult.second; if (!_didDoFLERewrite) { - BSONObj queryShape = query_shape::extractQueryShape( - *parsedFind, - SerializationOptions::kRepresentativeQueryShapeSerializeOptions, - expCtx); - query_stats::registerRequest(opCtx, expCtx->ns, [&]() { - return std::make_unique<query_stats::FindKeyGenerator>( - expCtx, *parsedFind, std::move(queryShape)); - }); + query_stats::registerRequest( + opCtx, + expCtx->ns, + [&]() { + // This callback is either never invoked or invoked immediately within + // registerRequest, so use-after-move of parsedFind isn't an issue. + BSONObj queryShape = query_shape::extractQueryShape( + *parsedFind, + SerializationOptions::kRepresentativeQueryShapeSerializeOptions, + expCtx); + return std::make_unique<query_stats::FindKeyGenerator>( + expCtx, *parsedFind, std::move(queryShape)); + }, + /*requiresFullQueryStatsFeatureFlag*/ false); } auto cq = uassertStatusOK(CanonicalQuery::canonicalize(expCtx, std::move(parsedFind))); diff --git a/src/mongo/s/query/SConscript b/src/mongo/s/query/SConscript index 7b73be8f98a..e3c580513db 100644 --- a/src/mongo/s/query/SConscript +++ b/src/mongo/s/query/SConscript @@ -16,8 +16,8 @@ env.Library( '$BUILD_DIR/mongo/db/commands', '$BUILD_DIR/mongo/db/curop_failpoint_helpers', '$BUILD_DIR/mongo/db/query/command_request_response', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_common', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/s/sharding_router_api', 'async_results_merger', 'cluster_cursor_cleanup_job', @@ -114,8 +114,8 @@ env.Library( '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/auth/authprivilege', '$BUILD_DIR/mongo/db/generic_cursor', - '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/session/kill_sessions', '$BUILD_DIR/mongo/db/session/logical_session_cache', '$BUILD_DIR/mongo/db/session/logical_session_id', diff --git a/src/mongo/s/query/cluster_aggregate.cpp b/src/mongo/s/query/cluster_aggregate.cpp index c18f98d5acb..f49b9c9505a 100644 --- a/src/mongo/s/query/cluster_aggregate.cpp +++ b/src/mongo/s/query/cluster_aggregate.cpp @@ -56,8 +56,9 @@ #include "mongo/db/query/explain_common.h" #include "mongo/db/query/find_common.h" #include "mongo/db/query/fle/server_rewrite.h" -#include "mongo/db/query/query_stats.h" -#include "mongo/db/query/query_stats_aggregate_key_generator.h" +#include "mongo/db/query/query_stats/aggregate_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/timeseries/timeseries_gen.h" #include "mongo/db/timeseries/timeseries_options.h" #include "mongo/db/views/resolved_view.h" diff --git a/src/mongo/s/query/cluster_client_cursor_impl.cpp b/src/mongo/s/query/cluster_client_cursor_impl.cpp index 3523aba0e0e..eb6ccfc4929 100644 --- a/src/mongo/s/query/cluster_client_cursor_impl.cpp +++ b/src/mongo/s/query/cluster_client_cursor_impl.cpp @@ -32,7 +32,7 @@ #include <memory> #include "mongo/db/curop.h" -#include "mongo/db/query/query_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/logv2/log.h" #include "mongo/s/query/router_stage_limit.h" #include "mongo/s/query/router_stage_merge.h" diff --git a/src/mongo/s/query/cluster_client_cursor_mock.h b/src/mongo/s/query/cluster_client_cursor_mock.h index d6bcb754a18..fbfca636533 100644 --- a/src/mongo/s/query/cluster_client_cursor_mock.h +++ b/src/mongo/s/query/cluster_client_cursor_mock.h @@ -33,7 +33,7 @@ #include <functional> #include <queue> -#include "mongo/db/query/query_stats_key_generator.h" +#include "mongo/db/query/query_stats/key_generator.h" #include "mongo/db/session/logical_session_id.h" #include "mongo/s/query/cluster_client_cursor.h" diff --git a/src/mongo/s/query/cluster_cursor_manager.cpp b/src/mongo/s/query/cluster_cursor_manager.cpp index 20e65d08dd3..c22640f7552 100644 --- a/src/mongo/s/query/cluster_cursor_manager.cpp +++ b/src/mongo/s/query/cluster_cursor_manager.cpp @@ -38,7 +38,7 @@ #include "mongo/db/allocate_cursor_id.h" #include "mongo/db/curop.h" #include "mongo/db/query/query_knobs_gen.h" -#include "mongo/db/query/query_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/session/kill_sessions_common.h" #include "mongo/db/session/logical_session_cache.h" #include "mongo/logv2/log.h" diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp index 830e2f68683..773fc90ab4d 100644 --- a/src/mongo/s/query/cluster_find.cpp +++ b/src/mongo/s/query/cluster_find.cpp @@ -48,7 +48,7 @@ #include "mongo/db/query/find_common.h" #include "mongo/db/query/getmore_command_gen.h" #include "mongo/db/query/query_planner_common.h" -#include "mongo/db/query/query_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/logv2/log.h" #include "mongo/platform/overflow_arithmetic.h" |