diff options
Diffstat (limited to 'jstests/core')
49 files changed, 2603 insertions, 351 deletions
diff --git a/jstests/core/aggregation_accepts_write_concern.js b/jstests/core/aggregation_accepts_write_concern.js index 2c764414a1d..8117c296e03 100644 --- a/jstests/core/aggregation_accepts_write_concern.js +++ b/jstests/core/aggregation_accepts_write_concern.js @@ -1,7 +1,11 @@ /** * Confirms that the aggregate command accepts writeConcern regardless of whether the pipeline * writes or is read-only. - * @tags: [assumes_write_concern_unchanged, does_not_support_stepdowns] + * @tags: [ + * assumes_write_concern_unchanged, + * does_not_support_stepdowns, + * references_foreign_collection + * ] */ (function() { "use strict"; diff --git a/jstests/core/api_version_pipeline_stages.js b/jstests/core/api_version_pipeline_stages.js index c9772e4fa0c..c9d092a9a06 100644 --- a/jstests/core/api_version_pipeline_stages.js +++ b/jstests/core/api_version_pipeline_stages.js @@ -27,7 +27,6 @@ const unstablePipelines = [ [{$planCacheStats: {}}], [{$unionWith: {coll: "coll2", pipeline: [{$collStats: {latencyStats: {}}}]}}], [{$lookup: {from: "coll2", pipeline: [{$indexStats: {}}]}}], - [{$lookup: {from: "coll2", _internalCollation: {locale: "simple"}}}], [{$facet: {field1: [], field2: [{$indexStats: {}}]}}], ]; diff --git a/jstests/core/api_version_test_expression.js b/jstests/core/api_version_test_expression.js index 41bbd9c0402..3252ebda83e 100644 --- a/jstests/core/api_version_test_expression.js +++ b/jstests/core/api_version_test_expression.js @@ -8,6 +8,7 @@ * assumes_unsharded_collection, * uses_api_parameters, * no_selinux, + * references_foreign_collection, * ] */ diff --git a/jstests/core/arrayfind8.js b/jstests/core/arrayfind8.js index 87a3a8d701a..ee74aae33ef 100644 --- a/jstests/core/arrayfind8.js +++ b/jstests/core/arrayfind8.js @@ -81,7 +81,6 @@ function checkQuery(subQuery, bothMatch, elemMatch, nonElemMatch, additionalCons insertValueIfNotNull(bothMatch); insertValueIfNotNull(elemMatch); insertValueIfNotNull(nonElemMatch); - checkMatch(bothMatch, elemMatch, nonElemMatch, standardQuery, elemMatchQuery, 'unindexed'); // Check matching and index bounds for a single key index. diff --git a/jstests/core/bypass_doc_validation.js b/jstests/core/bypass_doc_validation.js index d290508835a..1fd32db658b 100644 --- a/jstests/core/bypass_doc_validation.js +++ b/jstests/core/bypass_doc_validation.js @@ -6,6 +6,7 @@ // uses_map_reduce_with_temp_collections, // # Tenant migrations don't support applyOps. // tenant_migration_incompatible, +// references_foreign_collection, // ] /** diff --git a/jstests/core/bypass_empty_ts_replacement.js b/jstests/core/bypass_empty_ts_replacement.js new file mode 100644 index 00000000000..9bd74f3f0ca --- /dev/null +++ b/jstests/core/bypass_empty_ts_replacement.js @@ -0,0 +1,202 @@ +/** + * Tests the "bypassEmptyTsReplacement" option. + * + * @tags: [ + * requires_fcv_60, + * ] + */ +(function() { +"use strict"; + +const coll = db.jstests_bypass_empty_ts_replacement; +const collName = coll.getName(); +const emptyTs = Timestamp(0, 0); + +coll.drop(); + +function doInsert(docs, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand( + {insert: collName, documents: docs, bypassEmptyTsReplacement: bypassEmptyTsReplacement}); + assert.commandWorked(cmdRes); +} + +function doUpdate(filter, update, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand({ + update: collName, + updates: [{q: filter, u: update}], + bypassEmptyTsReplacement: bypassEmptyTsReplacement + }); + assert.commandWorked(cmdRes); +} + +function doUpdateWithUpsert(filter, update, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand({ + update: collName, + updates: [{q: filter, u: update, upsert: true}], + bypassEmptyTsReplacement: bypassEmptyTsReplacement + }); + assert.commandWorked(cmdRes); +} + +function doFindAndModify(filter, update, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand({ + findAndModify: collName, + query: filter, + update: update, + bypassEmptyTsReplacement: bypassEmptyTsReplacement + }); + assert.commandWorked(cmdRes); +} + +function doFindAndModifyWithUpsert(filter, update, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand({ + findAndModify: collName, + query: filter, + update: update, + upsert: true, + bypassEmptyTsReplacement: bypassEmptyTsReplacement + }); + assert.commandWorked(cmdRes); +} + +let numCallsToRunTests = 0; + +function runTests(bypassEmptyTsReplacement) { + ++numCallsToRunTests; + + let first = (numCallsToRunTests == 1); + let startId = numCallsToRunTests * 100; + + function getId(i) { + return startId + i; + } + + // The insert/update/findAndModify commands below will be run with the bypassEmptyTsReplacement + // parameter set to true or false (depending whether the 'bypassEmptyTsReplacement' variable is + // true or false). + + // Insert several documents. + doInsert([{_id: getId(1), a: emptyTs}], bypassEmptyTsReplacement); + doInsert([{_id: getId(2), a: 1}], bypassEmptyTsReplacement); + doInsert([{_id: getId(3), a: 2}], bypassEmptyTsReplacement); + doInsert([{_id: getId(6), a: 3}], bypassEmptyTsReplacement); + doInsert([{_id: getId(7), a: 4}], bypassEmptyTsReplacement); + doInsert([{_id: getId(8), a: 5}], bypassEmptyTsReplacement); + doInsert([{_id: getId(9), a: 6}], bypassEmptyTsReplacement); + doInsert([{_id: getId(10), a: 7}], bypassEmptyTsReplacement); + doInsert([{_id: getId(11), a: 8}], bypassEmptyTsReplacement); + + // Use a replacement-style update to update the doc with _id=getId(2). + // + // When bypassEmptyTsReplacement=true, this is an example of how the special "empty timestamp" + // behavior can be suppressed when doing a replacement-style update. + doUpdate({_id: getId(2)}, {a: emptyTs}, bypassEmptyTsReplacement); + + // Use a replacement-style findAndModify to update the doc with _id=getId(3). + doFindAndModify({_id: getId(3)}, {a: emptyTs}, bypassEmptyTsReplacement); + + // Do a replacement-style update to add a new document with _id=getId(4). + doUpdateWithUpsert({_id: getId(4)}, {a: emptyTs}, bypassEmptyTsReplacement); + + // Do a replacement-style findAndModify to add a new document with _id=getId(5). + doFindAndModifyWithUpsert({_id: getId(5)}, {a: emptyTs}, bypassEmptyTsReplacement); + + // Do an update-operator-style update to update the doc with _id=getId(6). + doUpdate({_id: getId(6)}, {$set: {a: emptyTs}}, bypassEmptyTsReplacement); + + // Do an update-operator-style findAndModify to update the doc with _id=getId(7). + doFindAndModify({_id: getId(7)}, {$set: {a: emptyTs}}, bypassEmptyTsReplacement); + + // Do a pipeline-style update to update the doc with _id=getId(8). + doUpdate({_id: getId(8)}, [{$addFields: {a: emptyTs}}], bypassEmptyTsReplacement); + + // Do a pipeline-style findAndModify to update the doc with _id=getId(9). + doFindAndModify({_id: getId(9)}, [{$addFields: {a: emptyTs}}], bypassEmptyTsReplacement); + + // Do a pipeline-style update with $internalApplyOplogUpdate to update the doc with + // _id=getId(10). + doUpdate({_id: getId(10)}, + [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + bypassEmptyTsReplacement); + + // Do a pipeline-style findAndModify with $internalApplyOplogUpdate to update the doc with + // _id=getId(11). + doFindAndModify({_id: getId(11)}, + [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + bypassEmptyTsReplacement); + + // Do an update-operator-style update to add a new document with _id=getId(12). + doUpdateWithUpsert({_id: getId(12)}, {$set: {a: emptyTs}}, bypassEmptyTsReplacement); + + // Do an update-operator-style findAndModify to add a new document with _id=getId(13). + doFindAndModifyWithUpsert({_id: getId(13)}, {$set: {a: emptyTs}}, bypassEmptyTsReplacement); + + // Do a pipeline-style update to add a new document with _id=getId(14). + doUpdateWithUpsert({_id: getId(14)}, [{$addFields: {a: emptyTs}}], bypassEmptyTsReplacement); + + // Do a pipeline-style findAndModify to add a new document with _id=getId(15). + doFindAndModifyWithUpsert( + {_id: getId(15)}, [{$addFields: {a: emptyTs}}], bypassEmptyTsReplacement); + + // Do a pipline-style update with $internalApplyOplogUpdate to add a new document with + // _id=getId(16). + doUpdateWithUpsert( + {_id: getId(16)}, + [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + bypassEmptyTsReplacement); + + // Do pipeline-style findAndModify with $internalApplyOplogUpdate to add a new document + // with _id=getId(17). + doFindAndModifyWithUpsert( + {_id: getId(17)}, + [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + bypassEmptyTsReplacement); + + // Verify that all the insert, update, and findAndModify commands behaved the way we expected. + // If 'bypassEmptyTsReplacement' is true, then we expect field "a" will be equal to the empty + // timestamp for all 17 documents. If 'bypassEmptyTsReplacement' is false, then we expect + // field "a" will be equal to the current timestamp for the first 5 documents and it will + // be equal to the empty timestamp for the other 12 documents. + for (let i = 1; i <= 17; ++i) { + let result = coll.findOne({_id: getId(i)}); + + if (bypassEmptyTsReplacement || i >= 6) { + assert.eq(tojson(result.a), tojson(emptyTs), "_id=" + getId(i)); + } else { + assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + getId(i)); + } + } + + if (first) { + // If this is the first time that runTests() is invoked, insert a document with + // _id=Timestamp(0,0), and then verify that the document we just inserted can be + // retrieved using the filter "{_id: Timestamp(0,0)}". + doInsert([{_id: emptyTs, a: 9}], bypassEmptyTsReplacement); + + let result = coll.findOne({_id: emptyTs}); + assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); + assert.eq(tojson(result.a), tojson(9), "_id=" + tojson(emptyTs)); + } + + // Do a replacement-style update on the document with _id=Timestamp(0,0). + doUpdate({_id: emptyTs}, {_id: emptyTs, a: emptyTs}, bypassEmptyTsReplacement); + + // Verify the document we just updated can still be retrieved using "{_id: Timestamp(0,0)}". + let result = coll.findOne({_id: emptyTs}); + assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); + + if (bypassEmptyTsReplacement) { + // Verify that an empty timestamp value was stored in field "a". + assert.eq(tojson(result.a), tojson(emptyTs), "_id=" + tojson(emptyTs)); + } else { + // Verify that field "a" was set to the current timestamp. + assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + tojson(emptyTs)); + } +} + +let bypassEmptyTsReplacement = true; +runTests(bypassEmptyTsReplacement); + +bypassEmptyTsReplacement = false; +runTests(bypassEmptyTsReplacement); +}()); diff --git a/jstests/core/bypass_empty_ts_replacement_timeseries.js b/jstests/core/bypass_empty_ts_replacement_timeseries.js new file mode 100644 index 00000000000..69a5bcf281c --- /dev/null +++ b/jstests/core/bypass_empty_ts_replacement_timeseries.js @@ -0,0 +1,73 @@ +/** + * Tests inserts, updates, and upserts on a timeseries collection with "bypassEmptyTsReplacement" + * set to true. + * + * @tags: [ + * assumes_no_implicit_collection_creation_after_drop, + * does_not_support_transactions, + * requires_fcv_60, + * requires_multi_updates, + * requires_non_retryable_writes, + * requires_timeseries, + * ] + */ +(function() { +"use strict"; + +const coll = db.jstests_bypass_empty_ts_replacement_timeseries; +const collName = coll.getName(); +const emptyTs = Timestamp(0, 0); + +coll.drop(); + +assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: "t", metaField: "m"}})); + +function doInsert(docs, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand( + {insert: collName, documents: docs, bypassEmptyTsReplacement: bypassEmptyTsReplacement}); + assert.commandWorked(cmdRes); +} + +function doUpdate(filter, update, bypassEmptyTsReplacement) { + let cmdRes = db.runCommand({ + update: collName, + updates: [{q: filter, u: update, multi: true}], + bypassEmptyTsReplacement: bypassEmptyTsReplacement + }); + assert.commandWorked(cmdRes); +} + +let numCallsToRunTests = 0; + +function runTests(bypassEmptyTsReplacement) { + ++numCallsToRunTests; + + let startId = numCallsToRunTests * 100; + + function getId(i) { + return startId + i; + } + + // Insert two documents. + doInsert([{_id: getId(1), t: new Date(100), m: emptyTs, a: emptyTs}], bypassEmptyTsReplacement); + doInsert([{_id: getId(2), t: new Date(200), m: getId(2), a: emptyTs}], + bypassEmptyTsReplacement); + + // Do a replacement-style update containing an empty timestamp value. + doUpdate({m: getId(2)}, {$set: {m: emptyTs}}, bypassEmptyTsReplacement); + + // Verify that the commands above didn't mutate any of the empty timestamp values in the + // collection. + for (let i = 1; i <= 2; ++i) { + let result = coll.findOne({_id: getId(i)}); + assert.eq(tojson(result.m), tojson(emptyTs), "_id=" + getId(i)); + assert.eq(tojson(result.a), tojson(emptyTs), "_id=" + getId(i)); + } +} + +let bypassEmptyTsReplacement = true; +runTests(bypassEmptyTsReplacement); + +bypassEmptyTsReplacement = false; +runTests(bypassEmptyTsReplacement); +}()); diff --git a/jstests/core/check_shard_index.js b/jstests/core/check_shard_index.js index 2beb3c12891..23f69c6b757 100644 --- a/jstests/core/check_shard_index.js +++ b/jstests/core/check_shard_index.js @@ -21,12 +21,12 @@ f.createIndex({x: 1, y: 1}); assert.eq(0, f.count(), "1. initial count should be zero"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(true, res.ok, "1a"); +assert.commandWorked(res, "1a " + tojson(res)); f.save({x: 1, y: 1}); assert.eq(1, f.count(), "1. count after initial insert should be 1"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(true, res.ok, "1b"); +assert.commandWorked(res, "1b " + tojson(res)); // ------------------------- // Case 2: entry with null values would make an index suitable @@ -40,16 +40,16 @@ f.save({x: 1, y: 1}); f.save({x: null, y: 1}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(true, res.ok, "2a " + tojson(res)); +assert.commandWorked(res, "2a " + tojson(res)); f.save({y: 2}); assert.eq(3, f.count(), "2. count after initial insert should be 3"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(true, res.ok, "2b " + tojson(res)); +assert.commandWorked(res, "2b " + tojson(res)); // Check _id index res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {_id: 1}}); -assert.eq(true, res.ok, "2c " + tojson(res)); +assert.commandWorked(res, "2c " + tojson(res)); assert(res.idskip, "2d " + tojson(res)); // ------------------------- @@ -65,7 +65,7 @@ f.save({x: [1, 2], y: 2}); assert.eq(2, f.count(), "3. count after initial insert should be 2"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok, "3a " + tojson(res)); +assert.commandFailed(res, "3a " + tojson(res)); f.remove({y: 2}); f.dropIndex({x: 1, y: 1}); @@ -73,13 +73,13 @@ f.createIndex({x: 1, y: 1}); assert.eq(1, f.count(), "3. count after removing array value should be 1"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(true, res.ok, "3b " + tojson(res)); +assert.commandWorked(res, "3b " + tojson(res)); f.save({x: 2, y: [1, 2]}); assert.eq(2, f.count(), "3. count after adding array value should be 2"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok, "3c " + tojson(res)); +assert.commandFailed(res, "3c " + tojson(res)); // ------------------------- // Case 4: Handles prefix shard key indexes. @@ -93,21 +93,21 @@ f.save({x: 1, y: 1, z: 1}); assert.eq(1, f.count(), "4. count after initial insert should be 1"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1}}); -assert.eq(true, res.ok, "4a " + tojson(res)); +assert.commandWorked(res, "4a " + tojson(res)); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(true, res.ok, "4b " + tojson(res)); +assert.commandWorked(res, "4b " + tojson(res)); f.save({x: [1, 2], y: 2, z: 2}); assert.eq(2, f.count(), "4. count after adding array value should be 2"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1}}); -assert.eq(false, res.ok, "4c " + tojson(res)); +assert.commandFailed(res, "4c " + tojson(res)); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok, "4d " + tojson(res)); +assert.commandFailed(res, "4d " + tojson(res)); res = db.runCommand( {checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}}); -assert.eq(false, res.ok, "4e " + tojson(res)); +assert.commandFailed(res, "4e " + tojson(res)); f.remove({y: 2}); f.dropIndex({x: 1, y: 1, z: 1}); @@ -116,18 +116,18 @@ f.createIndex({x: 1, y: 1, z: 1}); assert.eq(1, f.count(), "4. count after removing array value should be 1"); res = db.runCommand( {checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}}); -assert.eq(true, res.ok, "4f " + tojson(res)); +assert.commandWorked(res, "4f " + tojson(res)); f.save({x: 3, y: [1, 2], z: 3}); assert.eq(2, f.count(), "4. count after adding array value on second key should be 2"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1}}); -assert.eq(false, res.ok, "4g " + tojson(res)); +assert.commandFailed(res, "4g " + tojson(res)); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok, "4h " + tojson(res)); +assert.commandFailed(res, "4h " + tojson(res)); res = db.runCommand( {checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}}); -assert.eq(false, res.ok, "4i " + tojson(res)); +assert.commandFailed(res, "4i " + tojson(res)); f.remove({x: 3}); // Necessary so that the index is no longer marked as multikey @@ -137,18 +137,18 @@ f.createIndex({x: 1, y: 1, z: 1}); assert.eq(1, f.count(), "4. count after removing array value should be 1 again"); res = db.runCommand( {checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}}); -assert.eq(true, res.ok, "4e " + tojson(res)); +assert.commandWorked(res, "4e " + tojson(res)); f.save({x: 4, y: 4, z: [1, 2]}); assert.eq(2, f.count(), "4. count after adding array value on third key should be 2"); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1}}); -assert.eq(false, res.ok, "4c " + tojson(res)); +assert.commandFailed(res, "4c " + tojson(res)); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok, "4d " + tojson(res)); +assert.commandFailed(res, "4d " + tojson(res)); res = db.runCommand( {checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}}); -assert.eq(false, res.ok, "4e " + tojson(res)); +assert.commandFailed(res, "4e " + tojson(res)); // ------------------------- // Test error messages of checkShardingIndex failing: @@ -157,21 +157,21 @@ assert.eq(false, res.ok, "4e " + tojson(res)); f.drop(); f.createIndex({x: 1}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Shard key is not a prefix of index key.")); // Index key is partial: f.drop(); f.createIndex({x: 1, y: 1}, {partialFilterExpression: {y: {$gt: 0}}}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index key is partial.")); // Index key is sparse: f.drop(); f.createIndex({x: 1, y: 1}, {sparse: true}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index key is sparse.")); // Index key is multikey: @@ -179,21 +179,21 @@ f.drop(); f.createIndex({x: 1, y: 1}); f.save({y: [1, 2, 3, 4, 5]}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index key is multikey.")); // Index key has a non-simple collation: f.drop(); f.createIndex({x: 1, y: 1}, {collation: {locale: "en"}}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index has a non-simple collation.")); // Index key is sparse and index has non-simple collation: f.drop(); f.createIndex({x: 1, y: 1}, {sparse: true, collation: {locale: "en"}}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index key is sparse.") && res.errmsg.includes("Index has a non-simple collation.")); @@ -203,7 +203,7 @@ f.createIndex({x: 1, y: 1}, {name: "index_1_part", partialFilterExpression: {x: f.createIndex({x: 1, y: 1}, {name: "index_2"}); f.save({y: [1, 2, 3, 4, 5]}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index key is multikey.") && res.errmsg.includes("Index key is partial.")); @@ -212,7 +212,7 @@ f.drop(); f.createIndex({x: 1, y: 1}, {name: "index_1_part", partialFilterExpression: {x: {$gt: 0}}}); f.createIndex({x: 1, y: 1}, {name: "index_2_sparse", sparse: true}); res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}}); -assert.eq(false, res.ok); +assert.commandFailed(res); assert(res.errmsg.includes("Index key is partial.") && res.errmsg.includes("Index key is sparse.")); print("PASSED"); diff --git a/jstests/core/collmod.js b/jstests/core/collmod.js index 4362fc110a4..3362650a77a 100644 --- a/jstests/core/collmod.js +++ b/jstests/core/collmod.js @@ -131,3 +131,17 @@ assert.commandFailed( // Fails with an unknown key pattern. assert.commandFailed(db.runCommand( {collmod: coll, index: {keyPattern: {doesnotexist: 1}, expireAfterSeconds: 100}})); + +// Exclude from multiversion suites as SERVER-84531 will not be backported to older versions +const isMultiversion = jsTestOptions().mixedBinVersions || jsTestOptions().shardMixedBinVersions || + jsTestOptions().useRandomBinVersionsWithinReplicaSet; +if (!isMultiversion) { + // The timeseriesBucketsMayHaveMixedSchemaData option can only be used on time-series + // collections. + assert.commandFailedWithCode( + db.runCommand({collMod: coll, timeseriesBucketsMayHaveMixedSchemaData: true}), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode( + db.runCommand({collMod: coll, timeseriesBucketsMayHaveMixedSchemaData: false}), + ErrorCodes.InvalidOptions); +} diff --git a/jstests/core/command_let_variables_expressions.js b/jstests/core/command_let_variables_expressions.js new file mode 100644 index 00000000000..c551c30be48 --- /dev/null +++ b/jstests/core/command_let_variables_expressions.js @@ -0,0 +1,208 @@ +// Tests that commands like find, aggregate and update accept a 'let' parameter which defines +// variables for use in expressions within the command. Specifically covers cases where the +// parameter definitions contain expressions. +// @tags: [ +// requires_non_retryable_writes, +// does_not_support_transactions, +// tenant_migration_incompatible +// ] +(function() { +"use strict"; + +load('jstests/aggregation/extras/utils.js'); // For assertArrayEq(). + +const coll = db.getCollection('command_let_variables_expressions'); + +function setupColl() { + coll.drop(); + assert.commandWorked(coll.insert([ + {_id: 2, a: {}}, + {_id: 3, b: 1}, + {_id: 4, a: "$notAFieldPath"}, + ])); +} +setupColl(); + +const missingLetParam = { + $getField: {field: "c", input: {a: 1}} +}; +const literalLetParam = { + $literal: "$notAFieldPath" +}; + +// Run find commands with 'let' parameters containing expressions. +{ + // 'let' parameter expression that resolves to a missing value should not cause a query error. + let result = assert.commandWorked(db.runCommand({ + find: coll.getName(), + filter: {$expr: {$eq: ["$a", "$$c"]}}, + let : {c: missingLetParam}, + })); + assertArrayEq({ + actual: result.cursor.firstBatch, + expected: [{_id: 3, b: 1}], + }); + + // 'let' parameter expression that includes $literal and a $-prefixed string should not be + // misinterpreted as a field path. + result = assert.commandWorked(db.runCommand({ + find: coll.getName(), + filter: {$expr: {$eq: ["$a", "$$c"]}}, + let : {c: literalLetParam}, + })); + assertArrayEq({ + actual: result.cursor.firstBatch, + expected: [{_id: 4, a: "$notAFieldPath"}], + }); +} + +// Run aggregate commands with 'let' parameters containing expressions. +{ + let result = + coll.aggregate([{$match: {$expr: {$eq: ["$a", "$$c"]}}}], {let : {c: missingLetParam}}) + .toArray(); + assertArrayEq({ + actual: result, + expected: [{_id: 3, b: 1}], + }); + + result = coll.aggregate([{$match: {$expr: {$eq: ["$a", "$$c"]}}}], {let : {c: literalLetParam}}) + .toArray(); + assertArrayEq({ + actual: result, + expected: [{_id: 4, a: "$notAFieldPath"}], + }); +} + +// Same thing but for update. +{ + assert.commandWorked(db.runCommand({ + update: coll.getName(), + updates: [{ + q: {$expr: {$eq: ["$a", "$$c"]}}, + u: [{$set: {c: "updated"}}], + multi: true, + }], + let : {c: missingLetParam} + })); + assertArrayEq({ + actual: coll.find().toArray(), + expected: [{_id: 2, a: {}}, {_id: 3, b: 1, c: "updated"}, {_id: 4, a: "$notAFieldPath"}], + }); + + // Undo changes. + setupColl(); + + assert.commandWorked(db.runCommand({ + update: coll.getName(), + updates: [{ + q: {$expr: {$eq: ["$a", "$$c"]}}, + u: [{$set: {c: "updated"}}], + multi: true, + }], + let : {c: literalLetParam} + })); + assertArrayEq({ + actual: coll.find().toArray(), + expected: [ + {_id: 2, a: {}}, + {_id: 3, b: 1}, + {_id: 4, a: "$notAFieldPath", c: "updated"}, + ], + }); +} + +// Run findAndModify commands with 'let' parameters containing expressions. +{ + setupColl(); + + assert.commandWorked(db.runCommand({ + findAndModify: coll.getName(), + query: {_id: 3, $expr: {$eq: ["$a", "$$c"]}}, + update: [{$set: {c: "updated"}}], + let : {c: missingLetParam} + })); + assertArrayEq({ + actual: coll.find().toArray(), + expected: [{_id: 2, a: {}}, {_id: 3, b: 1, c: "updated"}, {_id: 4, a: "$notAFieldPath"}], + }); + + // Undo changes. + setupColl(); + + assert.commandWorked(db.runCommand({ + findAndModify: coll.getName(), + query: {_id: 4, $expr: {$eq: ["$a", "$$c"]}}, + update: [{$set: {c: "updated"}}], + let : {c: literalLetParam} + })); + assertArrayEq({ + actual: coll.find().toArray(), + expected: [ + {_id: 2, a: {}}, + {_id: 3, b: 1}, + {_id: 4, a: "$notAFieldPath", c: "updated"}, + ], + }); +} + +// Run delete commands with 'let' parameters containing expressions. +{ + setupColl(); + + assert.commandWorked(db.runCommand({ + delete: coll.getName(), + deletes: [{ + q: {$expr: {$eq: ["$a", "$$c"]}}, + limit: 0, // multi + }], + let : {c: missingLetParam} + })); + assertArrayEq({ + actual: coll.find().toArray(), + expected: [{_id: 2, a: {}}, {_id: 4, a: "$notAFieldPath"}], + }); + + // Undo changes. + setupColl(); + + assert.commandWorked(db.runCommand({ + delete: coll.getName(), + deletes: [{ + q: {$expr: {$eq: ["$a", "$$c"]}}, + limit: 0, // multi + }], + let : {c: literalLetParam} + })); + assertArrayEq({ + actual: coll.find().toArray(), + expected: [{_id: 2, a: {}}, {_id: 3, b: 1}], + }); +} + +// Run a $lookup with let/pipeline syntax. In sharded environments, this will require serializing +// 'let' variables to send to the shards. One of the documents in the collection is missing the +// local field ("$a"), which will cause a missing 'let' variable to be serialized. +{ + setupColl(); + + const result = coll.aggregate([ + { + $lookup: { + from: coll.getName(), + as: "res", + let: {local_a: "$a"}, + pipeline: [{$match: {$expr: {$eq: ["$$local_a", "$a"]},}}, {$project: {_id: 1}}] + } + } + ]).toArray(); + assertArrayEq({ + actual: result, + expected: [ + {_id: 2, a: {}, res: [{_id: 2}]}, + {_id: 3, b: 1, res: [{_id: 3}]}, + {_id: 4, a: "$notAFieldPath", res: [{_id: 4}]}, + ], + }); +} +}()); diff --git a/jstests/core/currentop_shell.js b/jstests/core/currentop_shell.js index c96ef2507f1..31b084bf132 100644 --- a/jstests/core/currentop_shell.js +++ b/jstests/core/currentop_shell.js @@ -2,7 +2,16 @@ * Tests that the shell helper db.currentOpCursor isn't constrained by the legacy currentOp server * command - ie. the result set isn't limited to 16MB and long operations aren't truncated. * + * Note: On newer branches, this test contains additional cases for the currentOp command (without a + * shell helper) and the $currentOp pipeline stage, which are not included here. Those cases would + * behave unreliably because of SERVER-92284. + * * @tags: [ + * # The collection may be completely moved to another shard, which results in currentOp not + * # returning the expected command. + * assumes_balancer_off, + * # The test runs commands that are not allowed with security token: getLog. + * not_allowed_with_signed_security_token, * uses_parallel_shell, * # This test uses currentOp to check whether an aggregate command is running. In replica set * # environments, because currentOp is run against the admin database it is routed to the @@ -11,6 +20,8 @@ * # currentOp results. * assumes_read_preference_unchanged, * no_selinux, + * # Uses $function operator. + * requires_scripting, * ] */ @@ -18,31 +29,45 @@ "use strict"; load("jstests/libs/fixture_helpers.js"); // for FixtureHelpers +load('jstests/libs/parallel_shell_helpers.js'); const coll = db.currentOp_cursor; coll.drop(); -for (let i = 0; i < 3; i++) { +for (let i = 0; i < 100; i++) { assert.commandWorked(coll.insert({val: 1})); } +// // Test that db.currentOpCursor() returns an iterable cursor. -let res = db.currentOpCursor(); -assert(res.hasNext()); -assert(res.next()); - -// Test that db.currentOp() interface does not change. -res = db.currentOp(); -assert("inprog" in res, "Result contains 'inprog' field"); -assert("ok" in res, "Result contains 'ok' field"); - -// Attempting to access the fsyncLock field from the results throws with an error message. -let error = assert.throws(() => res.fsyncLock); +// +const cursorFromCurrentOp = db.currentOpCursor(); +assert(cursorFromCurrentOp.hasNext()); +assert(cursorFromCurrentOp.next()); + +// +// Test that db.currentOp() returns an object in the expected format. +// +const currentOpRes = db.currentOp(); +assert("inprog" in currentOpRes, "Result contains 'inprog' field"); +assert("ok" in currentOpRes, "Result contains 'ok' field"); + +// +// Test that attempting to access the fsyncLock field from the results throws with an error message. +// +const error = assert.throws(() => currentOpRes.fsyncLock); assert( /fsyncLock is no longer included in the currentOp shell helper, run db\.runCommand\({currentOp: 1}\) instead/ .test(error)); -function shellOp() { +// +// Start a pipeline with a large command object in a parallel shell and then test three different +// methods of executing "currentOp" queries to ensure that they all observe the operation and that +// they do or do not truncate its command object (according to each one's specification). +// + +// Starts the query. Intended to b e called from a parallel shell. +function startLongRunningAggregation(collName, comment) { function createLargeDoc() { let doc = {}; for (let i = 0; i < 100; i++) { @@ -52,90 +77,96 @@ function shellOp() { } assert.commandFailedWithCode(db.runCommand({ - aggregate: "currentOp_cursor", + aggregate: collName, pipeline: [{ $addFields: { newVal: {$function: {args: [], body: "sleep(1000000)", lang: "js"}}, bigDoc: createLargeDoc() } }], - comment: TestData.comment, + comment: comment, cursor: {} }), ErrorCodes.Interrupted); } -function startShellWithOp(comment) { - TestData.comment = comment; - const awaitShell = startParallelShell(shellOp); - - // Confirm that the operation has started in the parallel shell. +// Repeatedly executes 'getOperationsFunction()' until it returns exactly one operation for each +// shard in a sharded collection or exactly one operation for an unsharded collection. +function awaitOperations(getOperationsFunction) { + let operations; assert.soon( function() { - let aggRes = - db.getSiblingDB("admin") - .aggregate([ - {$currentOp: {}}, - {$match: {ns: "test.currentOp_cursor", "command.comment": TestData.comment}} - ]) - .toArray(); - return aggRes.length >= 1; + const numShards = FixtureHelpers.numberOfShardsForCollection(coll); + operations = getOperationsFunction(); + + // No shard should have more than one operation matching the query comment. First check + // that the total number of operations is no greater than the total number of shards. + assert.lte(operations.length, numShards, operations); + + // Also explicitly check that each shard appears no more than once in the list of + // operations. + const distinctShardNames = new Set(operations.map(op => "shard" in op ? op.shard : "")); + assert.eq(operations.length, distinctShardNames.size, {operations, numShards}); + + if (operations.length < numShards) { + print(`Found ${operations.length} operation(s); waiting until there are ${ + numShards} operation(s)`); + return false; + } else if (operations.some(op => op.op !== "getmore" && "cursor" in op && + op.cursor.batchSize === 0)) { + print(`Found command with empty 'batchSize' value; waiting for getmore: ${ + tojson(operations)}`); + return false; + } + + return true; }, function() { return "Failed to find parallel shell operation in $currentOp output: " + tojson(db.currentOp()); }); - return awaitShell; -} -// Test that the currentOp server command truncates long operations with a warning logged. -const serverCommandTest = startShellWithOp("currentOp_server"); -res = db.adminCommand({ - currentOp: true, - $and: [{"ns": "test.currentOp_cursor"}, {"command.comment": "currentOp_server"}] -}); + return operations; +} -if (FixtureHelpers.isMongos(db) && FixtureHelpers.isSharded(coll)) { - // Assert currentOp truncation behavior for each shard in the cluster. - assert(res.inprog.length >= 1, res); - res.inprog.forEach((result) => { - assert.eq(result.op, "getmore", result); - assert(result.cursor.originatingCommand.hasOwnProperty("$truncated"), result); - }); -} else { - // Assert currentOp truncation behavior for unsharded collections. - assert.eq(res.inprog.length, 1, res); - assert.eq(res.inprog[0].op, "command", res); - assert(res.inprog[0].command.hasOwnProperty("$truncated"), res); +function getCommandFromCurrentOpEntry(entry) { + if (entry.op === "command" && "command" in entry) { + return entry.command; + } else if (entry.op === "getmore" && "cursor" in entry && + "originatingCommand" in entry.cursor) { + return entry.cursor.originatingCommand; + } else { + assert(false, entry); + } } -const log = FixtureHelpers.getPrimaryForNodeHostingDatabase(db).adminCommand({getLog: "global"}); -assert(/will be truncated/.test(log.log)); +const comment = "long_running_aggregation"; +const awaitShell = + startParallelShell(funWithArgs(startLongRunningAggregation, coll.getName(), comment)); -res.inprog.forEach((op) => { - assert.commandWorked(db.killOp(op.opid)); -}); +const filter = { + ns: coll.getFullName(), + "command.comment": comment, -serverCommandTest(); - -// Test that the db.currentOp() shell helper does not truncate ops. -const shellHelperTest = startShellWithOp("currentOp_shell"); -res = db.currentOp({"ns": "test.currentOp_cursor", "command.comment": "currentOp_shell"}); - -if (FixtureHelpers.isMongos(db) && FixtureHelpers.isSharded(coll)) { - assert(res.inprog.length >= 1, res); - res.inprog.forEach((result) => { - assert.eq(result.op, "getmore", result); - assert(!result.cursor.originatingCommand.hasOwnProperty("$truncated"), result); - }); -} else { - assert.eq(res.inprog.length, 1, res); - assert(!res.inprog[0].command.hasOwnProperty("$truncated"), res); -} + // On the replica set endpoint, currentOp reports both router and shard operations. So filter + // out one of them. + role: TestData.testingReplicaSetEndpoint ? "ClusterRole{router}" : {$exists: false} +}; -res.inprog.forEach((op) => { - assert.commandWorked(db.killOp(op.opid)); +// The 'currentOp' shell helper should _not_ truncate the command. +const operationsViaCurrentOpShellHelper = awaitOperations(function() { + return db.currentOp(filter).inprog; }); - -shellHelperTest(); +assert(operationsViaCurrentOpShellHelper.every(op => { + const command = getCommandFromCurrentOpEntry(op); + return !("$truncated" in command) && command.aggregate == coll.getName(); +}), + operationsViaCurrentOpShellHelper); + +// Finish the test by killing the long-running aggregation pipeline and joining the parallel shell +// that launched it. +for (let op of operationsViaCurrentOpShellHelper) { + assert.commandWorked(db.killOp(op.opid)); +} +awaitShell(); })(); diff --git a/jstests/core/elemmatch_index.js b/jstests/core/elemmatch_index.js index a1941620a48..f0aaf35cdba 100644 --- a/jstests/core/elemmatch_index.js +++ b/jstests/core/elemmatch_index.js @@ -80,6 +80,12 @@ assertIndexResults(coll, {a: {$elemMatch: {"": {$elemMatch: {"b.c": "x"}}}}}, fa // Tests $elemMatch with supporting index and no path components that are empty strings. assertIndexResults(coll, {a: {$elemMatch: {"b.c": "x"}}}, true, 1); assertIndexResults(coll, {a: {$all: [{$elemMatch: {"b.c": "x"}}]}}, true, 1); + +// Tests that $elemMatch with path components that are empty strings are correctly handled in the +// plan enumerator in case if an index is used on another predicate of the query. +assertIndexResults(coll, {$and: [{"a.b.c": "x"}, {"": {$elemMatch: {"a.b.c": "x"}}}]}, true, 0); + +assertIndexResults(coll, {$and: [{"a.b.c": "x"}, {"a.b.c": {$elemMatch: {"": "x"}}}]}, true, 0); })(); (function() { diff --git a/jstests/core/empty_ts.js b/jstests/core/empty_ts.js new file mode 100644 index 00000000000..926de0fd790 --- /dev/null +++ b/jstests/core/empty_ts.js @@ -0,0 +1,134 @@ +/** + * Test how inserts and updates behave with "Timestamp(0,0)" values. + * + * @tags: [ + * requires_fcv_60, + * ] + */ +(function() { +"use strict"; + +const coll = db.jstests_core_empty_ts; +const emptyTs = Timestamp(0, 0); + +coll.drop(); + +// Insert several documents. For the first document inserted (_id=101), the empty timestamp value +// in field "a" should get replaced with the current timestamp. +assert.commandWorked(coll.insert({_id: 101, a: emptyTs})); +assert.commandWorked(coll.insert({_id: 102, a: 1})); +assert.commandWorked(coll.insert({_id: 103, a: 2})); +assert.commandWorked(coll.insert({_id: 106, a: 3})); +assert.commandWorked(coll.insert({_id: 107, a: 4})); +assert.commandWorked(coll.insert({_id: 108, a: 5})); +assert.commandWorked(coll.insert({_id: 109, a: 6})); +assert.commandWorked(coll.insert({_id: 110, a: 7})); +assert.commandWorked(coll.insert({_id: 111, a: 8})); + +// Use a replacement-style update to update _id=102. This should result in field "a" being set to +// the current timestamp. +// +// This is an example of how the special "empty timestamp" behavior works with a replacement-style +// update. +assert.commandWorked(coll.update({_id: 102}, {a: emptyTs})); + +// Use a replacement-style findAndModify to update _id=103. This should result in field "a" being +// set to the current timestamp. +let findAndModifyResult = coll.findAndModify({query: {_id: 103}, update: {a: emptyTs}}); +assert.eq(findAndModifyResult, {_id: 103, a: 2}); + +// Do a replacement-style update to add a new document with _id=104. This should result in field "a" +// being set to the current timestamp. +assert.commandWorked(coll.update({_id: 104}, {a: emptyTs}, {upsert: true})); + +// Do a replacement-style findAndModify to add a new document with _id=105. This should result in +// field "a" being set to the current timestamp. +findAndModifyResult = coll.findAndModify({query: {_id: 105}, update: {a: emptyTs}, upsert: true}); +assert.eq(findAndModifyResult, null); + +// For the rest of the commands below, the empty timestamp values stored in field "a" should be +// preserved as-is. + +// Do an update-operator-style update to update _id=106. +assert.commandWorked(coll.update({_id: 106}, {$set: {a: emptyTs}})); + +// Do an update-operator-style findAndModify to update _id=107. +findAndModifyResult = coll.findAndModify({query: {_id: 107}, update: {$set: {a: emptyTs}}}); +assert.eq(findAndModifyResult, {_id: 107, a: 4}); + +// Do a pipeline-style update to update _id=108. +assert.commandWorked(coll.update({_id: 108}, [{$addFields: {a: emptyTs}}])); + +// Do a pipeline-style findAndModify to update _id=109. +findAndModifyResult = coll.findAndModify({query: {_id: 109}, update: [{$addFields: {a: emptyTs}}]}); +assert.eq(findAndModifyResult, {_id: 109, a: 6}); + +// Do a pipeline-style update with $internalApplyOplogUpdate to update _id=110. +assert.commandWorked(coll.update( + {_id: 110}, [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}])); + +// Do a pipeline-style findAndModify with $internalApplyOplogUpdate to update _id=111. +findAndModifyResult = coll.findAndModify({ + query: {_id: 111}, + update: [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}] +}); +assert.eq(findAndModifyResult, {_id: 111, a: 8}); + +// Do an update-operator-style update to add a new document with _id=112. +assert.commandWorked(coll.update({_id: 112}, {$set: {a: emptyTs}}, {upsert: true})); + +// Do an update-operator-style findAndModify to add a new document with _id=113. +findAndModifyResult = + coll.findAndModify({query: {_id: 113}, update: {$set: {a: emptyTs}}, upsert: true}); +assert.eq(findAndModifyResult, null); + +// Do a pipeline-style update to add a new document with _id=114. +assert.commandWorked(coll.update({_id: 114}, [{$addFields: {a: emptyTs}}], {upsert: true})); + +// Do a pipeline-style findAndModify to add a new document with _id=115. +findAndModifyResult = + coll.findAndModify({query: {_id: 115}, update: [{$addFields: {a: emptyTs}}], upsert: true}); +assert.eq(findAndModifyResult, null); + +// Do a pipline-style update with $internalApplyOplogUpdate to add a new document _id=116. +assert.commandWorked( + coll.update({_id: 116}, + [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + {upsert: true})); + +// Do pipeline-style findAndModify with $internalApplyOplogUpdate to add a new document _id=117. +findAndModifyResult = coll.findAndModify({ + query: {_id: 117}, + update: [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + upsert: true +}); +assert.eq(findAndModifyResult, null); + +// Verify that all the insert, update, and findAndModify commands behaved the way we expected. +for (let i = 101; i <= 117; ++i) { + let result = coll.findOne({_id: i}); + + if (i >= 106) { + assert.eq(tojson(result.a), tojson(emptyTs), "_id=" + i); + } else { + assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + i); + } +} + +// Insert a document with _id=Timestamp(0,0). +assert.commandWorked(coll.insert({_id: emptyTs, a: 9})); + +// Verify the document we just inserted can be retrieved using the filter "{_id: Timestamp(0,0)}". +let result = coll.findOne({_id: emptyTs}); +assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); +assert.eq(tojson(result.a), tojson(9), "_id=" + tojson(emptyTs)); + +// Do a replacement-style update on the document. +assert.commandWorked(coll.update({_id: emptyTs}, {_id: emptyTs, a: emptyTs})); + +// Verify the document we just updated can still be retrieved using "{_id: Timestamp(0,0)}", and +// verify that field "a" was set to the current timestamp. +result = coll.findOne({_id: emptyTs}); +assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); +assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + tojson(emptyTs)); +}()); diff --git a/jstests/core/explain_agg_write_concern.js b/jstests/core/explain_agg_write_concern.js index 9ff556489fa..ec246140abe 100644 --- a/jstests/core/explain_agg_write_concern.js +++ b/jstests/core/explain_agg_write_concern.js @@ -5,6 +5,7 @@ // assumes_unsharded_collection, // assumes_write_concern_unchanged, // does_not_support_stepdowns, +// references_foreign_collection, // requires_non_retryable_commands, // ] diff --git a/jstests/core/explain_skip.js b/jstests/core/explain_skip.js new file mode 100644 index 00000000000..b6e01d6a436 --- /dev/null +++ b/jstests/core/explain_skip.js @@ -0,0 +1,28 @@ +/** + * Tests that explain reports the skip stage statistics correctly. + * + * @tags: [ + * assumes_unsharded_collection, + * assumes_against_mongod_not_mongos, + * ] + */ +load("jstests/libs/analyze_plan.js"); // For getWinningPlan. + +var coll = db.explain_skip; +coll.drop(); + +assert.commandWorked(coll.insert({a: 1, b: 1})); +assert.commandWorked(coll.insert({a: 1, b: 2})); +assert.commandWorked(coll.insert({a: 1, b: 3})); +assert.commandWorked(coll.insert({a: 2, b: 1})); +assert.commandWorked(coll.insert({a: 2, b: 2})); +assert.commandWorked(coll.insert({a: 2, b: 3})); + +assert.commandWorked(coll.createIndex({a: 1})); +assert.commandWorked(coll.createIndex({b: 1})); + +var explain = coll.find({a: 1, b: 1}).sort({_id: 1}).skip(5).explain(); +var winningPlan = getWinningPlan(explain.queryPlanner); +var skipStage = getPlanStage(winningPlan, "SKIP"); +assert.neq(null, skipStage, explain); +assert.eq(5, skipStage.skipAmount, explain); diff --git a/jstests/core/fts_mix.js b/jstests/core/fts_mix.js index 5942a85ec2c..f8bf850fb29 100644 --- a/jstests/core/fts_mix.js +++ b/jstests/core/fts_mix.js @@ -1,4 +1,3 @@ - (function() { load("jstests/libs/fts.js"); load("jstests/aggregation/extras/utils.js"); // For resultsEq. diff --git a/jstests/core/index_filter_commands.js b/jstests/core/index_filter_commands.js index 6c5ff8920c8..6e5f5c5726a 100644 --- a/jstests/core/index_filter_commands.js +++ b/jstests/core/index_filter_commands.js @@ -30,7 +30,8 @@ * assumes_read_preference_unchanged, * assumes_unsharded_collection, * does_not_support_stepdowns, - * requires_fcv_60 + * requires_fcv_60, + * references_foreign_collection, * ] */ diff --git a/jstests/core/index_stats.js b/jstests/core/index_stats.js index ce5c9901e48..b0abd2e35a1 100644 --- a/jstests/core/index_stats.js +++ b/jstests/core/index_stats.js @@ -12,6 +12,7 @@ // # Tenant migrations passthrough suites automatically retry operations on TenantMigrationAborted // # errors. // tenant_migration_incompatible, +// references_foreign_collection, // ] (function() { diff --git a/jstests/core/json_schema/json_schema.js b/jstests/core/json_schema/json_schema.js index c1e06344498..a69db77c60d 100644 --- a/jstests/core/json_schema/json_schema.js +++ b/jstests/core/json_schema/json_schema.js @@ -1,7 +1,7 @@ // listCollections tests expect that a collection is not implicitly created after a drop. // @tags: [ // assumes_no_implicit_collection_creation_after_drop, -// requires_non_retryable_commands, +// requires_non_retryable_commands // ] /** diff --git a/jstests/core/json_schema/misc_validation.js b/jstests/core/json_schema/misc_validation.js index 58e76fc0d68..569d98be258 100644 --- a/jstests/core/json_schema/misc_validation.js +++ b/jstests/core/json_schema/misc_validation.js @@ -19,6 +19,7 @@ * requires_replication, * # This test depends on hardcoded database name equality. * tenant_migration_incompatible, + * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/negated_geo_queries.js b/jstests/core/negated_geo_queries.js new file mode 100644 index 00000000000..e6c0cfbbf03 --- /dev/null +++ b/jstests/core/negated_geo_queries.js @@ -0,0 +1,93 @@ +/** + * With a normal ascending index on a geospatial field, {$not: {$geoWithin: <>}} and + * {$not: {$geoIntersects: <>}} queries should return proper results. Previously, those queries + * failed due to attempting to build geospatial index bounds on the non-geo index. See SERVER-92193 + * for more details. + */ +(function() { +"use strict"; + +const coll = db.negated_geo_queries; +coll.drop(); + +assert.commandWorked(coll.insert( + {loc: {type: "Polygon", coordinates: [[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]]}, a: 0})); +assert.commandWorked(coll.insert( + {loc: {type: "Polygon", coordinates: [[[30, 0], [30, 1], [32, 1], [32, 0], [30, 0]]]}, a: 0})); +assert.commandWorked(coll.insert( + {loc: {type: "Polygon", coordinates: [[[-55, 10], [-35, 15], [-45, 10], [-55, 10]]]}, a: 1})); + +const targetPolygonWithin = { + type: "Polygon", + coordinates: [[[-5, -5], [5, -5], [5, 5], [-5, 5], [-5, -5]]] +}; + +const targetLineIntersect = { + type: "LineString", + coordinates: [[29, -1], [31, 1]] +}; + +function runTest() { + // Tests $geoWithin. + let geoQuery = {loc: {$geoWithin: {$geometry: targetPolygonWithin}}}; + let res = coll.find(geoQuery); + assert.eq(res.itcount(), 1); + + // Tests $geoWithin under a $not. + geoQuery = {loc: {$not: {$geoWithin: {$geometry: targetPolygonWithin}}}}; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 2); + + // Tests $geoIntersects. + geoQuery = {loc: {$geoIntersects: {$geometry: targetLineIntersect}}}; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 1); + + // Tests $geoIntersects under a $not. + geoQuery = {loc: {$not: {$geoIntersects: {$geometry: targetLineIntersect}}}}; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 2); + + // Tests an $and of the negated $geoWithin and negated $geoIntersects. + geoQuery = { + $and: [ + {loc: {$not: {$geoWithin: {$geometry: targetPolygonWithin}}}}, + {loc: {$not: {$geoIntersects: {$geometry: targetLineIntersect}}}} + ] + }; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 1); + + // Tests the same logic as above written with a $nor. + geoQuery = { + $nor: [ + {loc: {$geoWithin: {$geometry: targetPolygonWithin}}}, + {loc: {$geoIntersects: {$geometry: targetLineIntersect}}} + ] + }; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 1); + + // Tests an $and of the negated $geoWithin and an expression on a non-geo field. + geoQuery = {$and: [{loc: {$not: {$geoWithin: {$geometry: targetPolygonWithin}}}}, {a: 0}]}; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 1); + + // Tests an $and of the negated $geoIntersects and an expression on a non-geo field. + geoQuery = {$and: [{loc: {$not: {$geoIntersects: {$geometry: targetLineIntersect}}}}, {a: 0}]}; + res = coll.find(geoQuery); + assert.eq(res.itcount(), 1); +} + +// Run test with just a simple btree index. +assert.commandWorked(coll.createIndex({loc: 1})); +runTest(); + +// Run test with a simple btree index and a geo index. +assert.commandWorked(coll.createIndex({loc: "2dsphere"})); +runTest(); + +// Run test with just a geo index. +assert.commandWorked(coll.dropIndex({loc: 1})); +runTest(); +})(); diff --git a/jstests/core/notablescan.js b/jstests/core/notablescan.js deleted file mode 100644 index e99ecd013e8..00000000000 --- a/jstests/core/notablescan.js +++ /dev/null @@ -1,86 +0,0 @@ -// check notablescan mode -// -// @tags: [ -// # The test runs commands that are not allowed with security token: setParameter. -// not_allowed_with_security_token, -// assumes_against_mongod_not_mongos, -// # This test attempts to perform read operations after having enabled the notablescan server -// # parameter. The former operations may be routed to a secondary in the replica set, whereas the -// # latter must be routed to the primary. -// assumes_read_preference_unchanged, -// assumes_superuser_permissions, -// does_not_support_stepdowns, -// # Server parameters are stored in-memory only so are not transferred onto the recipient. This -// # test sets the server parameter "notablescan" to force the node to not execute queries that -// # require a collection scan and return an error. -// tenant_migration_incompatible, -// ] - -(function() { -load("jstests/libs/analyze_plan.js"); -load("jstests/libs/collection_drop_recreate.js"); - -function checkError(err) { - assert.includes(err.toString(), "'notablescan'"); -} - -const colName = jsTestName(); -let coll = db.getCollection(colName); -coll.drop(); - -assert.commandWorked(db.adminCommand({setParameter: 1, notablescan: true})); - -{ - if (0) { - // TODO: SERVER-2222 This should actually throw an error as it performs a collection - // scan. - assert.throws(function() { - coll.find({a: 1}).toArray(); - }); - } - - coll.insert({a: 1}); - let err = assert.throws(function() { - coll.count({a: 1}); - }); - checkError(err); - - // TODO: SERVER-2222 This should actually throw an error as it performs a collection scan. - assert.eq(1, coll.find({}).itcount()); - - err = assert.throws(function() { - coll.find({a: 1}).toArray(); - }); - checkError(err); - - err = assert.throws(function() { - coll.find({a: 1}).hint({$natural: 1}).toArray(); - }); - assert.includes(err.toString(), "$natural"); - checkError(err); - - coll.createIndex({a: 1}); - assert.eq(0, coll.find({a: 1, b: 1}).itcount()); - assert.eq(1, coll.find({a: 1, b: null}).itcount()); -} - -{ // Run the testcase with a clustered index. - assertDropAndRecreateCollection(db, colName, {clusteredIndex: {key: {_id: 1}, unique: true}}); - coll = db.getCollection(colName); - assert.commandWorked(coll.insert({_id: 22})); - assert.eq(1, coll.find({_id: 22}).itcount()); - let plan = coll.find({_id: 22}).explain(); - // Make sure the plan has a clustered index scan. - assert(isClusteredIxscan(db, plan)); - - // Make sure the same works with an aggregate. - assert.eq(1, coll.aggregate([{$match: {_id: 22}}]).itcount()); - plan = coll.explain().aggregate([{$match: {_id: 22}}]); - // Make sure the plan has a clustered index scan. - assert(isClusteredIxscan(db, plan)); - assert.commandWorked( - db.runCommand({aggregate: colName, pipeline: [{$match: {_id: 22}}], cursor: {}})); -} -// Set it back to the original value. -assert.commandWorked(db.adminCommand({setParameter: 1, notablescan: false})); -})(); diff --git a/jstests/core/opcounters_write_cmd.js b/jstests/core/opcounters_write_cmd.js index 6f74185e9f7..9e0d68cd928 100644 --- a/jstests/core/opcounters_write_cmd.js +++ b/jstests/core/opcounters_write_cmd.js @@ -2,6 +2,7 @@ // @tags: [ // uses_multiple_connections, // assumes_standalone_mongod, +// inspects_command_opcounters, // ] // Legacy write mode test also available at jstests/gle. diff --git a/jstests/core/plan_cache_list_plans.js b/jstests/core/plan_cache_list_plans.js index ca571a9e39e..802a25635e0 100644 --- a/jstests/core/plan_cache_list_plans.js +++ b/jstests/core/plan_cache_list_plans.js @@ -12,6 +12,7 @@ // assumes_unsharded_collection, // does_not_support_stepdowns, // inspects_whether_plan_cache_entry_is_active, +// references_foreign_collection, // ] (function() { diff --git a/jstests/core/profile_agg.js b/jstests/core/profile_agg.js index d6611fa98ab..b1ce28dd696 100644 --- a/jstests/core/profile_agg.js +++ b/jstests/core/profile_agg.js @@ -1,6 +1,7 @@ // @tags: [ // does_not_support_stepdowns, // requires_profiling, +// references_foreign_collection, // ] // Confirms that profiled aggregation execution contains all expected metrics with proper values. diff --git a/jstests/core/query/find_all_values_at_path_expression.js b/jstests/core/query/find_all_values_at_path_expression.js new file mode 100644 index 00000000000..22a26c593ed --- /dev/null +++ b/jstests/core/query/find_all_values_at_path_expression.js @@ -0,0 +1,33 @@ +/** + * Tests that $_internalFindAllValuesAtPath asserts when provided a non-string input. + */ +const coll = db.find_all_values_at_path_expression; +coll.drop(); + +// In query stats, this fails with a different error code. +const errorCodes = [9567004, 9423101]; + +// Insert some documents because running aggregation on a non-existent collection on mongos will +// return empty instead of erroring. +let documents = [{a: 4}, {a: 5, b: 1}, {a: 0, b: 1}, {a: 0, b: 1}, {a: 2, b: {c: 1}}]; +assert.commandWorked(coll.insert(documents)); +assert.commandFailedWithCode(db.runCommand({ + aggregate: coll.getName(), + pipeline: [ + {$replaceRoot: {newRoot: {$_internalFindAllValuesAtPath: null}}}, + {$unwind: {path: "$_internalUnwoundField", preserveNullAndEmptyArrays: true}}, + {$group: {_id: null, distinct: {$addToSet: "$<key>"}}} + ], + cursor: {} +}), + errorCodes); +assert.commandFailedWithCode(db.runCommand({ + aggregate: coll.getName(), + pipeline: [ + {$replaceRoot: {newRoot: {$_internalFindAllValuesAtPath: 12345}}}, + {$unwind: {path: "$_internalUnwoundField", preserveNullAndEmptyArrays: true}}, + {$group: {_id: null, distinct: {$addToSet: "$<key>"}}} + ], + cursor: {} +}), + errorCodes); diff --git a/jstests/core/query/internal_strip_invalid_assignment.js b/jstests/core/query/internal_strip_invalid_assignment.js index 4385e6730a4..b8868c6e951 100644 --- a/jstests/core/query/internal_strip_invalid_assignment.js +++ b/jstests/core/query/internal_strip_invalid_assignment.js @@ -3,6 +3,11 @@ * assertion failure when we strip its index assignment from special indexes. * * Originally intended to reproduce SERVER-82717. + * + * @tags: [ + * # Explain command does not support read concerns other than local. + * assumes_read_concern_local, + * ] */ (function() { load('jstests/libs/analyze_plan.js'); // for planHasStage diff --git a/jstests/core/rename_collection_system_db.js b/jstests/core/rename_collection_system_db.js index 8d46ad27a04..e9e166914c0 100644 --- a/jstests/core/rename_collection_system_db.js +++ b/jstests/core/rename_collection_system_db.js @@ -16,10 +16,9 @@ systemUsers.drop(); coll.drop(); coll.insert({}); -// system.foo isn't in the allowlist so it can't be renamed to or from +// system.foo and system.users aren't in the allowlist so they can't be renamed to or from assert.commandFailed(coll.renameCollection(systemFoo.getName())); assert.commandFailed(systemFoo.renameCollection(coll.getName())); -// system.users is allowlisted so these should work -assert.commandWorked(coll.renameCollection(systemUsers.getName())); -assert.commandWorked(systemUsers.renameCollection(coll.getName())); +assert.commandFailed(coll.renameCollection(systemUsers.getName())); +assert.commandFailed(systemUsers.renameCollection(coll.getName())); diff --git a/jstests/core/timeseries/libs/timeseries.js b/jstests/core/timeseries/libs/timeseries.js index d9e22926520..540e0bcea55 100644 --- a/jstests/core/timeseries/libs/timeseries.js +++ b/jstests/core/timeseries/libs/timeseries.js @@ -11,6 +11,20 @@ var TimeseriesTest = class { return FeatureFlagUtil.isEnabled(conn, "TimeseriesBucketCompression"); } + static bucketsMayHaveMixedSchemaData(coll) { + const catalog = coll.aggregate([{$listCatalog: {}}]).toArray()[0]; + const tsMixedSchemaOptionNewFormat = catalog.md.options.storageEngine && + catalog.md.options.storageEngine.wiredTiger && + catalog.md.options.storageEngine.wiredTiger.configString; + // TODO SERVER-92533 Simplify once SERVER-91195 is backported to all supported branches + if (tsMixedSchemaOptionNewFormat !== undefined) { + return tsMixedSchemaOptionNewFormat == + "app_metadata=(timeseriesBucketsMayHaveMixedSchemaData=true)"; + } else { + return catalog.md.timeseriesBucketsMayHaveMixedSchemaData; + } + } + /** * Returns whether time-series updates and deletes are supported. */ diff --git a/jstests/core/timeseries/timeseries_collmod.js b/jstests/core/timeseries/timeseries_collmod.js index 69fd7dbeb82..07c9166b632 100644 --- a/jstests/core/timeseries/timeseries_collmod.js +++ b/jstests/core/timeseries/timeseries_collmod.js @@ -19,6 +19,17 @@ assert.commandWorked( db.createCollection(collName, {timeseries: {timeField: "time", granularity: 'seconds'}})); assert.commandWorked(coll.createIndex({"time": 1})); +// Setting prepareUnique should return an error on a time-series collection. +assert.commandFailedWithCode( + db.runCommand( + {"collMod": collName, "index": {"keyPattern": {"time": 1}, "prepareUnique": true}}), + ErrorCodes.InvalidOptions); + +assert.commandFailedWithCode( + db.runCommand( + {"collMod": collName, "index": {"keyPattern": {"time": 1}, "prepareUnique": false}}), + ErrorCodes.InvalidOptions); + // Tries to convert a time-series secondary index to TTL index. assert.commandFailedWithCode( db.runCommand( @@ -56,4 +67,4 @@ assert.commandWorked(db.runCommand({"collMod": collName, "expireAfterSeconds": 6 // Successfully sets the granularity for a time-series collection. assert.commandWorked( db.runCommand({"collMod": collName, "timeseries": {"granularity": "minutes"}})); -})();
\ No newline at end of file +})(); diff --git a/jstests/core/timeseries/timeseries_computed_field.js b/jstests/core/timeseries/timeseries_computed_field.js new file mode 100644 index 00000000000..fab03d1a3b0 --- /dev/null +++ b/jstests/core/timeseries/timeseries_computed_field.js @@ -0,0 +1,856 @@ +/** + * Test use of computed fields in aggregations on time series collections. + * @tags: [ + * requires_timeseries, + * does_not_support_stepdowns, + * directly_against_shardsvrs_incompatible, + * # "Explain of a resolved view must be executed by mongos" + * directly_against_shardsvrs_incompatible, + * # Some suites use mixed-binary cluster setup where some nodes might have the flag enabled while + * # others -- not. For this test we need control over whether the flag is set on the node that + * # ends up executing the query. + * assumes_standalone_mongod + * ] + */ +load("jstests/core/timeseries/libs/timeseries.js"); + +TimeseriesTest.run((insert) => { + const datePrefix = 1680912440; + + let coll = db.timeseries_computed_field; + const bucketsColl = db.getCollection('system.buckets.' + coll.getName()); + + const timeFieldName = 'time'; + const metaFieldName = 'measurement'; + + coll.drop(); + assert.commandWorked(db.createCollection(coll.getName(), { + timeseries: {timeField: timeFieldName, metaField: metaFieldName}, + })); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + insert(coll, { + _id: 0, + [timeFieldName]: new Date(datePrefix + 100), // ISODate("1970-01-20T10:55:12.540Z") + [metaFieldName]: "cpu", + topLevelScalar: 123, + topLevelScalarDouble: 123.8778645, + topLevelLargeNumber: 12345678912345, + topLevelArray: [1, 2, 3, 4], + arrOfObj: [{x: 1}, {x: 2}, {x: 3}, {x: 4}], + obj: {a: 123}, + length: 0, + }); + insert(coll, { + _id: 1, + [timeFieldName]: new Date(datePrefix + 200), // ISODate("1970-01-20T10:55:12.640Z") + [metaFieldName]: "cpu", + topLevelScalar: 456, + topLevelScalarDouble: 546.76858699, + topLevelArray: [101, 102, 103, 104], + arrOfObj: [{x: 101}, {x: 102}, {x: 103}, {x: 104}], + obj: {a: 456}, + length: 23, + }); + // Insert a document that will be placed in a different bucket. + insert(coll, { + _id: 2, + [timeFieldName]: new Date(datePrefix + 300), + [metaFieldName]: "gpu", + length: -2, + }); + + // Computing a field on a dotted path which is an array, then grouping on it. Note that the + // semantics for setting a computed field on a dotted array path are particularly strange, but + // should be preserved for backwards compatibility. + { + const res = coll.aggregate([ + {$addFields: {"arrOfObj.x": {$trim: {input: "test string"}}}}, + {$match: {topLevelScalar: {$gte: 0}}}, + {$group: {_id: null, max: {$max: "$arrOfObj.x"}}} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].max, ["test string", "test string", "test string", "test string"], res); + } + + { + const res = coll.aggregate([ + {$addFields: {"arrOfObj.x": {$add: ["$topLevelScalar", 1]}}}, + {$match: {topLevelScalar: {$gte: 0}}}, + {$group: {_id: null, max: {$max: "$arrOfObj.x"}}} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].max, [457, 457, 457, 457], res); + } + + // Computing a field and then filtering by it. + { + const res = coll.aggregate([ + {$addFields: {"arrOfObj.x": {$trim: {input: "test string"}}}}, + {$match: {"arrOfObj.x": "test string"}}, + {$group: {_id: null, max: {$max: "$arrOfObj.x"}}} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].max, ["test string", "test string", "test string", "test string"], res); + } + + { + // Computing a field based on a dotted path which does not traverse arrays. + const res = coll.aggregate([ + {$addFields: {"computedA": {$add: ["$obj.a", 1]}}}, + // Only one document should have a value where obj.a was 457 + // (456 + 1). + {$match: {"computedA": 457}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + // Include an unnecessary computed field before counting the number of documents. + const res = coll.aggregate([{$addFields: {"a": 1}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 3, res); + } + + // mathematical expressions + { + let pipeline = [ + {$addFields: {"computedA": {$add: ["$topLevelScalar", 1]}}}, + {$match: {"computedA": 457}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$subtract: ["$topLevelScalar", 1]}}}, + {$match: {"computedA": 455}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$multiply: ["$topLevelScalar", 10]}}}, + {$match: {"computedA": 4560}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$divide: ["$topLevelScalar", 2]}}}, + {$match: {"computedA": 228}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$add: [1, "$topLevelScalar"]}}}, + {$match: {"computedA": 457}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$subtract: [200, "$topLevelScalar"]}}}, + {$match: {"computedA": 77}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$multiply: [10, "$topLevelScalar"]}}}, + {$match: {"computedA": 4560}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$divide: [4560, "$topLevelScalar"]}}}, + {$match: {"computedA": 10}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + let pipeline = [ + {$addFields: {"computedA": {$add: ["$topLevelScalar", "$topLevelScalar"]}}}, + {$match: {"computedA": 912}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$subtract: ["$topLevelScalar", "$topLevelScalar"]}}}, + {$match: {"computedA": 0}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$multiply: ["$topLevelScalar", "$topLevelScalar"]}}}, + {$match: {"computedA": 15129}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$divide: ["$topLevelScalar", "$topLevelScalar"]}}}, + {$match: {"computedA": 1}}, + {$count: "count"} + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + let pipeline = [ + { + $addFields: { + "hourDiff": { + $dateDiff: { + "startDate": "$time", + "endDate": new Date("1970-01-21"), + "unit": "hour" + } + } + } + }, + {$match: {"hourDiff": {$gte: 12}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "open": {"$first": "$topLevelScalar"}, + } + } + ]; + + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "open": 123}], + coll.aggregate(pipeline).toArray()); + } + + { + let pipeline = [ + {$addFields: {"mt": {$multiply: ["$topLevelScalar", 10]}}}, + {$match: {"mt": {$gte: 2000}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "open": {"$first": "$topLevelScalar"}, + } + } + ]; + + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "open": 456}], + coll.aggregate(pipeline).toArray()); + } + + { + let pipeline = [ + { + $addFields: { + "hourDiff": { + $dateDiff: { + "startDate": "$time", + "endDate": new Date("1970-01-21"), + "unit": "hour" + } + } + } + }, + {$match: {$or: [{topLevelScalar: {$gt: 200, $lt: 800}}, {hourDiff: {$gte: 12}}]}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "open": {"$first": "$topLevelScalar"}, + } + } + ]; + + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "open": 123}], + coll.aggregate(pipeline).toArray()); + } + + { + // Try a project stage which adds and remove subfields. + const res = coll.aggregate([ + {$project: {"obj.newField": "$topLevelScalar"}}, + {$project: {"_id": 0, "obj.a": 0}} + ]) + .toArray(); + assert.eq(res.length, 3, res); + assert.eq(res[0], {"obj": {"newField": 123}}, res); + assert.eq(res[1], {"obj": {"newField": 456}}, res); + assert.eq(res[2], {"obj": {}}, res); + } + + { + // Try a replaceRoot stage which remove all fields. + const res = coll.aggregate([ + {$match: {"time": {$gte: new Date(datePrefix + 200)}}}, + {$addFields: {}}, + {$project: {"measurement0": {$floor: "$topLevelScalar"}}}, + {$replaceRoot: {newRoot: {}}} + ]) + .toArray(); + assert.eq(res.length, 2, res); + assert.eq(res[0], {}, res); + assert.eq(res[1], {}, res); + } + + { + // Try a $match that works on fields that are not projected. + const res = + coll.aggregate([ + {$project: {"obj.a": 1}}, + {$match: {$or: [{"topLevelScalar": {$gt: 10}}, {$expr: {$literal: true}}]}}, + {$sort: {_id: 1}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 3, res); + } + + { + const res = coll.aggregate([{ + "$project": { + "t": { + "$dateDiff": { + "startDate": "$time", + "endDate": new Date(datePrefix + 150), + "unit": "millisecond" + } + } + } + }]) + .toArray(); + assert.eq(res.length, 3, res); + assert.docEq(res[0], {_id: 0, "t": NumberLong(50)}, res); + assert.docEq(res[1], {_id: 1, "t": NumberLong(-50)}, res); + assert.docEq(res[2], {_id: 2, "t": NumberLong(-150)}, res); + } + + { + const res = coll.aggregate([{ + "$project": { + "t": { + "$dateDiff": { + "startDate": new Date(datePrefix - 10), + "endDate": "$time", + "unit": "millisecond" + } + } + } + }]) + .toArray(); + assert.eq(res.length, 3, res); + assert.docEq(res[0], {_id: 0, "t": NumberLong(110)}, res); + assert.docEq(res[1], {_id: 1, "t": NumberLong(210)}, res); + assert.docEq(res[2], {_id: 2, "t": NumberLong(310)}, res); + } + + { + const res = coll.aggregate([ + { + $addFields: { + "computedField": { + $dateTrunc: { + date: "$time", + unit: "second", + } + } + } + }, + {$match: {computedField: new Date(datePrefix - 440)}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 3, res); + } + + { + const res = coll.aggregate([{$match: {topLevelScalar: {$exists: true}}}, {$count: "count"}]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = + coll.aggregate([{$match: {topLevelScalar: {$lt: 456}}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + const res = + coll.aggregate([{$match: {topLevelScalar: {$lte: 456}}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = + coll.aggregate([{$match: {topLevelScalar: {$gt: 123}}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + const res = + coll.aggregate([{$match: {topLevelScalar: {$gte: 123}}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = + coll.aggregate([{$match: {topLevelScalar: {$eq: 123}}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + const res = + coll.aggregate([{$match: {topLevelScalar: {$ne: 123}}}, {$count: "count"}]).toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = coll.aggregate([ + { + $match: { + $or: [ + {"topLevelScalar": 123}, + {"topLevelScalar": 456}, + ] + } + }, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = coll.aggregate([ + {$addFields: {"computedField": {$and: ["$topLevelScalar", true]}}}, + {$match: {computedField: true}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = coll.aggregate([ + {$addFields: {"computedField": {$and: ["$topLevelScalar", "$obj.a"]}}}, + {$match: {computedField: true}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = coll.aggregate([ + {$addFields: {"computedField": {$or: ["$topLevelScalar", false]}}}, + {$match: {computedField: true}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = coll.aggregate([ + {$addFields: {"computedField": {$or: ["$topLevelScalar", "$obj.a"]}}}, + {$match: {computedField: true}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = coll.aggregate([ + {$addFields: {"computedField": {$not: ["$topLevelScalar"]}}}, + {$match: {computedField: true}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + const res = + coll.aggregate([ + {$addFields: {"computedField": {$anyElementTrue: [["$topLevelScalar"]]}}}, + {$match: {computedField: true}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 2, res); + } + + { + const res = + coll.aggregate([ + {$match: {length: {$ne: 0}}}, + {$set: {"computedA": {$multiply: ["$topLevelScalar", "$topLevelScalar"]}}}, + {$addFields: {"ratio": {$divide: ["$computedA", "$length"]}}}, + {$match: {ratio: {$gt: 0}}}, + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0]._id, 1, res); + } + + { + let pipeline = [ + {$addFields: {"computedA": {$round: ["$topLevelScalarDouble", 2]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 670.65}], res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$round: ["$topLevelScalarDouble"]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 671}], res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$round: [2, "$topLevelScalarDouble"]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + assert.throws(() => coll.aggregate(pipeline)); + } + { + let pipeline = [ + {$addFields: {"computedA": {$round: ["$topLevelScalarDouble", "$topLevelScalar"]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + assert.throws(() => coll.aggregate(pipeline)); + } + + { + let pipeline = [ + {$addFields: {"computedA": {$trunc: ["$topLevelScalarDouble", 2]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 670.63}], res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$trunc: ["$topLevelScalarDouble"]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + const res = coll.aggregate(pipeline).toArray(); + assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 669}], res); + } + { + let pipeline = [ + {$addFields: {"computedA": {$trunc: [2, "$topLevelScalarDouble"]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + assert.throws(() => coll.aggregate(pipeline)); + } + { + let pipeline = [ + {$addFields: {"computedA": {$trunc: ["$topLevelScalarDouble", "$topLevelScalar"]}}}, + {$match: {"computedA": {$gte: 100}}}, + { + $group: { + "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, + "value": {$sum: "$computedA"}, + } + } + ]; + + assert.throws(() => coll.aggregate(pipeline)); + } + + { + let pipeline = [ + {$addFields: {md: {$mod: ["$topLevelScalar", 5]}}}, + { + $group: { + "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, + "total1": {$sum: "$md"}, + "total2": {$sum: "$topLevelScalar"} + } + } + ]; + + assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 4, "total2": 579}], + coll.aggregate(pipeline).toArray()); + } + + { + let pipeline = [ + {$addFields: {md: {$mod: [500, "$topLevelScalar"]}}}, + { + $group: { + "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, + "total1": {$sum: "$md"}, + "total2": {$sum: "$topLevelScalar"} + } + } + ]; + + assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 52, "total2": 579}], + coll.aggregate(pipeline).toArray()); + } + + { + let pipeline = [ + {$addFields: {md: {$mod: ["$topLevelScalar", "$topLevelScalar"]}}}, + { + $group: { + "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, + "total1": {$sum: "$md"}, + "total2": {$sum: "$topLevelScalar"} + } + } + ]; + + assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 0, "total2": 579}], + coll.aggregate(pipeline).toArray()); + } + + { + let pipeline = [ + {$addFields: {md: {$mod: ["$topLevelScalar", 5]}}}, + {$match: {md: {$mod: [4, 1]}}}, + { + $group: { + "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, + "total1": {$sum: "$md"}, + "total2": {$sum: "$topLevelScalar"} + } + } + ]; + + assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 1, "total2": 456}], + coll.aggregate(pipeline).toArray()); + } + { + let pipeline = [{$match: {topLevelLargeNumber: {$mod: [1, 0]}}}, {$count: "count"}]; + + const res = coll.aggregate(pipeline).toArray(); + + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + { + const res = + coll.aggregate([ + { + $addFields: { + "computedField": + {$dateAdd: {startDate: "$time", unit: "millisecond", amount: 100}} + } + }, + {$match: {computedField: new Date(datePrefix + 400)}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + const res = coll.aggregate([ + { + $addFields: { + "computedField": { + $dateSubtract: + {startDate: "$time", unit: "millisecond", amount: 100} + } + } + }, + {$match: {computedField: new Date(datePrefix)}}, + {$count: "count"} + ]) + .toArray(); + assert.eq(res.length, 1, res); + assert.eq(res[0].count, 1, res); + } + + { + // Test the case where a computed meta field is computed to missing. + const res = + coll.aggregate([{"$project": {"_id": 0, [metaFieldName]: "$$REMOVE"}}]).toArray(); + assert.eq(res.length, coll.count(), res); + for (let doc of res) { + assert.eq(doc, {}, res); + } + } + + { + // Test the case where a computed meta field is computed to missing because of a project + // out. + const res = coll.aggregate([ + {"$project": {[metaFieldName]: 0}}, + {"$project": {[metaFieldName]: "$" + metaFieldName}}, + {$match: {a: 1}} + ]) + .toArray(); + assert.eq(res.length, 0, res); + } + + { + // Test the case where a field is projected out and then projected back in. + const res = coll.aggregate([ + {"$project": {"_id": 0, [metaFieldName]: 1}}, + {"$project": {"_id": 0, [timeFieldName]: "$" + timeFieldName}}, + {"$project": {"_id": 1, [timeFieldName]: 1}} + ]) + .toArray(); + assert.eq(res.length, coll.count(), res); + for (let doc of res) { + assert.eq(doc, {}, res); + } + } + + { + // Test the case where a field is projected out and then projected back in. + const res = coll.aggregate([ + {"$project": {"_id": 0, [metaFieldName]: 1}}, + {"$project": {"_id": 0, [timeFieldName]: 1}} + ]) + .toArray(); + assert.eq(res.length, coll.count(), res); + for (let doc of res) { + assert.eq(doc, {}, res); + } + } + + { + // Test the case where a field is projected out and then projected back in. + const res = coll.aggregate([ + {"$project": {"_id": 0, [metaFieldName]: 0}}, + {"$project": {"_id": 0, [metaFieldName]: 1}}, + {"$project": {"_id": 0, [timeFieldName]: 1}} + ]) + .toArray(); + assert.eq(res.length, coll.count(), res); + for (let doc of res) { + assert.eq(doc, {}, res); + } + } +}); diff --git a/jstests/core/timeseries/timeseries_filter_extended_range.js b/jstests/core/timeseries/timeseries_filter_extended_range.js index 2b86e29c943..34018b3d2f4 100644 --- a/jstests/core/timeseries/timeseries_filter_extended_range.js +++ b/jstests/core/timeseries/timeseries_filter_extended_range.js @@ -1,6 +1,6 @@ /** - * Test that find/match type queries work properly on dates ouside the 32 bit epoch range, - * [1970-01-01 00:00:00 UTC - 2038-01-29 03:13:07 UTC]. + * Test that find and match type queries work properly on dates outside the 32 bit epoch range, + * [1970-01-01 00:00:00 UTC - 2038-01-19 03:14:07 UTC]. * * @tags: [ * # Refusing to run a test that issues an aggregation command with explain because it may @@ -12,174 +12,301 @@ * does_not_support_transactions, * # Explain of a resolved view must be executed by mongos. * directly_against_shardsvrs_incompatible, + * requires_fcv_60 * ] */ (function() { "use strict"; + const timeFieldName = "time"; +const standardDocs = [ + {[timeFieldName]: ISODate("1975-12-01")}, + {[timeFieldName]: ISODate("1980-01-13")}, + {[timeFieldName]: ISODate("2018-07-14")}, + {[timeFieldName]: ISODate("2030-09-30")}, +]; +const beforeEpochDocs = [ + {[timeFieldName]: ISODate("1969-12-31T23:00:59.001Z")}, + {[timeFieldName]: ISODate("1969-12-31T23:59:59.001Z")} /* one millisecond before the epoch */, +]; +const afterEpochDocs = [ + // This date is one millisecond after the maximum (the largest 32 bit integer) number of seconds + // since the epoch. + {[timeFieldName]: ISODate("2038-01-19T03:14:07.001Z")}, + {[timeFieldName]: ISODate("2050-01-20T03:14:00.003Z")} +]; +const allExtendedRangeDocs = beforeEpochDocs.concat(afterEpochDocs); /* - * Creates a collection, populates it, runs the `query` and ensures that the result set - * is equal to `results`. - * - * If overflow is set we create a document with dates above the 32 bit range (year 2040) - * If underflow is set, we create a document with dates below the 32 bit range (year 1965) + * Creates a collection, populates it using the `standardDocs` and any dates passed in by + * `extendedRangeDocs`, runs the `query` and ensures that the result set is equal to `results`. */ -function runTest(underflow, overflow, query, results) { - // Setup our DB & our collections. +function runTest({query, results, extendedRangeDocs}) { + // Setup the collection and insert the dates in `standardDocs` and `extendedRangeDocs`. const tsColl = db.getCollection(jsTestName()); tsColl.drop(); - assert.commandWorked( db.createCollection(tsColl.getName(), {timeseries: {timeField: timeFieldName}})); - - const dates = [ - // If underflow, we want to insert a date that would fall below the epoch - // i.e. 1970-01-01 00:00:00 UTC. Otherwise we use a date within the epoch. - {[timeFieldName]: underflow ? new Date("1965-01-01") : new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - // If overflow, we want to insert a date that would use more than 32 bit milliseconds after - // the epoch. This overflow will occur 2038-01-29 03:13:07 UTC. Otherwise we go slightly - // before the end of the 32 bit epoch. - {[timeFieldName]: overflow ? new Date("2040-01-01") : new Date("2030-01-01")} - ]; - assert.commandWorked(tsColl.insert(dates)); - - // Make sure the expected results are in the correct order for comparison below. - function cmpTimeFields(a, b) { - return (b[timeFieldName].getTime() - a[timeFieldName].getTime()); - } - results.sort(cmpTimeFields); + assert.commandWorked(tsColl.insert(standardDocs.concat(extendedRangeDocs))); const pipeline = [{$match: query}, {$project: {_id: 0, [timeFieldName]: 1}}]; - const plan = tsColl.explain().aggregate(pipeline); - // Verify agg pipeline. We don't want to go through a plan that encourages a sort order to - // avoid BUS and index selection, so we sort after gathering the results. + // Verify the results. const aggActuals = tsColl.aggregate(pipeline).toArray(); - aggActuals.sort(cmpTimeFields); - assert.docEq(results, aggActuals, JSON.stringify(plan, null, 4)); + assert.sameMembers(results, aggActuals, JSON.stringify(plan, null, 4)); - // Verify the equivalent find command. We again don't want to go through a plan that - // encourages a sort order to avoid BUS and index selection, so we sort after gathering the - // results. + // Verify the equivalent find command. let findActuals = tsColl.find(query, {_id: 0, [timeFieldName]: 1}).toArray(); - findActuals.sort(cmpTimeFields); - assert.docEq(findActuals, results); + assert.sameMembers(results, findActuals, JSON.stringify(plan, null, 4)); } -runTest(false, - false, - {[timeFieldName]: {$eq: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1980-01-01")}]); -runTest(false, - true, - {[timeFieldName]: {$eq: new Date("2040-01-01")}}, - [{[timeFieldName]: new Date("2040-01-01")}]); -runTest(true, - false, - {[timeFieldName]: {$eq: new Date("1965-01-01")}}, - [{[timeFieldName]: new Date("1965-01-01")}]); - -runTest(false, - false, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1971-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); -runTest(false, - true, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1971-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); -runTest(true, - false, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1965-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); -runTest(true, - true, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1965-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); - -runTest(false, - false, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2030-01-01")}]); -runTest(false, - true, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2040-01-01")}]); -runTest(true, - false, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2030-01-01")}]); -runTest(true, - true, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2040-01-01")}]); - -runTest(false, false, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); -runTest(false, true, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); -runTest(true, false, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1965-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); -runTest(true, true, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1965-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); - -runTest(false, false, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2030-01-01")} -]); -runTest(false, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2040-01-01")} -]); -runTest(true, false, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2030-01-01")} -]); -runTest(true, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2040-01-01")} -]); - -// Verify ranges that straddle the lower and upper epoch boundaries work properly. -runTest( - true, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1965-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - ]); -runTest( - false, true, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [ - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2040-01-01")}, - ]); +// Verify when there are no extended range dates in the collection, we handle querying by extended +// range dates. The code for this is shared by multiple operators, so we don't have to test each +// operator separately. +runTest({ + query: {[timeFieldName]: {$lte: beforeEpochDocs[1].time}}, + results: [], + extendedRangeDocs: [] +}); +runTest({ + query: {[timeFieldName]: {$lte: afterEpochDocs[0].time}}, + results: standardDocs, + extendedRangeDocs: [] +}); +runTest({ + query: {[timeFieldName]: {$gt: beforeEpochDocs[1].time}}, + results: standardDocs, + extendedRangeDocs: [] +}); runTest( - false, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - ]); -runTest( - false, false, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [ - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2030-01-01")}, - ]); -})(); + {query: {[timeFieldName]: {$gt: afterEpochDocs[0].time}}, results: [], extendedRangeDocs: []}); +runTest({query: {[timeFieldName]: afterEpochDocs[0].time}, results: [], extendedRangeDocs: []}); + +/* + * Verify $eq queries for dates inside, below and above the epoch ranges. + */ +// Test in the epoch range when there are measurements before and after the epoch. +runTest({ + query: {[timeFieldName]: {$eq: standardDocs[1].time}}, + results: [standardDocs[1]], + extendedRangeDocs: allExtendedRangeDocs +}); + +// Test when there is one date above the epoch range. The date is one millisecond after the maximum +// (the largest 32 bit integer) number of seconds since the epoch. +runTest({ + query: {[timeFieldName]: {$eq: afterEpochDocs[0].time}}, + results: [afterEpochDocs[0]], + extendedRangeDocs: [afterEpochDocs[0]] +}); + +// Test when there are multiple dates after the epoch range. +runTest({ + query: {[timeFieldName]: {$eq: afterEpochDocs[0].time}}, + results: [afterEpochDocs[0]], + extendedRangeDocs: afterEpochDocs +}); + +// Test when there is one date before the epoch. The date is one millisecond before the epoch. +runTest({ + query: {[timeFieldName]: {$eq: beforeEpochDocs[1].time}}, + results: [beforeEpochDocs[1]], + extendedRangeDocs: [beforeEpochDocs[1]] +}); + +// Test when there are multiple dates before the epoch. +runTest({ + query: {[timeFieldName]: {$eq: beforeEpochDocs[0].time}}, + results: [beforeEpochDocs[0]], + extendedRangeDocs: beforeEpochDocs +}); + +/* + * $lt queries. + */ +// Test with dates below the epoch that match the predicate. +runTest({ + query: {[timeFieldName]: {$lt: beforeEpochDocs[1].time}}, + results: [beforeEpochDocs[0]], + extendedRangeDocs: beforeEpochDocs +}); + +// Test with dates above the epoch that do not match the predicate. +runTest({ + query: {[timeFieldName]: {$lt: standardDocs[2].time}}, + results: [standardDocs[0], standardDocs[1]], + extendedRangeDocs: afterEpochDocs +}); + +// Test with dates both above and below the epoch. Only the before dates will match the predicate. +runTest({ + query: {[timeFieldName]: {$lt: standardDocs[1].time}}, + results: [standardDocs[0], beforeEpochDocs[0], beforeEpochDocs[1]], + extendedRangeDocs: allExtendedRangeDocs +}); + +// Test with dates both above and below the epoch. Both before and after dates match the predicate. +runTest({ + query: {[timeFieldName]: {$lt: afterEpochDocs[1].time}}, + results: standardDocs.concat(beforeEpochDocs, [afterEpochDocs[0]]), + extendedRangeDocs: allExtendedRangeDocs +}); + +/* + * $gt queries. + */ +// Test with dates below the epoch that do not match the predicate. +runTest({ + query: {[timeFieldName]: {$gt: beforeEpochDocs[1].time}}, + results: standardDocs, + extendedRangeDocs: beforeEpochDocs +}); + +// Test with dates above the epoch that do not match the predicate. +runTest({ + query: {[timeFieldName]: {$gt: afterEpochDocs[1].time}}, + results: [], + extendedRangeDocs: afterEpochDocs +}); + +// Test with dates both above and below the epoch. Only the after dates will match the predicate. +runTest({ + query: {[timeFieldName]: {$gt: standardDocs[2].time}}, + results: [standardDocs[3], afterEpochDocs[0], afterEpochDocs[1]], + extendedRangeDocs: allExtendedRangeDocs +}); + +// Test with dates both above and below the epoch. Both before and after dates match the predicate. +runTest({ + query: {[timeFieldName]: {$gt: beforeEpochDocs[0].time}}, + results: standardDocs.concat([beforeEpochDocs[1]], afterEpochDocs), + extendedRangeDocs: allExtendedRangeDocs +}); + +/* + * $lte queries. + */ + +// Test with dates below the epoch that do match the predicate. +runTest({ + query: {[timeFieldName]: {$lte: standardDocs[0].time}}, + results: beforeEpochDocs.concat([standardDocs[0]]), + extendedRangeDocs: beforeEpochDocs +}); + +// Test with dates above the epoch that do match the predicate. +runTest({ + query: {[timeFieldName]: {$lte: standardDocs[2].time}}, + results: [standardDocs[0], standardDocs[1], standardDocs[2]], + extendedRangeDocs: afterEpochDocs +}); + +// Test with dates both above and below the epoch. Only the before dates will match the predicate. +runTest({ + query: {[timeFieldName]: {$lte: standardDocs[2].time}}, + results: beforeEpochDocs.concat(standardDocs.slice(0, -1)), + extendedRangeDocs: allExtendedRangeDocs +}); + +// Test with dates both above and below the epoch. Both before and after dates will match the +// predicate. +runTest({ + query: {[timeFieldName]: {$lte: afterEpochDocs[0].time}}, + results: standardDocs.concat(beforeEpochDocs, [afterEpochDocs[0]]), + extendedRangeDocs: allExtendedRangeDocs +}); + +/* + * $gte queries. + */ +// Test with dates below the epoch that do not match the predicate. +runTest({ + query: {[timeFieldName]: {$gte: standardDocs[1].time}}, + results: standardDocs.slice(1), + extendedRangeDocs: beforeEpochDocs +}); + +// Test with dates above the epoch that do match the predicate. +runTest({ + query: {[timeFieldName]: {$gte: standardDocs[2].time}}, + results: afterEpochDocs.concat(standardDocs.slice(2)), + extendedRangeDocs: afterEpochDocs +}); + +// Test with dates both above and below the epoch. Only the after dates will match the predicate. +runTest({ + query: {[timeFieldName]: {$gte: standardDocs[1].time}}, + results: afterEpochDocs.concat(standardDocs.slice(1)), + extendedRangeDocs: allExtendedRangeDocs +}); + +// Test with dates both above and below the epoch. Both before and after dates will match the +// predicate. +runTest({ + query: {[timeFieldName]: {$gte: beforeEpochDocs[1].time}}, + results: afterEpochDocs.concat(standardDocs, [beforeEpochDocs[1]]), + extendedRangeDocs: allExtendedRangeDocs +}); + +/* + * Compound predicates ($and, $or). + */ + +// Test with dates below the epoch that do match the predicate. +runTest({ + query: {[timeFieldName]: {$gt: ISODate("1920-01-01"), $lt: ISODate("1980-01-01")}}, + results: beforeEpochDocs.concat([standardDocs[0]]), + extendedRangeDocs: beforeEpochDocs +}); + +// Test with dates below the epoch that do not match the predicate. +runTest({ + query: + {[timeFieldName]: {$gt: ISODate("1970-01-01T00:00:00.001Z"), $lt: ISODate("1980-01-01")}}, + results: [standardDocs[0]], + extendedRangeDocs: beforeEpochDocs +}); + +// Test with dates above the epoch that do match the predicate. +runTest({ + query: { + [timeFieldName]: + {$gt: ISODate("2030-09-29T23:59:59.001Z"), $lt: ISODate("2050-01-20T03:14:00.001Z")} + }, + results: [standardDocs[3], afterEpochDocs[0]], + extendedRangeDocs: afterEpochDocs +}); + +// Test with dates above the epoch that do not match the predicate. +runTest({ + query: { + [timeFieldName]: + {$gt: ISODate("2030-09-29T23:59:59.001Z"), $lt: ISODate("2038-01-19T03:14:00.000Z")} + }, + results: [standardDocs[3]], + extendedRangeDocs: afterEpochDocs +}); + +// Test with dates both above and below the epoch. Both before and after dates will match the +// predicate. +runTest({ + query: { + $or: [ + {[timeFieldName]: {$lt: ISODate("1975-12-01T23:59:00.001Z")}}, + {[timeFieldName]: {$gt: ISODate("2038-01-19T03:14:07.002Z")}} + ] + }, + results: beforeEpochDocs.concat([standardDocs[0], afterEpochDocs[1]]), + extendedRangeDocs: allExtendedRangeDocs +}); + +// Test with dates both above and below the epoch. Neither before nor after dates will match the +// predicate. +runTest({ + query: {[timeFieldName]: {$in: [standardDocs[1].time, standardDocs[3].time]}}, + results: [standardDocs[1], standardDocs[3]], + extendedRangeDocs: allExtendedRangeDocs +}); +})();
\ No newline at end of file diff --git a/jstests/core/timeseries/timeseries_geonear_lookup.js b/jstests/core/timeseries/timeseries_geonear_lookup.js new file mode 100644 index 00000000000..78f68eb014e --- /dev/null +++ b/jstests/core/timeseries/timeseries_geonear_lookup.js @@ -0,0 +1,40 @@ +/** + * Test that time-series works as expected with $geoNear within a $lookup. + * + * @tags: [ + * does_not_support_transactions, + * requires_pipeline_optimization, + * # We need a timeseries collection. + * requires_timeseries, + * references_foreign_collection, + * ] + */ + +// create a timeseries collection +const timeFieldName = "time"; +const metaFieldName = "tags"; +const testDB = db; + +const tsColl = db.getCollection("ts_coll"); + +const kMaxDistance = Math.PI * 2.0; + +tsColl.drop(); +assert.commandWorked(testDB.createCollection( + tsColl.getName(), {timeseries: {timeField: timeFieldName, metaField: metaFieldName}})); + +assert.commandWorked(tsColl.createIndex({'tags.loc': '2dsphere'})); + +tsColl.insert({time: ISODate(), tags: {loc: [40, 40], descr: 0}, value: 0}); + +const coll2 = db.getCollection("store_min_max_values"); +coll2.drop(); +assert.commandWorked(coll2.insert({_id: 0, minimumDist: 0.0, maximumDist: kMaxDistance})); + +coll2.aggregate([{$lookup: {from: tsColl.getName(), +let: {minVal: "$minimumDist",maxVal:"$maximumDist"}, +pipeline: [ + {$geoNear: {near: {type: "Point", coordinates: [0, 0]}, + key: 'tags.loc', + distanceField: "tags.distance"}}], + as: 'output'}}]); diff --git a/jstests/core/timeseries/timeseries_graph_lookup.js b/jstests/core/timeseries/timeseries_graph_lookup.js index 7b152cc6b1f..25f47aa2cef 100644 --- a/jstests/core/timeseries/timeseries_graph_lookup.js +++ b/jstests/core/timeseries/timeseries_graph_lookup.js @@ -5,6 +5,7 @@ * does_not_support_transactions, * requires_timeseries, * requires_fcv_51, + * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/timeseries/timeseries_insert_mixed_schema_bucket.js b/jstests/core/timeseries/timeseries_insert_mixed_schema_bucket.js new file mode 100644 index 00000000000..cbce31e9795 --- /dev/null +++ b/jstests/core/timeseries/timeseries_insert_mixed_schema_bucket.js @@ -0,0 +1,75 @@ +/** + * Tests directly inserting a time-series bucket with mixed schema. + * + * @tags: [ + * # $listCatalog does not include the tenant prefix in its results. + * command_not_supported_in_serverless, + * requires_timeseries, + * # $listCatalog not supported inside of a multi-document transaction + * does_not_support_transactions, + * # $listCatalog only exists since v6 + * multiversion_incompatible, + * ] + */ +(function() { +"use strict"; + +load("jstests/core/timeseries/libs/timeseries.js"); // For 'TimeseriesTest'. + +TestData.skipEnforceTimeseriesBucketsAreAlwaysCompressedOnValidate = true; + +const testDB = db.getSiblingDB(jsTestName()); +const collName = "ts"; + +assert.commandWorkedOrFailedWithCode(testDB.runCommand({drop: collName}), + ErrorCodes.NamespaceNotFound); +assert.commandWorked( + testDB.createCollection(collName, {timeseries: {timeField: "t", metaField: "m"}})); +const coll = testDB[collName]; +const bucketsColl = testDB["system.buckets." + collName]; + +const bucket = { + _id: ObjectId("65a6eb806ffc9fa4280ecac4"), + control: { + version: NumberInt(1), + min: { + _id: ObjectId("65a6eba7e6d2e848e08c3750"), + t: ISODate("2024-01-16T20:48:00Z"), + a: 1, + }, + max: { + _id: ObjectId("65a6eba7e6d2e848e08c3751"), + t: ISODate("2024-01-16T20:48:39.448Z"), + a: "a", + }, + }, + meta: 0, + data: { + _id: { + 0: ObjectId("65a6eba7e6d2e848e08c3750"), + 1: ObjectId("65a6eba7e6d2e848e08c3751"), + }, + t: { + 0: ISODate("2024-01-16T20:48:39.448Z"), + 1: ISODate("2024-01-16T20:48:39.448Z"), + }, + a: { + 0: "a", + 1: 1, + }, + } +}; + +assert.commandFailedWithCode(bucketsColl.insert(bucket), + ErrorCodes.CannotInsertTimeseriesBucketsWithMixedSchema); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); +assert.commandWorked( + testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: true})); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); +assert.commandWorked(bucketsColl.insert(bucket)); +assert.commandWorked(bucketsColl.deleteOne({_id: bucket._id})); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); +assert.commandWorked( + testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: false})); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); +})(); diff --git a/jstests/core/timeseries/timeseries_lookup.js b/jstests/core/timeseries/timeseries_lookup.js index ba804c4ec92..6d559e9356d 100644 --- a/jstests/core/timeseries/timeseries_lookup.js +++ b/jstests/core/timeseries/timeseries_lookup.js @@ -5,11 +5,13 @@ * does_not_support_transactions, * requires_timeseries, * requires_fcv_51, + * references_foreign_collection, * ] */ (function() { "use strict"; +load("jstests/aggregation/extras/utils.js"); load("jstests/core/timeseries/libs/timeseries.js"); TimeseriesTest.run((insert) => { @@ -245,3 +247,91 @@ TimeseriesTest.run((insert) => { }); }); })(); + +{ + // Test that we get the right results for $lookup on a timeseries collection with an internal + // pipeline containing both correlated and uncorrelated $match stages. Ensures we do not cache + // the results of this internal pipeline incorrectly + // (src/mongo/db/pipeline/document_source_sequential_document_cache.h) regardless of the order + // of the $match stages. + const testDB = db.getSiblingDB(jsTestName()); + testDB.local.insertMany([{_id: 0, key: 1}, {_id: 1, key: 2}]); + testDB.createCollection("foreign", {timeseries: {timeField: "time", metaField: "meta"}}); + testDB.foreign.insertMany([ + {time: new Date(), _id: 0, meta: 1, val1: 42, val2: 100}, + {time: new Date(), _id: 2, meta: 2, val1: 17, val2: 100} + ]); + + // The $sequentialCache (document_source_sequential_document_cache.h) decides whether or not it + // can cache the results of the internal lookup pipeline based on if the pipeline is correlated + // or uncorrelated. With timeseries, many rewrites occur which can merge $match stages together, + // have them pushed into the $_internalUnpackBucket stage as an eventFilter, or push them in + // front of the + // $_internalUnpackBucket stage. We need to make sure that the ability of the cache to recognize + // when there is a correlated $match in the pipeline to remain regardless of these rewrites. + (function testUncorrelatedFollowedByCorrelatedMatch() { + const lookupStage = {$lookup: { + from: "foreign", + let: {lkey: "$key"}, + pipeline: [ + {$match: {$expr: {$lt: ["$val1","$val2"]}}}, + {$match: {$expr: {$eq: ["$meta","$$lkey"]}}}, + {$project: {lkey: "$$lkey",fkey: "$meta",val: "$val1",_id: 0,}}, + ], + as: "joined" + }}; + const result = testDB.local.aggregate(lookupStage); + assertArrayEq({ + actual: result.toArray(), + expected: [ + {"_id": 1, "key": 2, "joined": [{"lkey": 2, "fkey": 2, "val": 17}]}, + {"_id": 0, "key": 1, "joined": [{"lkey": 1, "fkey": 1, "val": 42}]} + ], + extraErrorMsg: `Unexpected result of running pipeline ${tojson(lookupStage)}` + }); + })(); + + (function testCorrelatedFollowedByUncorrelatedMatch() { + const lookupStage = {$lookup: { + from: "foreign", + let: {lkey: "$key"}, + pipeline: [ + {$match: {$expr: {$eq: ["$meta","$$lkey"]}}}, + {$match: {$expr: {$lt: ["$val1","$val2"]}}}, + {$project: {lkey: "$$lkey",fkey: "$meta",val: "$val1",_id: 0,}}, + ], + as: "joined" + }}; + const result = testDB.local.aggregate(lookupStage); + assertArrayEq({ + actual: result.toArray(), + expected: [ + {"_id": 1, "key": 2, "joined": [{"lkey": 2, "fkey": 2, "val": 17}]}, + {"_id": 0, "key": 1, "joined": [{"lkey": 1, "fkey": 1, "val": 42}]} + ], + extraErrorMsg: `Unexpected result of running pipeline ${tojson(lookupStage)}` + }); + })(); + + (function testUncorrelatedFollowedByUncorrelatedMatch() { + const lookupStage = {$lookup: { + from: "foreign", + let: {lkey: "$key"}, + pipeline: [ + {$match: {$expr: {$lt: ["$meta","$val1"]}}}, + {$match: {$expr: {$lt: ["$val1","$val2"]}}}, + {$project: {meta: "$meta",val: "$val1",_id: 0,}}, + ], + as: "joined" + }}; + const result = testDB.local.aggregate(lookupStage); + assertArrayEq({ + actual: result.toArray(), + expected: [ + {"_id": 0, "key": 1, "joined": [{"meta": 1, "val": 42}, {"meta": 2, "val": 17}]}, + {"_id": 1, "key": 2, "joined": [{"meta": 1, "val": 42}, {"meta": 2, "val": 17}]} + ], + extraErrorMsg: `Unexpected result of running pipeline ${tojson(lookupStage)}` + }); + })(); +} diff --git a/jstests/core/timeseries/timeseries_merge.js b/jstests/core/timeseries/timeseries_merge.js index 59d4dbee889..fb6a23df803 100644 --- a/jstests/core/timeseries/timeseries_merge.js +++ b/jstests/core/timeseries/timeseries_merge.js @@ -6,6 +6,7 @@ * does_not_support_stepdowns, * does_not_support_transactions, * requires_timeseries, + * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/timeseries/timeseries_project.js b/jstests/core/timeseries/timeseries_project.js index 11ca90f2efe..b735a8a3b66 100644 --- a/jstests/core/timeseries/timeseries_project.js +++ b/jstests/core/timeseries/timeseries_project.js @@ -129,4 +129,171 @@ tsDoc = tsColl.aggregate(pipeline).toArray(); regDoc = regColl.aggregate(pipeline).toArray(); assert.docEq(tsDoc, regDoc); })(); + +// Test that an includes projection that doesn't include previously included computed meta fields +// outputs correctly. +(function testIncludeComputedMetaFieldThenProjectWithoutComputedMetaField() { + coll.drop(); + assert.commandWorked( + db.createCollection(coll.getName(), {timeseries: {timeField: 'time', metaField: 'tag'}})); + + const doc = {_id: 0, time: ISODate("2024-01-01T00:00:00.000Z"), tag: {field: "A"}}; + assert.commandWorked(coll.insert(doc)); + let result = {}; + + // Added field which uses meta should not be in the result. + result = + coll.aggregate( + [{$addFields: {time: {$dateFromParts: {year: "$tag.none"}}}}, {$project: {tag: 1}}]) + .toArray(); + assert.docEq([{"tag": {"field": "A"}, "_id": 0}], result); + + // Test with non-null value. + result = coll.aggregate([{$addFields: {time: {$toLower: "$tag.field"}}}, {$project: {tag: 1}}]) + .toArray(); + assert.docEq([{"tag": {"field": "A"}, "_id": 0}], result); + + // Testing with $match between $addFields and $project. + result = coll.aggregate([ + {$addFields: {time: {$dateFromParts: {year: "$tag.none"}}}}, + {$match: {tag: {field: "A"}}}, + {$project: {tag: 1}} + ]) + .toArray(); + assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); + + // Test with non-null value. + result = + coll.aggregate( + [{$addFields: {time: "$tag.field"}}, {$match: {time: "A"}}, {$project: {tag: 1}}]) + .toArray(); + assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); + + // Testing with $project after $project. + result = coll.aggregate([{$project: {tag: 1}}, {$project: {time: 1}}]).toArray(); + assert.docEq([{"_id": 0}], result); + + // Testing with $set. + result = coll.aggregate([{$set: {tag: "$tag.field"}}, {$project: {time: 1}}]).toArray(); + assert.docEq([{"_id": 0, "time": ISODate("2024-01-01T00:00:00Z")}], result); + + // Test that computedMetaFields that are included by the $project are still included. + result = coll.aggregate([ + {$addFields: {hi: {$dateFromParts: {year: "$tag.none"}}}}, + {$addFields: {bye: {$dateFromParts: {year: "$tag.none"}}}}, + {$project: {hi: 1}} + ]) + .toArray(); + assert.docEq([{"_id": 0, "hi": null}], result); + + // Test with non-null value. + result = coll.aggregate([ + {$addFields: {hi: "$tag.field"}}, + {$addFields: {bye: "$tag.field"}}, + {$project: {hi: 1}} + ]) + .toArray(); + assert.docEq([{"_id": 0, "hi": "A"}], result); + + // Test that $project that replaces field that is used works. + result = coll.aggregate([ + {$addFields: {hello: "$tag.field"}}, + {$project: {newTime: "$time", time: "$tag.field"}} + ]) + .toArray(); + + assert.docEq([{"_id": 0, newTime: ISODate("2024-01-01T00:00:00.000Z"), time: "A"}], result); + + // Test adding fields and including one of them. + result = coll.aggregate([ + {$addFields: {a: {$toLower: "$tag.field"}}}, + {$addFields: {b: {$toUpper: "$tag.field"}}}, + {$project: {b: 1}} + ]) + .toArray(); + assert.docEq([{b: "A", "_id": 0}], result); +})(); + +// Test that an excludes projection that excludes previously included computed meta fields +// outputs correctly. +(function testIncludeComputedMetaFieldThenExcludeComputedMetaField() { + coll.drop(); + assert.commandWorked( + db.createCollection(coll.getName(), {timeseries: {timeField: 'time', metaField: 'tag'}})); + + const doc = {_id: 0, time: ISODate("2024-01-01T00:00:00.000Z"), tag: {field: "A"}}; + assert.commandWorked(coll.insert(doc)); + let result = {}; + + // Excluded '_computedMetaProjField' should not be included. + result = coll.aggregate([ + {$addFields: {hello: {$dateFromParts: {year: "$tag.none"}}}}, + {$project: {hello: 0}} + ]) + .toArray(); + assert.docEq([{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0}], + result); + + // Test with non-null value. + result = coll.aggregate([{$addFields: {time: {$toLower: "$tag.field"}}}, {$project: {time: 0}}]) + .toArray(); + assert.docEq([{"tag": {"field": "A"}, "_id": 0}], result); + + // Testing with $match between $addFields and $project. + result = coll.aggregate([ + {$addFields: {time: {$dateFromParts: {year: "$tag.none"}}}}, + {$match: {tag: {field: "A"}}}, + {$project: {time: 0}} + ]) + .toArray(); + assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); + + // Test with non-null value. + result = + coll.aggregate( + [{$addFields: {time: "$tag.field"}}, {$match: {time: "A"}}, {$project: {time: 0}}]) + .toArray(); + assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); + + // Testing with exclude $project after include $project. + result = coll.aggregate([{$project: {tag: 1}}, {$project: {tag: 0}}]).toArray(); + assert.docEq([{"_id": 0}], result); + + // Testing with $set. + result = coll.aggregate([{$set: {tag: "$tag.field"}}, {$project: {tag: 0}}]).toArray(); + assert.docEq([{"time": ISODate("2024-01-01T00:00:00Z"), "_id": 0}], result); + + // Exclude one added field. + result = coll.aggregate([ + {$addFields: {hi: {$dateFromParts: {year: "$tag.none"}}}}, + {$addFields: {bye: {$dateFromParts: {year: "$tag.none"}}}}, + {$project: {hi: 0}} + ]) + .toArray(); + assert.docEq( + [{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0, "bye": null}], + result); + + // Test with non-null value. + result = coll.aggregate([ + {$addFields: {hi: "$tag.field"}}, + {$addFields: {bye: "$tag.field"}}, + {$project: {hi: 0}} + ]) + .toArray(); + assert.docEq( + [{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0, "bye": "A"}], + result); + + // Test adding fields and excluding one of them. + result = coll.aggregate([ + {$addFields: {a: {$toLower: "$tag.field"}}}, + {$addFields: {b: {$toUpper: "$tag.field"}}}, + {$project: {b: 0}} + ]) + .toArray(); + assert.docEq( + [{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0, "a": "a"}], + result); +})(); })(); diff --git a/jstests/core/timeseries/timeseries_union_with.js b/jstests/core/timeseries/timeseries_union_with.js index b5d5369dcc8..e92e4c4ca37 100644 --- a/jstests/core/timeseries/timeseries_union_with.js +++ b/jstests/core/timeseries/timeseries_union_with.js @@ -8,6 +8,7 @@ * # migrations may result in writes splitting between two primaries, and thus different buckets. * does_not_support_stepdowns, * tenant_migration_incompatible, + * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js b/jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js new file mode 100644 index 00000000000..0211daa29d8 --- /dev/null +++ b/jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js @@ -0,0 +1,79 @@ +/** + * Tests directly updating a time-series bucket to contain mixed schema. + * + * @tags: [ + * # $listCatalog does not include the tenant prefix in its results. + * command_not_supported_in_serverless, + * requires_timeseries, + * # $listCatalog not supported inside of a multi-document transaction + * does_not_support_transactions, + * # $listCatalog only exists since v6 + * multiversion_incompatible, + * ] + */ +(function() { +"use strict"; +load("jstests/core/timeseries/libs/timeseries.js"); // For 'TimeseriesTest'. + +TestData.skipEnforceTimeseriesBucketsAreAlwaysCompressedOnValidate = true; + +const testDB = db.getSiblingDB(jsTestName()); +const collName = "ts"; + +assert.commandWorkedOrFailedWithCode(testDB.runCommand({drop: collName}), + ErrorCodes.NamespaceNotFound); +assert.commandWorked( + testDB.createCollection(collName, {timeseries: {timeField: "t", metaField: "m"}})); +const coll = testDB[collName]; +const bucketsColl = testDB["system.buckets." + collName]; + +const bucket = { + _id: ObjectId("65a6eb806ffc9fa4280ecac4"), + control: { + version: NumberInt(1), + min: { + _id: ObjectId("65a6eba7e6d2e848e08c3750"), + t: ISODate("2024-01-16T20:48:00Z"), + a: 0, + }, + max: { + _id: ObjectId("65a6eba7e6d2e848e08c3751"), + t: ISODate("2024-01-16T20:48:39.448Z"), + a: 1, + }, + }, + meta: 0, + data: { + _id: { + 0: ObjectId("65a6eba7e6d2e848e08c3750"), + 1: ObjectId("65a6eba7e6d2e848e08c3751"), + }, + t: { + 0: ISODate("2024-01-16T20:48:39.448Z"), + 1: ISODate("2024-01-16T20:48:39.448Z"), + }, + a: { + 0: 0, + 1: 1, + }, + } +}; + +const update = function() { + return bucketsColl.update({_id: bucket._id}, + {$set: {"control.min.a": 1, "control.max.a": "a", "data.a.0": "a"}}); +}; + +assert.commandWorked(bucketsColl.insert(bucket)); +assert.commandFailedWithCode(update(), ErrorCodes.CannotInsertTimeseriesBucketsWithMixedSchema); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); +assert.commandWorked( + testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: true})); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); +assert.commandWorked(update()); +assert.commandWorked(bucketsColl.deleteOne({_id: bucket._id})); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); +assert.commandWorked( + testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: false})); +assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); +})(); diff --git a/jstests/core/txns/abort_expired_transaction.js b/jstests/core/txns/abort_expired_transaction.js index b211ef7b132..54418596967 100644 --- a/jstests/core/txns/abort_expired_transaction.js +++ b/jstests/core/txns/abort_expired_transaction.js @@ -33,6 +33,11 @@ try { const session = db.getMongo().startSession(sessionOptions); const sessionDb = session.getDatabase(testDBName); + // Number of passes made by the "abortExpiredTransactions" thread before the transaction + // expires. + const abortExpiredTransactionsPassesPreAbort = + db.serverStatus().metrics.abortExpiredTransactions.passes; + let txnNumber = 0; jsTest.log("Insert a document starting a transaction."); @@ -64,6 +69,14 @@ try { "currentOp reports that the idle transaction still exists, it has not been " + "aborted as expected."); + assert.soon(() => { + // For this expired transaction to abort, the "abortExpiredTransactions" thread has to + // perform at least one pass. + const serverStatus = db.serverStatus(); + return abortExpiredTransactionsPassesPreAbort < + serverStatus.metrics.abortExpiredTransactions.passes; + }); + jsTest.log( "Attempt to do a write in the transaction, which should fail because the transaction " + "was aborted"); diff --git a/jstests/core/txns/aggregation_in_transaction.js b/jstests/core/txns/aggregation_in_transaction.js index ec1c62133bb..40d7030fe1f 100644 --- a/jstests/core/txns/aggregation_in_transaction.js +++ b/jstests/core/txns/aggregation_in_transaction.js @@ -1,5 +1,5 @@ // Tests that aggregation is supported in transactions. -// @tags: [uses_transactions, uses_snapshot_read_concern] +// @tags: [uses_transactions, uses_snapshot_read_concern, references_foreign_collection] (function() { "use strict"; diff --git a/jstests/core/txns/prepare_conflict_aggregation_behavior.js b/jstests/core/txns/prepare_conflict_aggregation_behavior.js index c62b7370dc6..c07f96e2b54 100644 --- a/jstests/core/txns/prepare_conflict_aggregation_behavior.js +++ b/jstests/core/txns/prepare_conflict_aggregation_behavior.js @@ -3,7 +3,12 @@ * should not block on prepare conflicts, but writing out to a collection as a part of an aggregate * pipeline should block on prepare conflicts. * - * @tags: [uses_transactions, uses_prepare_transaction] + * The test runs commands that are not allowed with security token: endSession, prepareTransaction. + * @tags: [ + * references_foreign_collection, + * uses_transactions, + * uses_prepare_transaction, + * ] */ (function() { "use strict"; diff --git a/jstests/core/views/invalid_system_views.js b/jstests/core/views/invalid_system_views.js index da2b60d170f..15b4ab6bc3b 100644 --- a/jstests/core/views/invalid_system_views.js +++ b/jstests/core/views/invalid_system_views.js @@ -14,6 +14,7 @@ * # The drop of offending views may not happen on the donor after a committed migration. * tenant_migration_incompatible, * uses_compact, + * references_foreign_collection, * ] */ diff --git a/jstests/core/views/views_aggregation.js b/jstests/core/views/views_aggregation.js index 537ec3ee008..287bc5af667 100644 --- a/jstests/core/views/views_aggregation.js +++ b/jstests/core/views/views_aggregation.js @@ -9,6 +9,7 @@ * requires_non_retryable_commands, * # Explain of a resolved view must be executed by mongos. * directly_against_shardsvrs_incompatible, + * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/views/views_collation.js b/jstests/core/views/views_collation.js index 169e9309740..65deb11edbc 100644 --- a/jstests/core/views/views_collation.js +++ b/jstests/core/views/views_collation.js @@ -6,6 +6,7 @@ // requires_non_retryable_commands, // # Explain of a resolved view must be executed by mongos. // directly_against_shardsvrs_incompatible, +// references_foreign_collection, // ] /** diff --git a/jstests/core/views/views_creation.js b/jstests/core/views/views_creation.js index 9528199996a..c24a008342d 100644 --- a/jstests/core/views/views_creation.js +++ b/jstests/core/views/views_creation.js @@ -11,6 +11,7 @@ * requires_non_retryable_commands, * # Tenant migrations don't support applyOps. * tenant_migration_incompatible, + * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/views/views_validation.js b/jstests/core/views/views_validation.js index 02c060fd50a..4dea7c9491b 100644 --- a/jstests/core/views/views_validation.js +++ b/jstests/core/views/views_validation.js @@ -2,6 +2,7 @@ // # Running getCollection on views in sharded suites tries to shard views, which fails. // assumes_unsharded_collection, // requires_non_retryable_commands, +// references_foreign_collection, // ] (function() { diff --git a/jstests/core/wildcard_index_validindex.js b/jstests/core/wildcard_index_validindex.js index 24ecdcf77cf..c4a2ea9c2ac 100644 --- a/jstests/core/wildcard_index_validindex.js +++ b/jstests/core/wildcard_index_validindex.js @@ -98,6 +98,18 @@ assert.commandFailedWithCode(coll.createIndex({"$**": "wildcard"}), ErrorCodes.C // Cannot create a compound wildcard index. assert.commandFailedWithCode(coll.createIndex({"$**": 1, "a": 1}), ErrorCodes.CannotCreateIndex); assert.commandFailedWithCode(coll.createIndex({"a": 1, "$**": 1}), ErrorCodes.CannotCreateIndex); +assert.commandFailedWithCode(coll.createIndex({"obj.$**": 1, "a": 1}), + ErrorCodes.CannotCreateIndex); +assert.commandFailedWithCode(db.runCommand({ + createIndexes: kCollectionName, + indexes: [{wildcardProjection: {a: 1, c: 1, d: 1}, key: {"$**": 1, b: 1}}] +}), + ErrorCodes.CannotCreateIndex); +assert.commandFailedWithCode(db.runCommand({ + createIndexes: kCollectionName, + indexes: [{key: {"$**": 1, b: 1}, wildcardProjection: {a: 1, c: 1, d: 1}}] +}), + ErrorCodes.CannotCreateIndex); // Cannot create an wildcard index with an invalid spec. assert.commandFailedWithCode(coll.createIndex({"a.$**.$**": 1}), ErrorCodes.CannotCreateIndex); @@ -141,4 +153,9 @@ assert.commandFailedWithCode( assert.commandFailedWithCode( createIndexHelper({"a.$**": 1}, {name: kIndexName, wildcardProjection: {b: 0}}), ErrorCodes.FailedToParse); +assert.commandFailedWithCode(db.runCommand({ + createIndexes: kCollectionName, + indexes: [{wildcardProjection: {b: 1}, key: {"a.$**": 1}}] +}), + ErrorCodes.FailedToParse); })(); |