diff options
Diffstat (limited to 'etc')
32 files changed, 4531 insertions, 5224 deletions
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index da0f807ff61..d0dae420296 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -4,790 +4,589 @@ # # Usage: # Add the server ticket number and the path to the test file for the test you intend to denylist -# under the appropriate multiversion branch. Any test in a (ticket, test_file) pair that appears in this file but +# under the appropriate suite. Any test in a (ticket, test_file) pair that appears in this file but # not in the last-lts or last-continuous branch version of this file indicates that a commit has # not yet been backported to the last-lts or last-continuous branch and will be excluded from the # multiversion suite corresponding to the root level suite key. # -# Example: To prevent 'my_test_file.js' from running with the last-continuous binary -# last-continuous: -# all: -# - test_file: jstests/core/my_test_file.js -# ticket: SERVER-1000 +# Example: +# replica_sets_multiversion: +# - ticket: SERVER-1000 +# test_file: jstests/core/my_test_file.js # # The above example will denylist jstests/core/my_test_file.js from the -# last-continuous branch until this file has been updated with the same -# (ticket, test_file) pair on the last-continuous branch. +# 'replica_sets_multiversion_gen' task until this file has been updated with the same +# (ticket, test_file) pair on the last-lts branch. # + last-continuous: + # Tests that should be excluded from ALL multiversion suites should be listed here. all: - - test_file: jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js - ticket: SERVER-88885 - - test_file: jstests/replsets/ignore_dbcheck_in_initial_sync.js - ticket: SERVER-80340 - - test_file: jstests/replsets/ignore_dbcheck_in_startup_recovery.js - ticket: SERVER-30832 - - test_file: jstests/replsets/ignore_dbcheck_in_rollback.js - ticket: SERVER-30832 - - test_file: jstests/replsets/dbcheck_fails_on_write_concern_error.js - ticket: SERVER-88318 - - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_pauses_watchdog_checks_during_storage_change.js - ticket: SERVER-83508 - - test_file: jstests/sharding/query/find_and_modify_with_partial_shard_key.js - ticket: SERVER-44422 - - test_file: jstests/sharding/query/delete_with_partial_shard_key.js - ticket: SERVER-44422 - - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js - ticket: SERVER-51943 - - test_file: jstests/sharding/query/collation_shard_targeting_hashed_shard_key.js - ticket: SERVER-53335 - - test_file: jstests/core/elemmatch_index.js - ticket: SERVER-90747 - - test_file: jstests/sharding/cwwc_conflict_add_shard.js - ticket: SERVER-56800 - - test_file: jstests/sharding/reconfig_fails_no_cwwc_set_sharding.js - ticket: SERVER-56846 - - test_file: jstests/sharding/database_versioning_all_commands.js - ticket: SERVER-56887 - - test_file: jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_retryable_write_retry.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_find_and_modify_retry.js - ticket: SERVER-57617 - - test_file: jstests/sharding/resharding_min_fetch_ts_with_txn.js - ticket: SERVER-57667 - - test_file: jstests/sharding/reshard_collection_joins_existing_operation.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_clones_duplicate_key.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_abort_command.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_clones_initial_data.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_commit.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_collection_cloner_resuming.js - ticket: SERVER-57667 - - test_file: jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_metrics.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_fails_on_nonempty_stash.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_txn_cloner.js - ticket: SERVER-57667 - - test_file: jstests/sharding/reshard_collection_joins_existing_operation.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_size_estimate.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_histogram_metrics.js - ticket: SERVER-57700 - - test_file: jstests/sharding/resharding_metrics.js - ticket: SERVER-57766 - - test_file: jstests/sharding/rewrite_state_change_errors.js - ticket: SERVER-57772 - - test_file: jstests/sharding/query/pipeline_length_limit.js - ticket: SERVER-58203 - - test_file: jstests/replsets/tenant_migration_aborted_buildindex.js - ticket: SERVER-58353 - - test_file: jstests/sharding/implicit_default_write_concern_add_shard.js - ticket: SERVER-58696 - - test_file: jstests/replsets/apply_ops_capped_collection.js - ticket: SERVER-58744 - - test_file: jstests/replsets/write_concern_write_to_local.js - ticket: SERVER-58898 - - test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js - ticket: SERVER-59023 - - test_file: jstests/replsets/rollback_transaction_table.js - ticket: SERVER-59057 - - test_file: jstests/sharding/change_stream_show_migration_events.js - ticket: SERVER-59424 - - test_file: jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js - ticket: SERVER-59525 - - test_file: jstests/core/timeseries/timeseries_bucket_rename.js - ticket: SERVER-59666 - - test_file: jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js - ticket: SERVER-59721 - - test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js - ticket: SERVER-59721 - - test_file: jstests/sharding/test_resharding_test_fixture_shutdown_retry_needed.js - ticket: SERVER-59923 - - test_file: jstests/sharding/cluster_time_across_add_shard.js - ticket: SERVER-60466 - - test_file: jstests/replsets/dont_set_invalid_rwconcern.js - ticket: SERVER-60817 - - test_file: jstests/replsets/dbhash_lock_acquisition.js - ticket: SERVER-60829 - - test_file: jstests/core/sbe/sbe_ixscan_explain.js - ticket: SERVER-61087 - - test_file: jstests/sharding/multi_writes_with_shard_version_ignored_dont_bubble_up_critical_section.js - ticket: SERVER-61127 - - test_file: jstests/replsets/tenant_migration_transaction_boundary.js - ticket: SERVER-61666 - - test_file: jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js - ticket: SERVER-61755 - - test_file: jstests/replsets/dbcheck.js - ticket: SERVER-61757 - - test_file: jstests/replsets/reconfig_may_not_remove_custom_wc_in_use.js - ticket: SERVER-61864 - - test_file: jstests/replsets/default_write_concern_race_with_config_tags.js - ticket: SERVER-61864 - - test_file: jstests/replsets/config_tags_race_with_default_write_concern.js - ticket: SERVER-61864 - - test_file: jstests/replsets/dbcheck_write_concern.js - ticket: SERVER-62212 - - test_file: jstests/sharding/recover_multiple_migrations_on_stepup.js - ticket: SERVER-62245 - - test_file: jstests/sharding/migration_recovers_unfinished_migrations.js - ticket: SERVER-62296 - - test_file: jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js - ticket: SERVER-62710 - - test_file: jstests/replsets/apply_ops_dropDatabase.js - ticket: SERVER-62759 - - test_file: jstests/noPassthrough/operator_counters_accumulators.js - ticket: SERVER-63049 - - test_file: jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_collection_ttl.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_recipient_current_op.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_retry_session_migration.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_recipient_startup_recovery.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_network_error_via_rollback.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js - ticket: SERVER-63129 - - test_file: jstests/core/dbstats.js - ticket: SERVER-63255 - - test_file: jstests/replsets/buildindexes_false_commit_quorum.js - ticket: SERVER-63531 - - test_file: jstests/core/in_with_mixed_values.js - ticket: SERVER-64141 - - test_file: jstests/core/plan_cache_sbe.js - ticket: SERVER-64315 - - test_file: jstests/core/sbe/plan_cache_sbe_with_or_queries.js - ticket: SERVER-64315 - - test_file: jstests/core/sbe_plan_cache_autoparameterize_collscan.js - ticket: SERVER-64315 - - test_file: jstests/core/wildcard_index_cached_plans.js - ticket: SERVER-64315 - - test_file: jstests/sharding/update_with_dollar_fields.js - ticket: SERVER-64485 - - test_file: jstests/sharding/resharding_change_stream_namespace_filtering.js - ticket: SERVER-64780 - - test_file: jstests/sharding/shard_key_index_must_exist.js - ticket: SERVER-6491 - - test_file: jstests/replsets/capped_deletes.js - ticket: SERVER-65261 - - test_file: jstests/aggregation/agg_infinite_recursion.js - ticket: SERVER-65773 - - test_file: jstests/sharding/migration_retries_on_write_conflict_exceptions.js - ticket: SERVER-65947 - - test_file: jstests/aggregation/match_no_swap_rand.js - ticket: SERVER-66072 - - test_file: jstests/sharding/timeseries_multiple_mongos.js - ticket: SERVER-66264 - - test_file: jstests/core/or_to_in.js - ticket: SERVER-66379 - - test_file: jstests/core/where_multiple_plans.js - ticket: SERVER-66389 - - test_file: jstests/core/clustered_collection_collation.js - ticket: SERVER-67105 - - test_file: jstests/core/clustered_collection_bounded_scan.js - ticket: SERVER-67105 - - test_file: jstests/sharding/min_max_key.js - ticket: SERVER-67529 - - test_file: jstests/core/clustered_collection_create_index_clustered.js - ticket: SERVER-67800 - - test_file: jstests/core/clustered_collection_creation.js - ticket: SERVER-67800 - - test_file: jstests/sharding/resharding_temp_ns_routing_info_unsharded.js - ticket: SERVER-68628 - - test_file: jstests/sharding/move_chunk_concurrent_cloning.js - ticket: SERVER-81201 - - test_file: jstests/sharding/move_chunk_interrupt_postimage.js - ticket: SERVER-68728 - - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_election_during_storage_change.js - ticket: SERVER-69861 - - test_file: jstests/core/timeseries/timeseries_filter_extended_range.js - ticket: SERVER-69952 - - test_file: jstests/aggregation/expressions/switch_errors.js - ticket: SERVER-70190 - - test_file: jstests/replsets/wtimeout_too_large.js - ticket: SERVER-70360 - - test_file: jstests/core/cover_null_queries.js - ticket: SERVER-70436 - - test_file: jstests/sharding/all_collection_stats.js - ticket: SERVER-71477 - - test_file: jstests/sharding/drop_collection.js - ticket: SERVER-71689 - - test_file: jstests/sharding/index_and_collection_option_propagation.js - ticket: SERVER-71819 - - test_file: jstests/core/clustered_collection_collation.js - ticket: SERVER-72224 - - test_file: jstests/core/clustered/clustered_collection_collation.js - ticket: SERVER-72224 - - test_file: jstests/core/project_with_collation.js - ticket: SERVER-72416 - - test_file: jstests/core/collation.js - ticket: SERVER-72416 - - test_file: jstests/core/or_to_in.js - ticket: SERVER-72416 - - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/oplog_rollover.js - ticket: SERVER-72422 - - test_file: jstests/sharding/timeseries_cluster_indexstats.js - ticket: SERVER-72620 - - test_file: jstests/replsets/quiesce_mode_fails_elections.js - ticket: SERVER-72774 - - test_file: jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js - ticket: SERVER-73110 - - test_file: jstests/core/doc_validation_with_now_variable.js - ticket: SERVER-73125 - - test_file: jstests/sharding/resharding_interrupt_before_create_state_machine.js - ticket: SERVER-74647 - - test_file: jstests/sharding/merge_let_params_size_estimation.js - ticket: SERVER-74806 - - test_file: jstests/core/query/elemmatch/elemmatch_or_pushdown_paths.js - ticket: SERVER-74954 - - test_file: jstests/core/command_let_variables.js - ticket: SERVER-75356 - - test_file: jstests/core/field_name_validation.js - ticket: SERVER-75517 - - test_file: jstests/core/timeseries/nondefault_collation.js - ticket: SERVER-81013 - - test_file: jstests/sharding/conn_pool_stats.js - ticket: SERVER-75911 - - test_file: jstests/sharding/database_versioning_all_commands.js - ticket: SERVER-75911 - - test_file: jstests/noPassthrough/clustered_collection_sorted_scan.js - ticket: SERVER-76102 - - test_file: jstests/sharding/implicit_create_collection_triggered_by_DDLs.js - ticket: SERVER-76489 - - test_file: jstests/sharding/shard_keys_with_dollar_sign.js - ticket: SERVER-76948 - - test_file: jstests/sharding/resharding_update_tag_zones_large.js - ticket: SERVER-76988 - - test_file: jstests/core/timeseries/timeseries_create_collection.js - ticket: SERVER-77382 - - test_file: jstests/core/timeseries/timeseries_field_parsed_as_bson.js - ticket: SERVER-77654 - - test_file: jstests/change_streams/oplog_rewrite/change_stream_nor_rewrite.js - ticket: SERVER-78650 - - test_file: jstests/core/timeseries/timeseries_groupby_reorder.js - ticket: SERVER-79136 - - test_file: jstests/sharding/fsync_deadlock.js - ticket: SERVER-78793 - - test_file: jstests/sharding/fsync_lock_ddl_lock.js - ticket: SERVER-78157 - - test_file: jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js - ticket: SERVER-78157 - - test_file: jstests/sharding/fsync_lock_unlock.js - ticket: SERVER-78149 - - test_file: jstests/noPassthrough/read_preference_metrics.js - ticket: SERVER-66018 - - test_file: jstests/sharding/timeseries_insert_targeting_normalize_metadata.js - ticket: SERVER-80203 - - test_file: jstests/sharding/block_chunk_migrations_without_hashed_shard_key_index.js - ticket: SERVER-80947 - - test_file: jstests/sharding/delete_range_deletion_tasks_on_dropped_hashed_shard_key_index.js - ticket: SERVER-80947 - - test_file: jstests/sharding/move_chunk_basic.js - ticket: SERVER-81201 - - test_file: jstests/sharding/flushRoutingTableCacheUpdates_enforced_on_collections.js - ticket: SERVER-81985 - - test_file: jstests/sharding/query/merge_nondefault_read_concern.js - ticket: SERVER-69244 - - test_file: jstests/core/timeseries/timeseries_project_pushdown.js - ticket: SERVER-82447 - - test_file: jstests/core/match_with_and_or_lockstep_enumeration.js - ticket: SERVER-83091 - - test_file: jstests/sharding/coordinate_txn_recover_on_stepup_with_tickets_exhausted.js - ticket: SERVER-82883 - - test_file: jstests/core/nested_or_duplicate_predicates_index_scan.js - ticket: SERVER-84013 - - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js - ticket: SERVER-82353 - - test_file: jstests/core/timeseries/timeseries_collmod.js - ticket: SERVER-83370 - - test_file: jstests/core/timeseries/timeseries_lookup.js - ticket: SERVER-79637 - - test_file: jstests/noPassthrough/notablescan.js - ticket: SERVER-83119 - - test_file: jstests/sharding/clustered_coll_scan.js - ticket: SERVER-83119 - - test_file: jstests/change_streams/split_large_event_with_other_stages.js - ticket: SERVER-85848 - - test_file: jstests/core/txns/out_not_blocked_by_txn.js - ticket: SERVER-72703 - - test_file: jstests/core/query/bittest.js - ticket: SERVER-86419 - - test_file: jstests/core/command_let_variables_expressions.js - ticket: SERVER-89268 - - test_file: jstests/core/query/internal_strip_invalid_assignment.js - ticket: SERVER-82717 - - test_file: jstests/core/timeseries/timeseries_project.js - ticket: SERVER-87987 - - test_file: jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js - ticket: SERVER-79049 - - test_file: jstests/noPassthrough/server_status_metrics_retryable_writes.js - ticket: SERVER-86253 - - test_file: jstests/replsets/reconstruct_prepared_transactions_initial_sync.js - ticket: SERVER-89618 - - test_file: jstests/core/timeseries/timeseries_geonear_lookup.js - ticket: SERVER-81454 - - test_file: jstests/core/timeseries/timeseries_computed_field.js - ticket: SERVER-88072 - - test_file: jstests/core/bypass_empty_ts_replacement.js - ticket: SERVER-88750 - - test_file: jstests/core/bypass_empty_ts_replacement_timeseries.js - ticket: SERVER-88750 - - test_file: jstests/aggregation/unwind_sort.js - ticket: SERVER-90485 - - test_file: jstests/aggregation/group_by_objectid.js - ticket: SERVER-90173 - - test_file: jstests/replsets/catchup_ignores_old_heartbeats.js - ticket: SERVER-86674 - - test_file: jstests/core/wildcard_index_validindex.js - ticket: SERVER-93105 - suites: null + - ticket: SERVER-56800 + test_file: jstests/sharding/cwwc_conflict_add_shard.js + - ticket: SERVER-57262 + test_file: jstests/replsets/catchup_takeover_with_higher_config.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_min_fetch_ts_with_txn.js + - ticket: SERVER-57667 + test_file: jstests/sharding/reshard_collection_joins_existing_operation.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_clones_duplicate_key.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_abort_command.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_clones_initial_data.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_commit.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_collection_cloner_resuming.js + - ticket: SERVER-57667 + test_file: jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_metrics.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_fails_on_nonempty_stash.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_txn_cloner.js + - ticket: SERVER-57667 + test_file: jstests/sharding/reshard_collection_joins_existing_operation.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_size_estimate.js + - ticket: SERVER-58898 + test_file: jstests/replsets/write_concern_write_to_local.js + - ticket: SERVER-56846 + test_file: jstests/sharding/reconfig_fails_no_cwwc_set_sharding.js + - ticket: SERVER-59057 + test_file: jstests/replsets/rollback_transaction_table.js + - ticket: SERVER-58353 + test_file: jstests/replsets/tenant_migration_aborted_buildindex.js + - ticket: SERVER-53335 + test_file: jstests/sharding/query/collation_shard_targeting_hashed_shard_key.js + - ticket: SERVER-58696 + test_file: jstests/sharding/implicit_default_write_concern_add_shard.js + - ticket: SERVER-59197 + test_file: jstests/replsets/sessions_collection_reaping.js + - ticket: SERVER-58203 + test_file: jstests/sharding/query/pipeline_length_limit.js + - ticket: SERVER-56887 + test_file: jstests/sharding/database_versioning_all_commands.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_retryable_write_retry.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_find_and_modify_retry.js + - ticket: SERVER-51943 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-59525 + test_file: jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js + - ticket: SERVER-59023 + test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js + - ticket: SERVER-57772 + test_file: jstests/sharding/rewrite_state_change_errors.js + - ticket: SERVER-59666 + test_file: jstests/core/timeseries/timeseries_bucket_rename.js + - ticket: SERVER-59424 + test_file: jstests/sharding/change_stream_show_migration_events.js + - ticket: SERVER-59108 + test_file: jstests/replsets/stepdown_race_with_transaction.js + - ticket: SERVER-58636 + test_file: jstests/replsets/initial_sync_replicate_drop_mid_secondary_batch.js + - ticket: SERVER-59613 + test_file: jstests/aggregation/range.js + - ticket: SERVER-59923 + test_file: jstests/sharding/test_resharding_test_fixture_shutdown_retry_needed.js + - ticket: SERVER-61087 + test_file: jstests/core/sbe/sbe_ixscan_explain.js + - ticket: SERVER-56127 + test_file: jstests/sharding/retryable_writes_nested_shard_key.js + - ticket: SERVER-54909 + test_file: jstests/replsets/replSetGetStatus_member_wall_times.js + - ticket: SERVER-58744 + test_file: jstests/replsets/apply_ops_capped_collection.js + - ticket: SERVER-57700 + test_file: jstests/sharding/resharding_histogram_metrics.js + - ticket: SERVER-59721 + test_file: jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js + - ticket: SERVER-59721 + test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js + - ticket: SERVER-57588 + test_file: jstests/core/match_numeric_components.js + - ticket: SERVER-60682 + test_file: jstests/sharding/coordinate_txn_commit_with_tickets_exhausted.js + - ticket: SERVER-60685 + test_file: jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js + - ticket: SERVER-58506 + test_file: jstests/noPassthrough/getParameterWithDetails.js + - ticket: SERVER-57766 + test_file: jstests/sharding/resharding_metrics.js + - ticket: SERVER-61757 + test_file: jstests/replsets/dbcheck.js + - ticket: SERVER-61955 + test_file: jstests/auth/dbcheck.js + - ticket: SERVER-61666 + test_file: jstests/replsets/tenant_migration_transaction_boundary.js + - ticket: SERVER-62245 + test_file: jstests/sharding/recover_multiple_migrations_on_stepup.js + - ticket: SERVER-64184 + test_file: jstests/noPassthrough/aggregate_metrics.js + - ticket: SERVER-62212 + test_file: jstests/replsets/dbcheck_write_concern.js + - ticket: SERVER-62296 + test_file: jstests/sharding/migration_recovers_unfinished_migrations.js + - ticket: SERVER-60817 + test_file: jstests/replsets/dont_set_invalid_rwconcern.js + - ticket: SERVER-61977 + test_file: jstests/replsets/rollback_during_step_up.js + - ticket: SERVER-61755 + test_file: jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js + - ticket: SERVER-62147 + test_file: jstests/core/exhaust.js + - ticket: SERVER-63141 + test_file: jstests/aggregation/lookup_let_optimization.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_collection_ttl.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_current_op.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_retry_session_migration.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_startup_recovery.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_network_error_via_rollback.js + - ticket: SERVER-63129 + test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js + - ticket: SERVER-61864 + test_file: jstests/replsets/reconfig_may_not_remove_custom_wc_in_use.js + - ticket: SERVER-61864 + test_file: jstests/replsets/default_write_concern_race_with_config_tags.js + - ticket: SERVER-61864 + test_file: jstests/replsets/config_tags_race_with_default_write_concern.js + - ticket: SERVER-62710 + test_file: jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js + - ticket: SERVER-62759 + test_file: jstests/replsets/apply_ops_dropDatabase.js + - ticket: SERVER-63732 + test_file: jstests/sharding/shard_collection_basic.js + - ticket: SERVER-6491 + test_file: jstests/sharding/shard_key_index_must_exist.js + - ticket: SERVER-64142 + test_file: jstests/sharding/refine_collection_shard_key_basic.js + - ticket: SERVER-64485 + test_file: jstests/sharding/update_with_dollar_fields.js + - ticket: SERVER-64141 + test_file: jstests/core/in_with_mixed_values.js + - ticket: SERVER-63531 + test_file: jstests/replsets/buildindexes_false_commit_quorum.js + - ticket: SERVER-64780 + test_file: jstests/sharding/resharding_change_stream_namespace_filtering.js + - ticket: SERVER-65261 + test_file: jstests/replsets/capped_deletes.js + - ticket: SERVER-65773 + test_file: jstests/aggregation/agg_infinite_recursion.js + - ticket: SERVER-64315 + test_file: jstests/core/plan_cache_sbe.js + - ticket: SERVER-64315 + test_file: jstests/core/sbe/plan_cache_sbe_with_or_queries.js + - ticket: SERVER-64315 + test_file: jstests/core/sbe_plan_cache_autoparameterize_collscan.js + - ticket: SERVER-64315 + test_file: jstests/core/wildcard_index_cached_plans.js + - ticket: SERVER-66719 + test_file: jstests/replsets/dbhash_lock_acquisition.js + - ticket: SERVER-62272 + test_file: jstests/sharding/chunk_migration_with_schema_validation.js + - ticket: SERVER-66389 + test_file: jstests/core/where_multiple_plans.js + - ticket: SERVER-66089 + test_file: jstests/replsets/initial_sync_with_partial_transaction.js + - ticket: SERVER-65947 + test_file: jstests/sharding/migration_retries_on_write_conflict_exceptions.js + - ticket: SERVER-66264 + test_file: jstests/sharding/timeseries_multiple_mongos.js + - ticket: SERVER-66379 + test_file: jstests/core/or_to_in.js + + # Tests that should only be excluded from particular suites should be listed under that suite. + suites: + last-lts: + # Tests that should be excluded from ALL multiversion suites should be listed here. all: - - test_file: jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js - ticket: SERVER-88885 - - test_file: jstests/replsets/ignore_dbcheck_in_initial_sync.js - ticket: SERVER-80340 - - test_file: jstests/replsets/dbcheck_fails_on_write_concern_error.js - ticket: SERVER-88318 - - test_file: jstests/replsets/ignore_dbcheck_in_startup_recovery.js - ticket: SERVER-30832 - - test_file: jstests/replsets/ignore_dbcheck_in_rollback.js - ticket: SERVER-30832 - - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_pauses_watchdog_checks_during_storage_change.js - ticket: SERVER-83508 - - test_file: jstests/core/query/null_query_semantics.js - ticket: SERVER-21929 - - test_file: jstests/core/or_to_in.js - ticket: SERVER-21929 - - test_file: jstests/aggregation/sources/lookup/lookup_null_semantics.js - ticket: SERVER-21929 - - test_file: jstests/replsets/disallow_adding_initialized_node1.js - ticket: SERVER-35649 - - test_file: jstests/replsets/cluster_chaining_override.js - ticket: SERVER-37904 - - test_file: jstests/sharding/scaled_collection_stats.js - ticket: SERVER-43902 - - test_file: jstests/sharding/query/find_and_modify_with_partial_shard_key.js - ticket: SERVER-44422 - - test_file: jstests/sharding/query/delete_with_partial_shard_key.js - ticket: SERVER-44422 - - test_file: jstests/core/apply_ops_system_dot_views.js - ticket: SERVER-47469 - - test_file: jstests/concurrency/fsm_workloads/view_catalog_direct_system_writes.js - ticket: SERVER-47469 - - test_file: jstests/replsets/invalidate_sessions_on_stepdown.js - ticket: SERVER-47645 - - test_file: jstests/core/txns/no_writes_to_config_transactions_with_prepared_transaction.js - ticket: SERVER-48525 - - test_file: jstests/replsets/change_sync_source_in_initial_sync.js - ticket: SERVER-50320 - - test_file: jstests/sharding/awaitable_hello_primary_failures.js - ticket: SERVER-50415 - - test_file: jstests/replsets/server_status_repl_is_writable_primary.js - ticket: SERVER-50420 - - test_file: jstests/sharding/log_remote_op_wait.js - ticket: SERVER-50559 - - test_file: jstests/core/views/views_all_commands.js - ticket: SERVER-50640 - - test_file: jstests/sharding/read_write_concern_defaults_application.js - ticket: SERVER-50640 - - test_file: jstests/sharding/safe_secondary_reads_drop_recreate.js - ticket: SERVER-50640 - - test_file: jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js - ticket: SERVER-50640 - - test_file: jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js - ticket: SERVER-50640 - - test_file: jstests/replsets/node_restarts_no_oplog_entry_at_stable.js - ticket: SERVER-51049 - - test_file: jstests/replsets/awaitable_hello_errors_on_horizon_change.js - ticket: SERVER-51259 - - test_file: jstests/replsets/awaitable_hello_fcv_change.js - ticket: SERVER-51259 - - test_file: jstests/replsets/awaitable_hello_metrics_on_state_change.js - ticket: SERVER-51259 - - test_file: jstests/replsets/awaitable_hello_on_nodes_with_invalid_configs.js - ticket: SERVER-51259 - - test_file: jstests/replsets/awaitable_hello_stepdown_stepup.js - ticket: SERVER-51259 - - test_file: jstests/replsets/quiesce_mode.js - ticket: SERVER-51259 - - test_file: jstests/sharding/mongos_quiesce_mode.js - ticket: SERVER-51259 - - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js - ticket: SERVER-51943 - - test_file: jstests/core/txns/errors_on_committed_transaction.js - ticket: SERVER-52547 - - test_file: jstests/replsets/stepdown_kill_other_ops.js - ticket: SERVER-53431 - - test_file: jstests/sharding/transactions_reject_writes_for_moved_chunks.js - ticket: SERVER-55111 - - test_file: jstests/replsets/rollback_with_coalesced_txn_table_updates_during_oplog_application.js - ticket: SERVER-55305 - - test_file: jstests/replsets/rollback_with_coalesced_txn_table_updates_from_vectored_inserts.js - ticket: SERVER-55305 - - test_file: jstests/sharding/time_zone_info_mongos.js - ticket: SERVER-56371 - - test_file: jstests/concurrency/fsm_workloads/findAndModify_flip_location.js - ticket: SERVER-56377 - - test_file: jstests/sharding/txn_writes_during_movechunk.js - ticket: SERVER-56518 - - test_file: jstests/concurrency/fsm_workloads/collmod_writeconflict.js - ticket: SERVER-56772 - - test_file: jstests/core/elemmatch_index.js - ticket: SERVER-90747 - - test_file: jstests/sharding/cwwc_conflict_add_shard.js - ticket: SERVER-56800 - - test_file: jstests/sharding/database_versioning_all_commands.js - ticket: SERVER-56887 - - test_file: jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_retryable_write_retry.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js - ticket: SERVER-57617 - - test_file: jstests/replsets/tenant_migration_find_and_modify_retry.js - ticket: SERVER-57617 - - test_file: jstests/sharding/resharding_min_fetch_ts_with_txn.js - ticket: SERVER-57667 - - test_file: jstests/sharding/reshard_collection_joins_existing_operation.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_clones_duplicate_key.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_abort_command.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_clones_initial_data.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_commit.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_collection_cloner_resuming.js - ticket: SERVER-57667 - - test_file: jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_metrics.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_fails_on_nonempty_stash.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_txn_cloner.js - ticket: SERVER-57667 - - test_file: jstests/sharding/reshard_collection_joins_existing_operation.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_size_estimate.js - ticket: SERVER-57667 - - test_file: jstests/sharding/resharding_histogram_metrics.js - ticket: SERVER-57700 - - test_file: jstests/core/timeseries/timeseries_bucket_drop.js - ticket: SERVER-57729 - - test_file: jstests/sharding/resharding_metrics.js - ticket: SERVER-57766 - - test_file: jstests/sharding/query/pipeline_length_limit.js - ticket: SERVER-58203 - - test_file: jstests/replsets/tenant_migration_aborted_buildindex.js - ticket: SERVER-58353 - - test_file: jstests/sharding/implicit_default_write_concern_add_shard.js - ticket: SERVER-58696 - - test_file: jstests/replsets/write_concern_write_to_local.js - ticket: SERVER-58898 - - test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js - ticket: SERVER-59023 - - test_file: jstests/replsets/rollback_transaction_table.js - ticket: SERVER-59057 - - test_file: jstests/sharding/change_stream_show_migration_events.js - ticket: SERVER-59424 - - test_file: jstests/core/timeseries/timeseries_find.js - ticket: SERVER-59505 - - test_file: jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js - ticket: SERVER-59525 - - test_file: jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js - ticket: SERVER-59721 - - test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js - ticket: SERVER-59721 - - test_file: jstests/aggregation/expressions/date_add_subtract.js - ticket: SERVER-59765 - - test_file: jstests/sharding/test_resharding_test_fixture_shutdown_retry_needed.js - ticket: SERVER-59923 - - test_file: jstests/sharding/write_transactions_during_migration.js - ticket: SERVER-59952 - - test_file: jstests/sharding/retryable_writes.js - ticket: SERVER-59952 - - test_file: jstests/sharding/move_chunk_find_and_modify_with_write_retryability.js - ticket: SERVER-59952 - - test_file: jstests/sharding/cluster_time_across_add_shard.js - ticket: SERVER-60466 - - test_file: jstests/replsets/dont_set_invalid_rwconcern.js - ticket: SERVER-60817 - - test_file: jstests/replsets/dbhash_lock_acquisition.js - ticket: SERVER-60829 - - test_file: jstests/core/sbe/sbe_ixscan_explain.js - ticket: SERVER-61087 - - test_file: jstests/sharding/multi_writes_with_shard_version_ignored_dont_bubble_up_critical_section.js - ticket: SERVER-61127 - - test_file: jstests/replsets/tenant_migration_transaction_boundary.js - ticket: SERVER-61666 - - test_file: jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js - ticket: SERVER-61755 - - test_file: jstests/replsets/dbcheck.js - ticket: SERVER-61757 - - test_file: jstests/replsets/reconfig_may_not_remove_custom_wc_in_use.js - ticket: SERVER-61864 - - test_file: jstests/replsets/default_write_concern_race_with_config_tags.js - ticket: SERVER-61864 - - test_file: jstests/replsets/config_tags_race_with_default_write_concern.js - ticket: SERVER-61864 - - test_file: jstests/sharding/recover_multiple_migrations_on_stepup.js - ticket: SERVER-62245 - - test_file: jstests/sharding/migration_recovers_unfinished_migrations.js - ticket: SERVER-62296 - - test_file: jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js - ticket: SERVER-62710 - - test_file: jstests/replsets/apply_ops_dropDatabase.js - ticket: SERVER-62759 - - test_file: jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_collection_ttl.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_recipient_current_op.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_retry_session_migration.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_recipient_startup_recovery.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_network_error_via_rollback.js - ticket: SERVER-63122 - - test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js - ticket: SERVER-63129 - - test_file: jstests/core/dbstats.js - ticket: SERVER-63255 - - test_file: jstests/replsets/buildindexes_false_commit_quorum.js - ticket: SERVER-63531 - - test_file: jstests/core/plan_cache_sbe.js - ticket: SERVER-64315 - - test_file: jstests/core/sbe/plan_cache_sbe_with_or_queries.js - ticket: SERVER-64315 - - test_file: jstests/core/sbe_plan_cache_autoparameterize_collscan.js - ticket: SERVER-64315 - - test_file: jstests/core/wildcard_index_cached_plans.js - ticket: SERVER-64315 - - test_file: jstests/sharding/update_with_dollar_fields.js - ticket: SERVER-64485 - - test_file: jstests/sharding/shard_key_index_must_exist.js - ticket: SERVER-6491 - - test_file: jstests/sharding/query/shard_refuses_cursor_ownership.js - ticket: SERVER-65259 - - test_file: jstests/replsets/capped_deletes.js - ticket: SERVER-65261 - - test_file: jstests/aggregation/agg_infinite_recursion.js - ticket: SERVER-65773 - - test_file: jstests/sharding/migration_retries_on_write_conflict_exceptions.js - ticket: SERVER-65947 - - test_file: jstests/aggregation/match_no_swap_rand.js - ticket: SERVER-66072 - - test_file: jstests/sharding/timeseries_multiple_mongos.js - ticket: SERVER-66264 - - test_file: jstests/core/or_to_in.js - ticket: SERVER-66379 - - test_file: jstests/core/where_multiple_plans.js - ticket: SERVER-66389 - - test_file: jstests/core/clustered_collection_collation.js - ticket: SERVER-67105 - - test_file: jstests/core/clustered_collection_bounded_scan.js - ticket: SERVER-67105 - - test_file: jstests/sharding/min_max_key.js - ticket: SERVER-67529 - - test_file: jstests/core/clustered_collection_create_index_clustered.js - ticket: SERVER-67800 - - test_file: jstests/core/clustered_collection_creation.js - ticket: SERVER-67800 - - test_file: jstests/sharding/resharding_temp_ns_routing_info_unsharded.js - ticket: SERVER-68628 - - test_file: jstests/sharding/move_chunk_concurrent_cloning.js - ticket: SERVER-81201 - - test_file: jstests/sharding/move_chunk_interrupt_postimage.js - ticket: SERVER-68728 - - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_election_during_storage_change.js - ticket: SERVER-69861 - - test_file: jstests/core/timeseries/timeseries_filter_extended_range.js - ticket: SERVER-69952 - - test_file: jstests/aggregation/expressions/switch_errors.js - ticket: SERVER-70190 - - test_file: jstests/replsets/wtimeout_too_large.js - ticket: SERVER-70360 - - test_file: jstests/core/cover_null_queries.js - ticket: SERVER-70436 - - test_file: jstests/sharding/all_collection_stats.js - ticket: SERVER-71477 - - test_file: jstests/sharding/drop_collection.js - ticket: SERVER-71689 - - test_file: jstests/sharding/index_and_collection_option_propagation.js - ticket: SERVER-71819 - - test_file: jstests/core/clustered/clustered_collection_collation.js - ticket: SERVER-72224 - - test_file: jstests/core/project_with_collation.js - ticket: SERVER-72416 - - test_file: jstests/core/collation.js - ticket: SERVER-72416 - - test_file: jstests/core/or_to_in.js - ticket: SERVER-72416 - - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/oplog_rollover.js - ticket: SERVER-72422 - - test_file: jstests/sharding/timeseries_cluster_indexstats.js - ticket: SERVER-72620 - - test_file: jstests/replsets/quiesce_mode_fails_elections.js - ticket: SERVER-72774 - - test_file: jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js - ticket: SERVER-73110 - - test_file: jstests/core/doc_validation_with_now_variable.js - ticket: SERVER-73125 - - test_file: jstests/sharding/resharding_interrupt_before_create_state_machine.js - ticket: SERVER-74647 - - test_file: jstests/sharding/merge_let_params_size_estimation.js - ticket: SERVER-74806 - - test_file: jstests/core/query/elemmatch/elemmatch_or_pushdown_paths.js - ticket: SERVER-74954 - - test_file: jstests/core/command_let_variables.js - ticket: SERVER-75356 - - test_file: jstests/core/field_name_validation.js - ticket: SERVER-75517 - - test_file: jstests/sharding/conn_pool_stats.js - ticket: SERVER-75911 - - test_file: jstests/sharding/database_versioning_all_commands.js - ticket: SERVER-75911 - - test_file: jstests/noPassthrough/clustered_collection_sorted_scan.js - ticket: SERVER-76102 - - test_file: jstests/sharding/implicit_create_collection_triggered_by_DDLs.js - ticket: SERVER-76489 - - test_file: jstests/sharding/shard_keys_with_dollar_sign.js - ticket: SERVER-76948 - - test_file: jstests/sharding/resharding_update_tag_zones_large.js - ticket: SERVER-76988 - - test_file: jstests/core/timeseries/timeseries_create_collection.js - ticket: SERVER-77382 - - test_file: jstests/change_streams/oplog_rewrite/change_stream_nor_rewrite.js - ticket: SERVER-78650 - - test_file: jstests/core/timeseries/timeseries_groupby_reorder.js - ticket: SERVER-79136 - - test_file: jstests/sharding/fsync_deadlock.js - ticket: SERVER-78793 - - test_file: jstests/sharding/fsync_lock_ddl_lock.js - ticket: SERVER-78157 - - test_file: jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js - ticket: SERVER-78157 - - test_file: jstests/sharding/fsync_lock_unlock.js - ticket: SERVER-78149 - - test_file: jstests/core/timeseries/nondefault_collation.js - ticket: SERVER-81013 - - test_file: jstests/noPassthrough/read_preference_metrics.js - ticket: SERVER-66018 - - test_file: jstests/sharding/timeseries_insert_targeting_normalize_metadata.js - ticket: SERVER-80203 - - test_file: jstests/sharding/block_chunk_migrations_without_hashed_shard_key_index.js - ticket: SERVER-80947 - - test_file: jstests/sharding/delete_range_deletion_tasks_on_dropped_hashed_shard_key_index.js - ticket: SERVER-80947 - - test_file: jstests/sharding/move_chunk_basic.js - ticket: SERVER-81201 - - test_file: jstests/sharding/flushRoutingTableCacheUpdates_enforced_on_collections.js - ticket: SERVER-81985 - - test_file: jstests/sharding/query/merge_nondefault_read_concern.js - ticket: SERVER-69244 - - test_file: jstests/core/timeseries/timeseries_project_pushdown.js - ticket: SERVER-82447 - - test_file: jstests/core/match_with_and_or_lockstep_enumeration.js - ticket: SERVER-83091 - - test_file: jstests/sharding/coordinate_txn_recover_on_stepup_with_tickets_exhausted.js - ticket: SERVER-82883 - - test_file: jstests/core/nested_or_duplicate_predicates_index_scan.js - ticket: SERVER-84013 - - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js - ticket: SERVER-82353 - - test_file: jstests/core/timeseries/timeseries_collmod.js - ticket: SERVER-83370 - - test_file: jstests/core/timeseries/timeseries_lookup.js - ticket: SERVER-79637 - - test_file: jstests/noPassthrough/notablescan.js - ticket: SERVER-83119 - - test_file: jstests/sharding/clustered_coll_scan.js - ticket: SERVER-83119 - - test_file: jstests/change_streams/split_large_event_with_other_stages.js - ticket: SERVER-85848 - - test_file: jstests/core/txns/out_not_blocked_by_txn.js - ticket: SERVER-72703 - - test_file: jstests/core/query/bittest.js - ticket: SERVER-86419 - - test_file: jstests/core/query/internal_strip_invalid_assignment.js - ticket: SERVER-82717 - - test_file: jstests/core/timeseries/timeseries_project.js - ticket: SERVER-87987 - - test_file: jstests/core/command_let_variables_expressions.js - ticket: SERVER-89268 - - test_file: jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js - ticket: SERVER-79049 - - test_file: jstests/noPassthrough/server_status_metrics_retryable_writes.js - ticket: SERVER-86253 - - test_file: jstests/replsets/log_wt_stats_during_secondary_oplog_application.js - ticket: SERVER-78509 - - test_file: jstests/replsets/reconstruct_prepared_transactions_initial_sync.js - ticket: SERVER-89618 - - test_file: jstests/core/timeseries/timeseries_geonear_lookup.js - ticket: SERVER-81454 - - test_file: jstests/core/timeseries/timeseries_computed_field.js - ticket: SERVER-88072 - - test_file: jstests/core/bypass_empty_ts_replacement.js - ticket: SERVER-88750 - - test_file: jstests/core/bypass_empty_ts_replacement_timeseries.js - ticket: SERVER-88750 - - test_file: jstests/aggregation/unwind_sort.js - ticket: SERVER-90485 - - test_file: jstests/aggregation/group_by_objectid.js - ticket: SERVER-90173 - - test_file: jstests/replsets/catchup_ignores_old_heartbeats.js - ticket: SERVER-86674 - - test_file: jstests/core/wildcard_index_validindex.js - ticket: SERVER-93105 - suites: null + - ticket: SERVER-49471 + test_file: jstests/replsets/apply_prepare_txn_write_conflict_robustness.js + - ticket: SERVER-47645 + test_file: jstests/replsets/invalidate_sessions_on_stepdown.js + - ticket: SERVER-47773 + test_file: jstests/core/geo_near_tailable.js + - ticket: SERVER-47469 + test_file: jstests/core/apply_ops_system_dot_views.js + - ticket: SERVER-48614 + test_file: jstests/core/wildcard_index_partial_index.js + - ticket: SERVER-51120 + test_file: jstests/core/merge_sort_collation.js + - ticket: SERVER-48525 + test_file: jstests/core/txns/no_writes_to_config_transactions_with_prepared_transaction.js + - ticket: SERVER-47810 + test_file: jstests/change_streams/report_post_batch_resume_token.js + - ticket: SERVER-50140 + test_file: jstests/replsets/initial_sync_fails_unclean_restart.js + - ticket: SERVER-49987 + test_file: jstests/core/hello_response_validation.js + - ticket: SERVER-49987 + test_file: jstests/replsets/check_replication_hello_response_fields.js + - ticket: SERVER-50640 + test_file: jstests/core/views/views_all_commands.js + - ticket: SERVER-56887 + test_file: jstests/sharding/database_versioning_all_commands.js + - ticket: SERVER-49988 + test_file: jstests/sharding/hello_response_validation.js + - ticket: SERVER-53335 + test_file: jstests/sharding/query/collation_shard_targeting_hashed_shard_key.js + - ticket: SERVER-50640 + test_file: jstests/sharding/read_write_concern_defaults_application.js + - ticket: SERVER-50640 + test_file: jstests/sharding/safe_secondary_reads_drop_recreate.js + - ticket: SERVER-50640 + test_file: jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js + - ticket: SERVER-50640 + test_file: jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js + - ticket: SERVER-64184 + test_file: jstests/noPassthrough/aggregate_metrics.js + - ticket: SERVER-49991 + test_file: jstests/replsets/get_replication_info_helper.js + - ticket: SERVER-47469 + test_file: jstests/concurrency/fsm_workloads/view_catalog_direct_system_writes.js + - ticket: SERVER-49990 + test_file: jstests/replsets/secondaryOk_slaveOk_aliases.js + - ticket: SERVER-43902 + test_file: jstests/sharding/scaled_collection_stats.js + - ticket: SERVER-50417 + test_file: jstests/replsets/no_disconnect_on_stepdown.js + - ticket: SERVER-50417 + test_file: jstests/replsets/not_primary_unacknowledged_write.js + - ticket: SERVER-50417 + test_file: jstests/replsets/read_operations_during_step_down.js + - ticket: SERVER-50417 + test_file: jstests/replsets/read_operations_during_step_up.js + - ticket: SERVER-50320 + test_file: jstests/replsets/change_sync_source_in_initial_sync.js + - ticket: SERVER-50415 + test_file: jstests/sharding/awaitable_hello_primary_failures.js + - ticket: SERVER-50420 + test_file: jstests/replsets/server_status_repl_is_writable_primary.js + - ticket: SERVER-51262 + test_file: jstests/replsets/read_operations_during_rollback.js + - ticket: SERVER-51262 + test_file: jstests/replsets/transactions_on_secondaries_not_allowed.js + - ticket: SERVER-43904 + test_file: jstests/replsets/election_handoff_skips_unelectable_nodes.js + - ticket: SERVER-51259 + test_file: jstests/replsets/awaitable_hello_errors_on_horizon_change.js + - ticket: SERVER-51259 + test_file: jstests/replsets/awaitable_hello_fcv_change.js + - ticket: SERVER-51259 + test_file: jstests/replsets/awaitable_hello_metrics_on_state_change.js + - ticket: SERVER-51259 + test_file: jstests/replsets/awaitable_hello_on_nodes_with_invalid_configs.js + - ticket: SERVER-51259 + test_file: jstests/replsets/awaitable_hello_stepdown_stepup.js + - ticket: SERVER-51259 + test_file: jstests/replsets/quiesce_mode.js + - ticket: SERVER-51259 + test_file: jstests/sharding/mongos_quiesce_mode.js + - ticket: SERVER-50412 + test_file: jstests/replsets/not_primary_errors_returned_if_client_sends_helloOk.js + - ticket: SERVER-50412 + test_file: jstests/sharding/mongos_helloOk_protocol.js + - ticket: SERVER-51049 + test_file: jstests/replsets/node_restarts_no_oplog_entry_at_stable.js + - ticket: SERVER-50414 + test_file: jstests/replsets/not_primary_errors_returned_during_rollback_if_helloOk.js + - ticket: SERVER-50769 + test_file: jstests/sharding/change_stream_empty_apply_ops.js + - ticket: SERVER-52547 + test_file: jstests/core/txns/errors_on_committed_transaction.js + - ticket: SERVER-53612 + test_file: jstests/replsets/election_handoff_not_immediately_electable.js + - ticket: SERVER-50486 + test_file: jstests/replsets/dont_refresh_session_prepare_secondary.js + - ticket: SERVER-53932 + test_file: jstests/replsets/rollback_reconstructs_transactions_prepared_before_stable.js + - ticket: SERVER-53431 + test_file: jstests/replsets/stepdown_kill_other_ops.js + - ticket: SERVER-53394 + test_file: jstests/sharding/sharding_task_executor_pool_matching_policy.js + - ticket: SERVER-53849 + test_file: jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js + - ticket: SERVER-53985 + test_file: jstests/replsets/unconditional_step_down.js + - ticket: SERVER-54366 + test_file: jstests/replsets/force_shutdown_primary.js + - ticket: SERVER-52953 + test_file: jstests/core/geo_near_point_query.js + - ticket: SERVER-48179 + test_file: jstests/replsets/reconfig_removes_node_in_rollback.js + - ticket: SERVER-56371 + test_file: jstests/sharding/time_zone_info_mongos.js + - ticket: SERVER-55305 + test_file: jstests/replsets/rollback_with_coalesced_txn_table_updates_during_oplog_application.js + - ticket: SERVER-55305 + test_file: jstests/replsets/rollback_with_coalesced_txn_table_updates_from_vectored_inserts.js + - ticket: SERVER-53760 + test_file: jstests/core/sort_spill_estimate_data_size.js + - ticket: SERVER-56772 + test_file: jstests/concurrency/fsm_workloads/collmod_writeconflict.js + - ticket: SERVER-55111 + test_file: jstests/sharding/transactions_reject_writes_for_moved_chunks.js + - ticket: SERVER-59952 + test_file: jstests/sharding/write_transactions_during_migration.js + - ticket: SERVER-59952 + test_file: jstests/sharding/retryable_writes.js + - ticket: SERVER-59952 + test_file: jstests/sharding/move_chunk_find_and_modify_with_write_retryability.js + - ticket: SERVER-57117 + test_file: jstests/sharding/read_pref_with_hedging_mode.js + - ticket: SERVER-56377 + test_file: jstests/concurrency/fsm_workloads/findAndModify_flip_location.js + - ticket: SERVER-56518 + test_file: jstests/sharding/txn_writes_during_movechunk.js + - ticket: SERVER-57476 + test_file: jstests/replsets/assert_on_prepare_conflict_with_hole.js + - ticket: SERVER-21929 + test_file: jstests/core/null_query_semantics.js + - ticket: SERVER-21929 + test_file: jstests/core/or_to_in.js + - ticket: SERVER-21929 + test_file: jstests/aggregation/sources/lookup/lookup_null_semantics.js + - ticket: SERVER-52906 + test_file: jstests/sharding/prepare_transaction_then_migrate.js + - ticket: SERVER-52906 + test_file: jstests/sharding/migration_waits_for_majority_commit.js + - ticket: SERVER-52906 + test_file: jstests/sharding/migration_ignore_interrupts_1.js + - ticket: SERVER-52906 + test_file: jstests/sharding/movechunk_interrupt_at_primary_stepdown.js + - ticket: SERVER-52906 + test_file: jstests/sharding/movechunk_parallel.js + - ticket: SERVER-52906 + test_file: jstests/sharding/txn_writes_during_movechunk.js + - ticket: SERVER-50559 + test_file: jstests/sharding/log_remote_op_wait.js + - ticket: SERVER-37904 + test_file: jstests/replsets/cluster_chaining_override.js + - ticket: SERVER-57729 + test_file: jstests/core/timeseries/timeseries_bucket_drop.js + - ticket: SERVER-59505 + test_file: jstests/core/timeseries/timeseries_find.js + - ticket: SERVER-56800 + test_file: jstests/sharding/cwwc_conflict_add_shard.js + - ticket: SERVER-57262 + test_file: jstests/replsets/catchup_takeover_with_higher_config.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_min_fetch_ts_with_txn.js + - ticket: SERVER-57667 + test_file: jstests/sharding/reshard_collection_joins_existing_operation.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_clones_duplicate_key.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_abort_command.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_clones_initial_data.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_commit.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_collection_cloner_resuming.js + - ticket: SERVER-57667 + test_file: jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_metrics.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_fails_on_nonempty_stash.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_txn_cloner.js + - ticket: SERVER-57667 + test_file: jstests/sharding/reshard_collection_joins_existing_operation.js + - ticket: SERVER-57667 + test_file: jstests/sharding/resharding_size_estimate.js + - ticket: SERVER-58898 + test_file: jstests/replsets/write_concern_write_to_local.js + - ticket: SERVER-59057 + test_file: jstests/replsets/rollback_transaction_table.js + - ticket: SERVER-58353 + test_file: jstests/replsets/tenant_migration_aborted_buildindex.js + - ticket: SERVER-58696 + test_file: jstests/sharding/implicit_default_write_concern_add_shard.js + - ticket: SERVER-59197 + test_file: jstests/replsets/sessions_collection_reaping.js + - ticket: SERVER-58203 + test_file: jstests/sharding/query/pipeline_length_limit.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_retryable_write_retry.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js + - ticket: SERVER-57617 + test_file: jstests/replsets/tenant_migration_find_and_modify_retry.js + - ticket: SERVER-51943 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-59525 + test_file: jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js + - ticket: SERVER-59023 + test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js + - ticket: SERVER-59765 + test_file: jstests/aggregation/expressions/date_add_subtract.js + - ticket: SERVER-59424 + test_file: jstests/sharding/change_stream_show_migration_events.js + - ticket: SERVER-59108 + test_file: jstests/replsets/stepdown_race_with_transaction.js + - ticket: SERVER-58636 + test_file: jstests/replsets/initial_sync_replicate_drop_mid_secondary_batch.js + - ticket: SERVER-59613 + test_file: jstests/aggregation/range.js + - ticket: SERVER-59923 + test_file: jstests/sharding/test_resharding_test_fixture_shutdown_retry_needed.js + - ticket: SERVER-61087 + test_file: jstests/core/sbe/sbe_ixscan_explain.js + - ticket: SERVER-56127 + test_file: jstests/sharding/retryable_writes_nested_shard_key.js + - ticket: SERVER-54909 + test_file: jstests/replsets/replSetGetStatus_member_wall_times.js + - ticket: SERVER-35649 + test_file: jstests/replsets/disallow_adding_initialized_node1.js + - ticket: SERVER-57700 + test_file: jstests/sharding/resharding_histogram_metrics.js + - ticket: SERVER-59721 + test_file: jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js + - ticket: SERVER-59721 + test_file: jstests/sharding/resharding_secondary_recovers_temp_ns_metadata.js + - ticket: SERVER-57588 + test_file: jstests/core/match_numeric_components.js + - ticket: SERVER-60682 + test_file: jstests/sharding/coordinate_txn_commit_with_tickets_exhausted.js + - ticket: SERVER-60685 + test_file: jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js + - ticket: SERVER-58506 + test_file: jstests/noPassthrough/getParameterWithDetails.js + - ticket: SERVER-57766 + test_file: jstests/sharding/resharding_metrics.js + - ticket: SERVER-61757 + test_file: jstests/replsets/dbcheck.js + - ticket: SERVER-61955 + test_file: jstests/auth/dbcheck.js + - ticket: SERVER-61666 + test_file: jstests/replsets/tenant_migration_transaction_boundary.js + - ticket: SERVER-62245 + test_file: jstests/sharding/recover_multiple_migrations_on_stepup.js + - ticket: SERVER-62296 + test_file: jstests/sharding/migration_recovers_unfinished_migrations.js + - ticket: SERVER-60817 + test_file: jstests/replsets/dont_set_invalid_rwconcern.js + - ticket: SERVER-61977 + test_file: jstests/replsets/rollback_during_step_up.js + - ticket: SERVER-61755 + test_file: jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js + - ticket: SERVER-62147 + test_file: jstests/core/exhaust.js + - ticket: SERVER-63141 + test_file: jstests/aggregation/lookup_let_optimization.js + - ticket: SERVER-63129 + test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js + - ticket: SERVER-61864 + test_file: jstests/replsets/reconfig_may_not_remove_custom_wc_in_use.js + - ticket: SERVER-61864 + test_file: jstests/replsets/default_write_concern_race_with_config_tags.js + - ticket: SERVER-61864 + test_file: jstests/replsets/config_tags_race_with_default_write_concern.js + - ticket: SERVER-62710 + test_file: jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js + - ticket: SERVER-62759 + test_file: jstests/replsets/apply_ops_dropDatabase.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_collection_ttl.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_current_op.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_retry_session_migration.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_startup_recovery.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_network_error_via_rollback.js + - ticket: SERVER-63732 + test_file: jstests/sharding/shard_collection_basic.js + - ticket: SERVER-6491 + test_file: jstests/sharding/shard_key_index_must_exist.js + - ticket: SERVER-63531 + test_file: jstests/replsets/buildindexes_false_commit_quorum.js + - ticket: SERVER-64485 + test_file: jstests/sharding/update_with_dollar_fields.js + - ticket: SERVER-64142 + test_file: jstests/sharding/refine_collection_shard_key_basic.js + - ticket: SERVER-65261 + test_file: jstests/replsets/capped_deletes.js + - ticket: SERVER-65773 + test_file: jstests/aggregation/agg_infinite_recursion.js + - ticket: SERVER-64315 + test_file: jstests/core/plan_cache_sbe.js + - ticket: SERVER-64315 + test_file: jstests/core/sbe/plan_cache_sbe_with_or_queries.js + - ticket: SERVER-64315 + test_file: jstests/core/sbe_plan_cache_autoparameterize_collscan.js + - ticket: SERVER-64315 + test_file: jstests/core/wildcard_index_cached_plans.js + - ticket: SERVER-66719 + test_file: jstests/replsets/dbhash_lock_acquisition.js + - ticket: SERVER-62272 + test_file: jstests/sharding/chunk_migration_with_schema_validation.js + - ticket: SERVER-66389 + test_file: jstests/core/where_multiple_plans.js + - ticket: SERVER-66089 + test_file: jstests/replsets/initial_sync_with_partial_transaction.js + - ticket: SERVER-65947 + test_file: jstests/sharding/migration_retries_on_write_conflict_exceptions.js + - ticket: SERVER-66264 + test_file: jstests/sharding/timeseries_multiple_mongos.js + - ticket: SERVER-66379 + test_file: jstests/core/or_to_in.js + + # Tests that should only be excluded from particular suites should be listed under that suite. + suites: diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 120d17c883a..6c7c873bf09 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -57,7 +57,6 @@ include: - filename: etc/evergreen_yml_components/variants/task_generation.yml - filename: etc/evergreen_yml_components/variants/sanitizer.yml - filename: etc/evergreen_yml_components/variants/in_memory.yml -- filename: etc/evergreen_yml_components/variants/ninja.yml variables: - &libfuzzertests @@ -73,11 +72,13 @@ variables: cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - windows-vsCurrent-small + modules: + - enterprise expansions: &enterprise-windows-nopush-expansions-template additional_package_targets: archive-mongocryptd archive-mongocryptd-debug msi exe: ".exe" content_type: application/zip - compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 num_scons_link_jobs_available: 0.5 python: '/cygdrive/c/python/python37/python.exe' ext: zip @@ -91,9 +92,6 @@ variables: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-large - name: burn_in_tests_gen - name: .aggfuzzer .common - name: audit @@ -123,6 +121,8 @@ variables: name: enterprise-rhel-70-64-bit display_name: "Enterprise RHEL 7.0" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel70-small expansions: &enterprise-rhel-70-64-bit-expansions-template @@ -141,14 +141,10 @@ variables: packager_distro: rhel70 repo_edition: enterprise scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - rhel70 - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel70 - name: compile_build_tools_next_TG distros: - rhel70 @@ -177,28 +173,27 @@ variables: - name: libunwind_tests - name: .logical_session_cache .one_sec - name: .ocsp - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough - name: .replica_sets .multi_oplog - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: sharding_auth_audit_gen - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt - name: unittest_shell_hang_analyzer_gen - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl + - name: test_packages + distros: + - ubuntu2004-package - name: selinux_rhel7_enterprise + - name: .publish + distros: + - rhel70-small - name: generate_buildid_to_debug_symbols_mapping @@ -224,10 +219,10 @@ buildvariants: - rhel80-medium expansions: resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-http-client=on --link-model=dynamic --use-diagnostic-latches=on --modules= + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-free-mon=on --enable-http-client=on --link-model=dynamic scons_cache_scope: shared scons_cache_mode: all - test_flags: --excludeWithAnyTags=requires_http_client --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=requires_http_client target_resmoke_time: 15 max_sub_suites: 5 num_scons_link_jobs_available: 0.99 @@ -250,6 +245,7 @@ buildvariants: - name: .clustered_collections - name: .misc_js - name: disk_wiredtiger + - name: free_monitoring - name: .integration !.audit - name: .jscore .common - name: jsCore_txns_large_txns_format @@ -288,11 +284,11 @@ buildvariants: expansions: use_wt_develop: true resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-http-client=on --link-model=dynamic --use-diagnostic-latches=on --modules= + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-free-mon=on --enable-http-client=on --link-model=dynamic scons_cache_mode: all scons_cache_scope: shared num_scons_link_jobs_available: 0.99 - test_flags: --excludeWithAnyTags=requires_http_client --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=requires_http_client - name: linux-64-duroff display_name: Linux (No Journal) @@ -300,13 +296,13 @@ buildvariants: run_on: - rhel80-small expansions: &linux-64-required-duroff-expansions - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-http-client=off --link-model=dynamic --use-diagnostic-latches=on --modules= + compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-free-mon=off --enable-http-client=off --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: targeted # Running WiredTiger with --nojournal in a replica set is no longer supported, so this variant # does not include replica set tests. Since transactions are only supported on replica sets, we # exclude those tests as well. - test_flags: --nojournal --excludeWithAnyTags=requires_journaling,requires_replication,requires_sharding,uses_transactions,requires_http_client --enableEnterpriseTests=off + test_flags: --nojournal --excludeWithAnyTags=requires_journaling,requires_replication,requires_sharding,uses_transactions,requires_http_client scons_cache_mode: all scons_cache_scope: shared num_scons_link_jobs_available: 0.99 @@ -345,21 +341,43 @@ buildvariants: tasks: - name: tla_plus -- &enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-dynamic-v4gcc-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain GCC DEBUG" +- name: enterprise-rhel80-ninja + display_name: "Ninja Build: RHEL 8.0" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions + stepback: false + expansions: + compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic + repo_edition: enterprise + has_packages: false + tasks: + - name: compile_ninja_next_TG + distros: + - rhel80-build + - name: compile_ninja_TG + distros: + - rhel80-build + +- &enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-dynamic-v4gcc-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain GCC DEBUG" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise + run_on: + - rhel80-small + expansions: &enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --use-diagnostic-latches=on + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars has_packages: false scons_cache_scope: shared scons_cache_mode: all large_distro_name: rhel80-medium num_scons_link_jobs_available: 0.99 - tasks: &enterprise-rhel8-dynamic-v4gcc-debug-experimental-tasks + tasks: &enterprise-rhel80-dynamic-v4gcc-debug-experimental-tasks - name: compile_test_and_package_parallel_core_stream_TG distros: - rhel80-xlarge @@ -395,6 +413,7 @@ buildvariants: - name: jsCore_txns_large_txns_format - name: json_schema - name: libunwind_tests + - name: mqlrun - name: .multi_shard !.multiversion - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: .read_write_concern .large !.multiversion @@ -411,7 +430,6 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.multiversion @@ -419,6 +437,7 @@ buildvariants: - name: .sharding .txns !.multiversion - name: .sharding .common !.multiversion - name: snmp + - name: .stitch - name: secondary_reads_passthrough_gen - name: server_discovery_and_monitoring_json_test_TG - name: .serverless !.multiversion @@ -427,33 +446,32 @@ buildvariants: - name: server_selection_json_test_TG distros: - rhel80-xlarge - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl -- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-dynamic-v4clang-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain Clang DEBUG" +- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-dynamic-v4clang-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain Clang DEBUG" expansions: - <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --use-diagnostic-latches=on + <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars -- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-dynamic-v4gcc-cxx20-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain GCC C++20 DEBUG" +- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-dynamic-v4gcc-cxx20-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain GCC C++20 DEBUG" expansions: - <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --use-diagnostic-latches=on + <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 -- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-dynamic-v4clang-cxx20-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain Clang C++20 DEBUG" +- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-dynamic-v4clang-cxx20-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain Clang C++20 DEBUG" expansions: - <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --use-diagnostic-latches=on + <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 -- name: enterprise-rhel-8-64-bit-coverage - display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage" +- name: enterprise-rhel-80-64-bit-coverage + display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage" + modules: + - enterprise run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -461,7 +479,7 @@ buildvariants: expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov - compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic large_distro_name: rhel80-medium multiversion_platform: rhel80 multiversion_edition: enterprise @@ -477,7 +495,7 @@ buildvariants: use_scons_cache: false gcov_tool: /opt/mongodbtoolchain/v3/bin/gcov num_scons_link_jobs_available: 0.99 - tasks: &enterprise-rhel-8-64-bit-coverage-tasks + tasks: &enterprise-rhel-80-64-bit-coverage-tasks - name: compile_test_and_package_parallel_core_stream_TG distros: - rhel80-large @@ -524,7 +542,6 @@ buildvariants: #- name: sasl #- name: search #- name: search_auth - #- name: search_pinned_connections_auth #- name: search_ssl #- name: secondary_reads_passthrough_gen #- name: session_jscore_passthrough @@ -533,8 +550,10 @@ buildvariants: #- name: snmp #- name: update_fuzzer_gen -- name: enterprise-rhel-8-64-bit-coverage-clang - display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage (clang)" +- name: enterprise-rhel-80-64-bit-coverage-clang + display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (clang)" + modules: + - enterprise run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -542,7 +561,7 @@ buildvariants: expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov - compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --link-model=dynamic large_distro_name: rhel80-medium multiversion_platform: rhel80 multiversion_edition: enterprise @@ -562,10 +581,12 @@ buildvariants: gcov_tool: /opt/mongodbtoolchain/v3/bin/gcov num_scons_link_jobs_available: 0.99 tasks: - *enterprise-rhel-8-64-bit-coverage-tasks + *enterprise-rhel-80-64-bit-coverage-tasks -- name: enterprise-rhel-8-64-bit-coverage-experimental - display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage (v4)" +- name: enterprise-rhel-80-64-bit-coverage-experimental + display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (v4)" + modules: + - enterprise run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -573,7 +594,7 @@ buildvariants: expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov - compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_gcc.vars --link-model=dynamic large_distro_name: rhel80-medium multiversion_platform: rhel80 multiversion_edition: enterprise @@ -590,10 +611,12 @@ buildvariants: gcov_tool: /opt/mongodbtoolchain/v4/bin/gcov num_scons_link_jobs_available: 0.99 tasks: - *enterprise-rhel-8-64-bit-coverage-tasks + *enterprise-rhel-80-64-bit-coverage-tasks -- name: enterprise-rhel-8-64-bit-coverage-clang-experimental - display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage (v4 clang)" +- name: enterprise-rhel-80-64-bit-coverage-clang-experimental + display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (v4 clang)" + modules: + - enterprise run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -601,7 +624,7 @@ buildvariants: expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov - compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars --link-model=dynamic large_distro_name: rhel80-medium multiversion_platform: rhel80 multiversion_edition: enterprise @@ -618,15 +641,17 @@ buildvariants: gcov_tool: /opt/mongodbtoolchain/v4/bin/llvm-cov gcov num_scons_link_jobs_available: 0.99 tasks: - *enterprise-rhel-8-64-bit-coverage-tasks + *enterprise-rhel-80-64-bit-coverage-tasks -- name: enterprise-rhel-8-64-bit-bson-column - display_name: "~ Shared Library Enterprise RHEL 8 (BSONColumn validate)" +- name: enterprise-rhel-80-64-bit-bson-column + display_name: "~ Shared Library Enterprise RHEL 8.0 (BSONColumn validate)" + modules: + - enterprise run_on: - rhel80-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise has_packages: false @@ -637,6 +662,7 @@ buildvariants: target_resmoke_time: 10 max_sub_suites: 5 large_distro_name: rhel80-medium + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion num_scons_link_jobs_available: 0.99 test_flags: >- --mongodSetParameters="{roundtripBsonColumnOnValidate: true}" @@ -690,6 +716,7 @@ buildvariants: - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests + - name: mqlrun - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: .query_fuzzer - name: .read_write_concern .large @@ -716,53 +743,52 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: snmp + - name: .stitch - name: .updatefuzzer - name: secondary_reads_passthrough_gen - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG distros: - rhel80-xlarge - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl -#- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template -# name: enterprise-rhel8-dynamic-v4gcc-cxx20-debug-pm-1328-experimental -# display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain GCC C++20 DEBUG + PM-1328" +#- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template +# name: enterprise-rhel80-dynamic-v4gcc-cxx20-debug-pm-1328-experimental +# display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain GCC C++20 DEBUG + PM-1328" # cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter # expansions: -# <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions +# <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions # compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --disable-warnings-as-errors -- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-dynamic-v4clang-cxx20-debug-pm-1328-experimental - display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain Clang C++20 DEBUG + PM-1328" +- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-dynamic-v4clang-cxx20-debug-pm-1328-experimental + display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain Clang C++20 DEBUG + PM-1328" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --use-diagnostic-latches=on + <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* -- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-v4gcc-cxx20-pm-1328-experimental - display_name: "~ Enterprise RHEL 8 v4 Toolchain GCC C++20 + PM-1328" +- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-v4gcc-cxx20-pm-1328-experimental + display_name: "~ Enterprise RHEL 8.0 v4 Toolchain GCC C++20 + PM-1328" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions - compile_flags: --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --disable-warnings-as-errors --use-diagnostic-latches=on + <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + compile_flags: --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --disable-warnings-as-errors -- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel8-v4clang-cxx20-pm-1328-experimental - display_name: "~ Enterprise RHEL 8 v4 Toolchain Clang C++20 + PM-1328" +- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel80-v4clang-cxx20-pm-1328-experimental + display_name: "~ Enterprise RHEL 8.0 v4 Toolchain Clang C++20 + PM-1328" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions - compile_flags: --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --use-diagnostic-latches=on + <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + compile_flags: --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* - name: stm-daily-cron + modules: + - enterprise display_name: "~ STM Daily Cron" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: @@ -776,6 +802,17 @@ buildvariants: - name: powercycle_sentinel - name: powercycle_smoke_skip_compile_gen +- name: security-daily-cron + modules: + - enterprise + display_name: "~ Security Daily Cron" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + run_on: + - rhel80-small + stepback: false + tasks: + - name: blackduck_scanner + ########################################### # Windows buildvariants # ########################################### @@ -788,7 +825,7 @@ buildvariants: expansions: exe: ".exe" content_type: application/zip - compile_flags: --dbg=on --opt=on --win-version-min=win10 -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") MONGO_DISTMOD=windows --use-diagnostic-latches=on --modules= + compile_flags: --dbg=on --opt=on --win-version-min=win10 -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") MONGO_DISTMOD=windows num_scons_link_jobs_available: 0.25 python: '/cygdrive/c/python/python37/python.exe' ext: zip @@ -796,14 +833,11 @@ buildvariants: multiversion_platform: windows multiversion_edition: enterprise large_distro_name: windows-vsCurrent-large - test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls,requires_latch_analyzer --enableEnterpriseTests=off + test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls tasks: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-xlarge - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-xlarge - name: compile_build_tools_next_TG distros: - windows-vsCurrent-xlarge @@ -818,6 +852,7 @@ buildvariants: distros: - windows-vsCurrent-large - name: disk_wiredtiger + - name: free_monitoring - name: initial_sync_fuzzer_gen - name: .integration !.audit distros: @@ -853,6 +888,8 @@ buildvariants: name: enterprise-windows-required display_name: "! Enterprise Windows" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter + modules: + - enterprise run_on: - windows-vsCurrent-small expansions: @@ -880,9 +917,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-xlarge - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-xlarge - name: compile_build_tools_next_TG distros: - windows-vsCurrent-xlarge @@ -907,6 +941,7 @@ buildvariants: - name: jsCore_ese - name: jsCore_txns_large_txns_format - name: .jstestfuzz .common + - name: mqlrun - name: noPassthrough_gen - name: noPassthroughWithMongod_gen - name: .replica_sets .common @@ -932,7 +967,7 @@ buildvariants: exe: ".exe" additional_package_targets: archive-mongocryptd archive-mongocryptd-debug msi archive-mh archive-mh-debug content_type: application/zip - compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 num_scons_link_jobs_available: 0.25 python: '/cygdrive/c/python/python37/python.exe' ext: zip @@ -946,13 +981,9 @@ buildvariants: # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml test_flags: >- - --additionalFeatureFlagsFile all_feature_flags.txt + --runAllFeatureFlagTests --excludeWithAnyTags=incompatible_with_windows_tls --excludeWithAnyTags=incompatible_with_shard_merge - # Uncomment expansion and `burn_in_tasks_gen` task below and add resmoke task name to burn-in. - # WARNING! Task splitting is not supported for burn-in tasks. Large unsplitted `_gen` tasks may - # run too long and hit execution timeouts. - # burn_in_task_name: jsCore tasks: - name: cqf - name: cqf_parallel @@ -963,7 +994,6 @@ buildvariants: distros: - windows-vsCurrent-xlarge - name: burn_in_tests_gen - # - name: burn_in_tasks_gen - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -983,6 +1013,7 @@ buildvariants: - name: jsCore_ese - name: jsCore_txns_large_txns_format - name: .jstestfuzz .common + - name: mqlrun - name: noPassthrough_gen - name: noPassthroughWithMongod_gen - name: .replica_sets .common !.ignore_non_generated_replica_sets_jscore_passthrough @@ -1004,16 +1035,33 @@ buildvariants: # distros: # - windows-vsCurrent-xlarge +- name: enterprise-windows-ninja + display_name: "Ninja Build: Enterprise Windows" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise + expansions: + compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 + tasks: + - name: compile_ninja_next_TG + distros: + - windows-vsCurrent-large + - name: compile_ninja_TG + distros: + - windows-vsCurrent-large + - name: enterprise-windows-cxx20-debug-experimental display_name: "~ Enterprise Windows C++20 DEBUG" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - windows-vsCurrent-small expansions: exe: ".exe" additional_package_targets: archive-mongocryptd archive-mongocryptd-debug msi archive-mh archive-mh-debug content_type: application/zip - compile_flags: --dbg=on --opt=on --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --cxx-std=20 --use-diagnostic-latches=on + compile_flags: --dbg=on --opt=on --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --cxx-std=20 num_scons_link_jobs_available: 0.25 python: '/cygdrive/c/python/python37/python.exe' ext: zip @@ -1030,9 +1078,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-large - name: .aggfuzzer - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen @@ -1057,18 +1102,21 @@ buildvariants: - name: .sharding .txns !.multiversion - name: .sharding .common !.csrs !.multiversion - name: .ssl + - name: .stitch - name: .updatefuzzer !.multiversion - name: enterprise-windows-debug-unoptimized display_name: Enterprise Windows DEBUG (Unoptimized) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - windows-vsCurrent-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug exe: ".exe" content_type: application/zip - compile_flags: --dbg=on --opt=off --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --use-diagnostic-latches=on + compile_flags: --dbg=on --opt=off --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 num_scons_link_jobs_available: 0.25 python: '/cygdrive/c/python/python37/python.exe' ext: zip @@ -1099,14 +1147,13 @@ buildvariants: run_on: - macos-1100 expansions: &macos-debug-expansions - test_flags: --excludeWithAnyTags=incompatible_with_macos --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=incompatible_with_macos resmoke_jobs_max: 6 compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl --dbg=on --opt=on -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars --modules= --use-diagnostic-latches=on + compile_flags: --ssl --dbg=on --opt=on -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_build_tools_next_TG - name: aggregation - name: auth_gen @@ -1120,27 +1167,44 @@ buildvariants: - name: replica_sets_large_txns_format_gen - name: .ssl - name: .stitch + - name: unittest_shell_hang_analyzer_gen + +- name: macos-enterprise-ninja + display_name: "Ninja Build: macOS Enterprise" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise + run_on: + - macos-1100 + expansions: + compile_env: DEVELOPER_DIR=/Applications/Xcode13.app + compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars + tasks: + - name: compile_ninja_next_TG + - name: compile_ninja_TG - name: enterprise-macos-rosetta-2 display_name: "Enterprise macOS Via Rosetta 2" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - macos-1100-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars --use-diagnostic-latches=on + compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars resmoke_jobs_max: 6 num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl - name: .jscore .common !.decimal !.sharding - name: .logical_session_cache .one_sec + - name: mqlrun # TODO(SERVER-64009): Re-enable replica_sets_auth_gen. # - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough_gen @@ -1150,23 +1214,26 @@ buildvariants: - name: enterprise-macos-cxx20 display_name: "Enterprise macOS C++20 DEBUG" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - macos-1100 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl --dbg=on --opt=on -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars --cxx-std=20 --use-diagnostic-latches=on + compile_flags: --ssl --dbg=on --opt=on -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars --cxx-std=20 resmoke_jobs_max: 6 num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough - - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl + # TODO: SERVER-66945 Re-enable ESE on enterprise macos + # - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl - name: .jscore .common !.decimal !.sharding - name: .logical_session_cache .one_sec + - name: mqlrun - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough_gen - name: sasl @@ -1174,17 +1241,18 @@ buildvariants: - name: enterprise-macos-arm64 display_name: "~ Enterprise macOS arm64" + modules: + - enterprise run_on: - macos-1100-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx_arm.vars --use-diagnostic-latches=on + compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx_arm.vars resmoke_jobs_max: 6 num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -1192,12 +1260,17 @@ buildvariants: - name: .jscore .common !.decimal !.sharding - name: .jstestfuzz .common - name: .logical_session_cache .one_sec + - name: mqlrun - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough - name: sasl - name: push + distros: + - rhel70-small - name: .crypt - name: .publish_crypt + distros: + - rhel70-small ########################################### # Embedded SDK buildvariants # @@ -1209,11 +1282,10 @@ buildvariants: - macos-1100 cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - test_flags: --excludeWithAnyTags=uses_transactions,incompatible_with_macos --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=uses_transactions,incompatible_with_macos cmake_path: /Applications/cmake-3.11.0-Darwin-x86_64/CMake.app/Contents/bin/cmake compile_env: DEVELOPER_DIR=/Applications/Xcode13.app compile_flags: >- - --modules= --lto --variables-files=etc/scons/xcode_macosx.vars -j$(sysctl -n hw.logicalcpu) @@ -1237,10 +1309,9 @@ buildvariants: run_on: - ubuntu1804-build expansions: - test_flags: --excludeWithAnyTags=uses_transactions --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=uses_transactions # We need --allocator=system here to work around SERVER-27675 compile_flags: >- - --modules= --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars -j$(grep -c ^processor /proc/cpuinfo) LIBPATH="\$BUILD_ROOT/mongo-embedded-sdk-\$MONGO_VERSION/lib" @@ -1258,15 +1329,17 @@ buildvariants: # Redhat buildvariants # ########################################### -- &enterprise-rhel-8-64-bit-dynamic-required-template - name: enterprise-rhel-8-64-bit-dynamic-required - display_name: "! Shared Library Enterprise RHEL 8" +- &enterprise-rhel-80-64-bit-dynamic-required-template + name: enterprise-rhel-80-64-bit-dynamic-required + display_name: "! Shared Library Enterprise RHEL 8.0" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel-8-64-bit-dynamic-required-expansions + expansions: &enterprise-rhel-80-64-bit-dynamic-required-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 multiversion_edition: enterprise @@ -1280,6 +1353,7 @@ buildvariants: idle_timeout_factor: 1.5 exec_timeout_factor: 1.5 large_distro_name: rhel80-medium + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_parallel_core_stream_TG @@ -1304,6 +1378,7 @@ buildvariants: - name: .aggregation - name: audit - name: .auth + - name: burn_in_tags_gen - name: buildscripts_test - name: resmoke_end2end_tests - name: unittest_shell_hang_analyzer_gen @@ -1343,14 +1418,11 @@ buildvariants: - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - name: .multiversion_sanity_check + - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -1376,7 +1448,6 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt @@ -1385,6 +1456,7 @@ buildvariants: - name: .sharding .common - name: sharding_opportunistic_secondary_targeting_gen - name: snmp + - name: .stitch - name: .crypt distros: - rhel80-xlarge @@ -1400,21 +1472,37 @@ buildvariants: - name: server_selection_json_test_TG distros: - rhel80-xlarge - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - - name: generate_buildid_to_debug_symbols_mapping -- &enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required-template - name: enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required - display_name: "! Shared Library Enterprise RHEL 8 (all feature flags)" +- name: enterprise-rhel-80-64-bit-dynamic-required-ninja + display_name: "Ninja Build: Enterprise RHEL 8.0" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise + run_on: + - rhel80-small + expansions: + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic + has_packages: false + tasks: + - name: compile_ninja_next_TG + distros: + - rhel80-xlarge + - name: compile_ninja_TG + distros: + - rhel80-xlarge + +- &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-template + name: enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required + display_name: "! Shared Library Enterprise RHEL 8.0 (all feature flags)" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter stepback: false + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel-8-64-bit-dynamic-all-feature-flags-expansions + expansions: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 multiversion_edition: enterprise @@ -1430,13 +1518,9 @@ buildvariants: # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml test_flags: >- - --additionalFeatureFlagsFile all_feature_flags.txt + --runAllFeatureFlagTests --excludeWithAnyTags=incompatible_with_shard_merge - # Uncomment expansion and `burn_in_tasks_gen` task below and add resmoke task name to burn-in. - # WARNING! Task splitting is not supported for burn-in tasks. Large unsplitted `_gen` tasks may - # run too long and hit execution timeouts. - # burn_in_task_name: jsCore - tasks: &enterprise-rhel-8-64-bit-dynamic-all-feature-flags-tasks + tasks: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-tasks - name: cqf - name: cqf_parallel - name: compile_test_and_package_parallel_core_stream_TG @@ -1453,7 +1537,6 @@ buildvariants: distros: - rhel80-large - name: burn_in_tests_gen - # - name: burn_in_tasks_gen - name: .aggfuzzer - name: .aggregation - name: audit @@ -1483,15 +1566,12 @@ buildvariants: - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests + - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: powercycle_smoke - name: .query_fuzzer - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -1517,7 +1597,6 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt @@ -1533,10 +1612,6 @@ buildvariants: - name: .serverless - name: .updatefuzzer - name: secondary_reads_passthrough_gen - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough # Disabling as the following tests are not aware of feature flags. # - name: server_discovery_and_monitoring_json_test_TG # distros: @@ -1544,21 +1619,19 @@ buildvariants: # - name: server_selection_json_test_TG # distros: # - rhel80-large - - name: generate_buildid_to_debug_symbols_mapping - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl -- &enterprise-rhel-8-64-bit-dynamic-sbe - name: enterprise-rhel-8-64-bit-dynamic-sbe - display_name: "Shared Library Enterprise RHEL 8 (SBE)" +- &enterprise-rhel-80-64-bit-dynamic-classic-engine + name: enterprise-rhel-80-64-bit-dynamic-classic-engine + display_name: "Shared Library Enterprise RHEL 8.0 (Classic Engine)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. stepback: false + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel-8-64-bit-dynamic-sbe-expansions + expansions: &enterprise-rhel-80-64-bit-dynamic-classic-engine-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise has_packages: false @@ -1569,9 +1642,10 @@ buildvariants: target_resmoke_time: 10 max_sub_suites: 5 large_distro_name: rhel80-medium + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion num_scons_link_jobs_available: 0.99 test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: false}" + --mongodSetParameters="{internalQueryForceClassicEngine: true}" tasks: - name: .aggfuzzer - name: .aggregation @@ -1606,11 +1680,13 @@ buildvariants: - name: .sharding .common - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .txns + - name: .stitch - name: .serverless distros: - rhel80-xlarge - name: .updatefuzzer - name: audit + - name: burn_in_tags_gen - name: burn_in_tests_gen - name: check_feature_flag_tags - name: check_for_todos @@ -1635,13 +1711,10 @@ buildvariants: - name: jsCore_txns_large_txns_format - name: json_schema - name: lint_fuzzer_sanity_patch + - name: mqlrun - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .multiversion_sanity_check - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: replica_sets_api_version_jscore_passthrough_gen - name: replica_sets_reconfig_jscore_passthrough_gen - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen @@ -1655,7 +1728,6 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: secondary_reads_passthrough_gen - name: server_discovery_and_monitoring_json_test_TG @@ -1667,12 +1739,11 @@ buildvariants: - name: snmp - name: test_api_version_compatibility - name: unittest_shell_hang_analyzer_gen - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl -- name: enterprise-rhel-8-64-bit-large-txns-format - display_name: "Enterprise RHEL 8 (large transactions format)" +- name: enterprise-rhel-80-64-bit-large-txns-format + display_name: "Enterprise RHEL 8.0 (large transactions format)" + modules: + - enterprise run_on: - rhel80-small cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -1684,7 +1755,6 @@ buildvariants: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic - --use-diagnostic-latches=on multiversion_platform: rhel80 multiversion_edition: enterprise repo_edition: enterprise @@ -1752,20 +1822,21 @@ buildvariants: # This build variant is used to run multiversion tests as part of burn_in_tags as these tests are # currently only run on our daily builders. -- &enterprise-rhel-8-64-bit-multiversion-template - name: enterprise-rhel-8-64-bit-multiversion - display_name: "Enterprise RHEL 8 (implicit multiversion)" +- &enterprise-rhel-80-64-bit-multiversion-template + name: enterprise-rhel-80-64-bit-multiversion + display_name: "Enterprise RHEL 8.0 (implicit multiversion)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel-8-64-bit-multiversion-expansions-template + expansions: &enterprise-rhel-80-64-bit-multiversion-expansions-template compile_flags: >- -j$(grep -c ^processor /proc/cpuinfo) --ssl --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars MONGO_DISTMOD=rhel80 --link-model=dynamic - --use-diagnostic-latches=on multiversion_platform: rhel80 multiversion_edition: enterprise repo_edition: enterprise @@ -1790,15 +1861,15 @@ buildvariants: - name: .multiversion_passthrough - name: .random_multiversion_ds -- <<: *enterprise-rhel-8-64-bit-multiversion-template - name: enterprise-rhel-8-64-bit-multiversion-all-feature-flags - display_name: "Enterprise RHEL 8 (implicit multiversion & all feature flags)" +- <<: *enterprise-rhel-80-64-bit-multiversion-template + name: enterprise-rhel-80-64-bit-multiversion-all-feature-flags + display_name: "Enterprise RHEL 8.0 (implicit multiversion & all feature flags)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. expansions: - <<: *enterprise-rhel-8-64-bit-multiversion-expansions-template + <<: *enterprise-rhel-80-64-bit-multiversion-expansions-template # No feature flag tests since they aren't compatible with the older binaries. test_flags: >- - --runNoFeatureFlagTests + --runAllFeatureFlagsNoTests --excludeWithAnyTags=incompatible_with_shard_merge tasks: - name: compile_test_and_package_parallel_core_stream_TG @@ -1815,9 +1886,11 @@ buildvariants: - name: .multiversion_passthrough - name: .random_multiversion_ds -- name: enterprise-rhel-8-64-bit-future-git-tag-multiversion - display_name: "Enterprise RHEL 8 (future git tag multiversion)" +- name: enterprise-rhel-80-64-bit-future-git-tag-multiversion + display_name: "Enterprise RHEL 8.0 (future git tag multiversion)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-small expansions: @@ -1827,7 +1900,6 @@ buildvariants: --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars MONGO_DISTMOD=rhel80 --link-model=dynamic - --use-diagnostic-latches=on multiversion_platform: rhel80 multiversion_edition: enterprise repo_edition: enterprise @@ -1842,9 +1914,6 @@ buildvariants: test_flags: >- --excludeWithAnyTags=future_git_tag_incompatible tasks: - - name: version_expansions_gen - distros: - - rhel80-small - name: compile_test_and_package_parallel_core_stream_TG distros: - rhel80-xlarge @@ -1857,8 +1926,8 @@ buildvariants: - name: .multiversion !.future_git_tag_incompatible - name: .multiversion_future_git_tag -#- name: rhel-8-64-bit-nossl -# display_name: "RHEL 8 Shared Library (No SSL)" +#- name: rhel-80-64-bit-nossl +# display_name: "RHEL 8.0 Shared Library (No SSL)" # run_on: # - rhel80-small # cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. @@ -1888,15 +1957,17 @@ buildvariants: # - rhel80-large # - name: jsCore -- &enterprise-rhel-8-64-bit-suggested-template - name: enterprise-rhel-8-64-bit-suggested - display_name: "* Enterprise RHEL 8" +- &enterprise-rhel-80-64-bit-suggested-template + name: enterprise-rhel-80-64-bit-suggested + display_name: "* Enterprise RHEL 8.0" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter + modules: + - enterprise run_on: - rhel80-build expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel80 multiversion_edition: enterprise has_packages: true @@ -1910,9 +1981,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-xlarge - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel80-xlarge - name: .aggfuzzer - name: audit - name: auth_audit_gen @@ -1932,14 +2000,14 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: sharding_auth_audit_gen - name: sharding_auth_gen - name: snmp - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl + - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: selinux_rhel8_enterprise - name: generate_buildid_to_debug_symbols_mapping @@ -1967,7 +2035,7 @@ buildvariants: expansions: <<: *enterprise-rhel-70-64-bit-expansions-template additional_package_targets: "" - compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enterprise-features=hot_backups --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enterprise-features=hot_backups --link-model=dynamic has_packages: false scons_cache_mode: all scons_cache_scope: shared @@ -1987,7 +2055,7 @@ buildvariants: - rhel70 expansions: <<: *enterprise-rhel-70-64-bit-expansions-template - compile_flags: --ssl MONGO_DISTMOD=rhel70 --use-libunwind=off --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel70 --use-libunwind=off --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic has_packages: false additional_package_targets: "" scons_cache_mode: all @@ -2022,14 +2090,11 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: sharding_auth_audit_gen - name: sharding_auth_gen - name: snmp - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl + - name: .stitch - name: ubuntu1804-container display_name: "Ubuntu 18.04 Container" @@ -2039,12 +2104,11 @@ buildvariants: expansions: resmoke_jobs_factor: 1 disable_shared_scons_cache: true - compile_flags: MONGO_DISTMOD=ubuntu1804 --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + compile_flags: MONGO_DISTMOD=ubuntu1804 --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars tooltags: "" build_mongoreplay: true test_flags: >- --excludeWithAnyTags=requires_os_access - --enableEnterpriseTests=off tasks: - name: compile_and_archive_dist_test_then_package_TG distros: @@ -2055,6 +2119,8 @@ buildvariants: - name: enterprise-rhel-72-s390x-compile display_name: Enterprise RHEL 7.2 s390x Compile + modules: + - enterprise run_on: - rhel72-zseries-test cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -2076,27 +2142,24 @@ buildvariants: multiversion_architecture: s390x multiversion_edition: enterprise num_scons_link_jobs_available: 0.99 - test_flags: >- - --excludeWithAnyTags=incompatible_with_s390x,requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - rhel72-zseries-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel72-zseries-build - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel8-dynamic-clang-tidy-required +- name: enterprise-rhel80-dynamic-clang-tidy-required display_name: "! Enterprise Clang Tidy" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter + modules: + - enterprise run_on: - rhel80-xlarge expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug lang_environment: LANG=C - compile_flags: --link-model=dynamic -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --use-diagnostic-latches=on + compile_flags: --link-model=dynamic -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars # Unlike static builds, dynamic builds have no need to # constrain the number of link jobs. Unfortunately, --jlink=1 # means one link job, not 100%. So this is a bit gross but set @@ -2109,20 +2172,18 @@ buildvariants: - name: clang_tidy_TG -- name: rhel8-debug-suggested - display_name: "* Shared Library RHEL 8 DEBUG" +- name: rhel80-debug-suggested + display_name: "* Shared Library RHEL 8.0 DEBUG" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - rhel80-small expansions: resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --modules= --use-diagnostic-latches=on + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 large_distro_name: rhel80-large - test_flags: >- - --excludeWithAnyTags=incompatible_with_s390x tasks: - name: compile_test_and_package_parallel_core_stream_TG distros: @@ -2140,20 +2201,21 @@ buildvariants: - name: .read_write_concern !.write !.aggregation - name: replica_sets_jscore_passthrough - name: replica_sets_large_txns_format_jscore_passthrough - - name: sharded_collections_jscore_passthrough_gen + - name: sharded_collections_jscore_passthrough - name: sharding_gen - name: sharding_auth_gen - name: .stitch - name: unittest_shell_hang_analyzer_gen - - name: generate_buildid_to_debug_symbols_mapping ################################ # storage engine buildvariants # ################################ -- name: enterprise-rhel-8-benchmarks - display_name: Enterprise RHEL 8 (Benchmarks) +- name: enterprise-rhel-80-benchmarks + display_name: Enterprise RHEL 8.0 (Benchmarks) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-medium expansions: @@ -2177,8 +2239,7 @@ buildvariants: --storageEngine=ephemeralForTest --excludeWithAnyTags=requires_persistence,requires_fsync,requires_journaling,requires_wiredtiger,uses_transactions,uses_speculative_majority,requires_snapshot_read,requires_majority_read_concern,uses_change_streams,requires_sharding,incompatible_with_eft --mongodSetParameters="{oplogApplicationEnforcesSteadyStateConstraints: false}" - --enableEnterpriseTests=off - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --modules= --use-diagnostic-latches=on + compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: targeted scons_cache_mode: all @@ -2215,13 +2276,16 @@ buildvariants: # rollback_fuzzer_gen - name: .updatefuzzer + ########################################### # Experimental buildvariants # ########################################### -- name: rhel8-debug-asan-all-feature-flags - display_name: "~ Shared Library ASAN Enterprise RHEL 8 DEBUG (all feature flags)" +- name: rhel80-debug-asan-all-feature-flags + display_name: "~ Shared Library ASAN Enterprise RHEL 8.0 DEBUG (all feature flags)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -2231,7 +2295,7 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. @@ -2242,15 +2306,13 @@ buildvariants: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml large_distro_name: rhel80-build test_flags: >- - --additionalFeatureFlagsFile all_feature_flags.txt + --runAllFeatureFlagTests --excludeWithAnyTags=incompatible_with_shard_merge - --enableEnterpriseTests=off separate_debug: off tasks: - name: cqf - name: cqf_parallel - name: compile_and_archive_dist_test_then_package_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_benchmarks - name: .aggregation # - name: .auth @@ -2260,6 +2322,7 @@ buildvariants: - name: .misc_js # - name: .concurrency !.ubsan !.no_txns !.kill_terminate # - name: .encrypt + - name: free_monitoring - name: external_auth - name: external_auth_aws - name: initial_sync_fuzzer_gen @@ -2287,19 +2350,18 @@ buildvariants: - name: snmp - name: .serverless - name: .watchdog - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough + - name: .stitch # Disabling the following tests as they are not aware of feature flags. # - name: .benchmarks # - name: unittest_shell_hang_analyzer_gen # - name: server_discovery_and_monitoring_json_test_TG # - name: server_selection_json_test_TG -- name: rhel8-debug-asan-sbe - display_name: ~ ASAN Enterprise RHEL 8 DEBUG (SBE) +- name: rhel80-debug-asan-classic-engine + display_name: ~ ASAN Enterprise RHEL 8.0 DEBUG (Classic Engine) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -2309,11 +2371,10 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: false}" + --mongodSetParameters="{internalQueryForceClassicEngine: true}" --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling - --enableEnterpriseTests=off multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. @@ -2324,7 +2385,6 @@ buildvariants: large_distro_name: rhel80-build tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_benchmarks - name: .aggregation - name: .auth @@ -2335,6 +2395,7 @@ buildvariants: - name: .misc_js - name: .concurrency !.ubsan !.no_txns !.kill_terminate - name: .encrypt + - name: free_monitoring - name: external_auth - name: external_auth_aws - name: initial_sync_fuzzer_gen @@ -2362,15 +2423,18 @@ buildvariants: - name: .sharding .common !.csrs - name: snmp - name: .watchdog + - name: .stitch - name: .serverless - name: unittest_shell_hang_analyzer_gen - name: .updatefuzzer - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG -- name: rhel8-debug-ubsan-all-feature-flags - display_name: "~ Shared Library UBSAN Enterprise RHEL 8 DEBUG (all feature flags)" +- name: rhel80-debug-ubsan-all-feature-flags + display_name: "~ Shared Library UBSAN Enterprise RHEL 8.0 DEBUG (all feature flags)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -2380,14 +2444,13 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml test_flags: >- --excludeWithAnyTags=requires_ocsp_stapling --excludeWithAnyTags=incompatible_with_shard_merge - --additionalFeatureFlagsFile all_feature_flags.txt - --enableEnterpriseTests=off + --runAllFeatureFlagTests multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. @@ -2408,6 +2471,7 @@ buildvariants: - name: .concurrency !.no_txns !.repl !.kill_terminate - name: disk_wiredtiger - name: .encrypt + - name: free_monitoring - name: initial_sync_fuzzer_gen - name: .integration !.audit - name: .jscore .common @@ -2430,18 +2494,20 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot - name: .sharding .common !.csrs - name: snmp + - name: .stitch - name: .serverless - name: .updatefuzzer - name: watchdog_wiredtiger - - name: query_stats_passthrough # Disabling the following tests as they are not aware of feature flags. # - name: .benchmarks # - name: server_discovery_and_monitoring_json_test_TG # - name: server_selection_json_test_TG -- name: rhel8-debug-ubsan-sbe - display_name: "~ UBSAN Enterprise RHEL 8 DEBUG (SBE)" +- name: rhel80-debug-ubsan-classic-engine + display_name: "~ UBSAN Enterprise RHEL 8.0 DEBUG (Classic Engine)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -2451,11 +2517,10 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: false}" - --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits - --enableEnterpriseTests=off + --mongodSetParameters="{internalQueryForceClassicEngine: true}" + --excludeWithAnyTags=requires_ocsp_stapling multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. @@ -2464,7 +2529,6 @@ buildvariants: large_distro_name: rhel80-build tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_benchmarks - name: .aggregation - name: .auth @@ -2476,6 +2540,7 @@ buildvariants: - name: .concurrency !.no_txns !.repl !.kill_terminate - name: disk_wiredtiger - name: .encrypt + - name: free_monitoring - name: initial_sync_fuzzer_gen - name: .integration !.audit - name: .jscore .common @@ -2498,16 +2563,19 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot - name: .sharding .common !.csrs - name: snmp + - name: .stitch - name: .updatefuzzer - name: .serverless - name: watchdog_wiredtiger - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG -- &rhel8-debug-aubsan-lite-required-template - name: rhel8-debug-aubsan-lite-required - display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8 DEBUG" +- &rhel80-debug-aubsan-lite-required-template + name: rhel80-debug-aubsan-lite-required + display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter + modules: + - enterprise run_on: - rhel80-build stepback: true @@ -2517,8 +2585,8 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --modules= --use-diagnostic-latches=on - test_flags: --excludeWithAnyTags=requires_ocsp_stapling --enableEnterpriseTests=off + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic + test_flags: --excludeWithAnyTags=requires_ocsp_stapling resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false scons_cache_scope: shared @@ -2540,11 +2608,10 @@ buildvariants: - name: jsCore - name: jsCore_txns - name: unittest_shell_hang_analyzer_gen - - name: generate_buildid_to_debug_symbols_mapping -- <<: *rhel8-debug-aubsan-lite-required-template - name: rhel8-debug-aubsan-lite-all-feature-flags-required - display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8 DEBUG (all feature flags)" +- <<: *rhel80-debug-aubsan-lite-required-template + name: rhel80-debug-aubsan-lite-all-feature-flags-required + display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG (all feature flags)" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug @@ -2552,14 +2619,20 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --modules= --use-diagnostic-latches=on + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml test_flags: >- --excludeWithAnyTags=requires_ocsp_stapling --excludeWithAnyTags=incompatible_with_shard_merge - --additionalFeatureFlagsFile all_feature_flags.txt - --enableEnterpriseTests=off + --runAllFeatureFlagTests + resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. + hang_analyzer_dump_core: false + scons_cache_scope: shared + scons_cache_mode: all + max_sub_suites: 3 + num_scons_link_jobs_available: 0.99 + separate_debug: off tasks: - name: compile_test_and_package_parallel_core_stream_TG distros: @@ -2574,11 +2647,12 @@ buildvariants: # distros: # - rhel80-xlarge # - name: unittest_shell_hang_analyzer_gen - - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel8-debug-tsan - display_name: ~ TSAN Enterprise RHEL 8 DEBUG +- name: enterprise-rhel80-debug-tsan + display_name: ~ TSAN Enterprise RHEL 8.0 DEBUG cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-large stepback: false @@ -2597,22 +2671,43 @@ buildvariants: # of the ServiceContext. # san_options: TSAN_OPTIONS="halt_on_error=1:report_thread_leaks=0:die_after_fork=0:suppressions=etc/tsan.suppressions:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=thread --ssl --use-libunwind=off -j$(grep -c ^processor /proc/cpuinfo) + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=thread --ssl --enable-free-mon=on --use-libunwind=off -j$(grep -c ^processor /proc/cpuinfo) # Avoid starting too many mongod's under TSAN build. resmoke_jobs_factor: 0.3 scons_cache_scope: shared separate_debug: off - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: jsCore -- name: enterprise-rhel8-unoptimized-64-bit - display_name: "~ Enterprise RHEL 8 DEBUG (Unoptimized)" +- name: enterprise-rhel80-unoptimized-64-bit + display_name: "~ Enterprise RHEL 8.0 (without Diagnostic Latches)" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise + expansions: + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=off --link-model=dynamic + scons_cache_mode: all + scons_cache_scope: shared + num_scons_link_jobs_available: 0.99 + tasks: + - name: compile_test_and_package_parallel_core_stream_TG + distros: + - rhel80-build + - name: compile_test_and_package_parallel_unittest_stream_TG + distros: + - rhel80-build + - name: compile_test_and_package_parallel_dbtest_stream_TG + distros: + - rhel80-build + +- name: enterprise-rhel80-no-latch-64-bit + display_name: "~ Enterprise RHEL 8.0 DEBUG (Unoptimized)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise expansions: - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --dbg=on --opt=off --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --detect-odr-violations --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --dbg=on --opt=off --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --detect-odr-violations scons_cache_mode: all scons_cache_scope: shared num_scons_link_jobs_available: 0.99 @@ -2627,16 +2722,18 @@ buildvariants: distros: - rhel80-build -- name: enterprise-rhel8-fixed-service-executor-64-bit - display_name: "~ Enterprise RHEL 8 (with FixedServiceExecutor)" +- name: enterprise-rhel80-fixed-service-executor-64-bit + display_name: "~ Enterprise RHEL 8.0 (with FixedServiceExecutor)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: >- @@ -2665,16 +2762,18 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel8-sdam-replica-set-monitor-64-bit - display_name: "~ Enterprise RHEL 8 (with SdamReplicaSetMonitor)" +- name: enterprise-rhel80-sdam-replica-set-monitor-64-bit + display_name: "~ Enterprise RHEL 8.0 (with SdamReplicaSetMonitor)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: >- @@ -2702,16 +2801,18 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel8-task-executor-pool-size-64-bit - display_name: "~ Enterprise RHEL 8 (with {taskExecutorPoolSize: 4})" +- name: enterprise-rhel80-task-executor-pool-size-64-bit + display_name: "~ Enterprise RHEL 8.0 (with {taskExecutorPoolSize: 4})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: |- # Set the taskExecutorPoolSize for all tests @@ -2737,16 +2838,18 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel8-sharding-task-executor-pool-rsm-matchPrimary-64-bit - display_name: "~ Enterprise RHEL 8 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchPrimaryNode\"})" +- name: enterprise-rhel80-sharding-task-executor-pool-rsm-matchPrimary-64-bit + display_name: "~ Enterprise RHEL 8.0 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchPrimaryNode\"})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: >- @@ -2773,16 +2876,18 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel8-sharding-task-executor-pool-rsm-matchBusiest-64-bit - display_name: "~ Enterprise RHEL 8 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchBusiestNode\"})" +- name: enterprise-rhel80-sharding-task-executor-pool-rsm-matchBusiest-64-bit + display_name: "~ Enterprise RHEL 8.0 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchBusiestNode\"})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: >- @@ -2809,16 +2914,18 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel8-sharding-task-executor-pool-rsm-disabled-64-bit - display_name: "~ Enterprise RHEL 8 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"disabled\"})" +- name: enterprise-rhel80-sharding-task-executor-pool-rsm-disabled-64-bit + display_name: "~ Enterprise RHEL 8.0 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"disabled\"})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: >- @@ -2845,14 +2952,16 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel8-join-ingress-sessions-on-shutdown - display_name: "~ Enterprise RHEL 8 (with {joinIngressSessionsOnShutdown: \"true\"})" +- name: enterprise-rhel80-join-ingress-sessions-on-shutdown + display_name: "~ Enterprise RHEL 8.0 (with {joinIngressSessionsOnShutdown: \"true\"})" activate: false run_on: - rhel80-small + modules: + - enterprise expansions: scons_cache_scope: shared - compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on + compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel80 multiversion_edition: enterprise test_flags: >- @@ -2863,9 +2972,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel80-build - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -2898,74 +3004,100 @@ buildvariants: - name: selected-tests display_name: "~ Selected Tests" + modules: + - enterprise run_on: - rhel80-small cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter stepback: false expansions: - selected_tests_buildvariants: enterprise-windows-required linux-64-debug-required enterprise-ubuntu-dynamic-1604-clang rhel8-debug-aubsan-lite-required + selected_tests_buildvariants: enterprise-windows-required linux-64-debug-required enterprise-ubuntu-dynamic-1604-clang rhel80-debug-aubsan-lite-required tasks: - name: selected_tests_gen +- <<: *enterprise-rhel-80-64-bit-dynamic-required-template + name: commit-queue + display_name: "~ Commit Queue" + cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter + stepback: false + tasks: + - name: compile_test_and_package_parallel_core_stream_TG + distros: + - rhel80-xlarge-commitqueue + - name: compile_test_and_package_parallel_unittest_stream_TG + distros: + - rhel80-xlarge-commitqueue + - name: compile_test_and_package_parallel_dbtest_stream_TG + distros: + - rhel80-xlarge-commitqueue + - name: jsCore + distros: + - rhel80-xlarge-commitqueue + - name: .lint + - name: test_api_version_compatibility + - name: validate_commit_message + - name: check_feature_flag_tags + - name: windows-dynamic-visibility-test display_name: "~ Shared Library Windows (visibility test)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. expansions: # --link-model=dynamic is set by visiblity_test's task_compile_flags # expansion - compile_flags: CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 --modules= --use-diagnostic-latches=on + compile_flags: CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 python: '/cygdrive/c/python/python37/python.exe' ext: zip has_packages: false scons_cache_scope: shared scons_cache_mode: all num_scons_link_jobs_available: 0.99 - test_flags: --enableEnterpriseTests=off tasks: - name: visibility_test_TG distros: - windows-vsCurrent-large ### QO & QE Patch-Specific Build Variants ### -- <<: *enterprise-rhel-8-64-bit-dynamic-sbe - name: enterprise-rhel-8-64-bit-dynamic-sbe-query-patch-only - display_name: "~ Shared Library Enterprise RHEL 8 Query Patch Only (SBE)" +- <<: *enterprise-rhel-80-64-bit-dynamic-classic-engine + name: enterprise-rhel-80-64-bit-dynamic-classic-engine-query-patch-only + display_name: "~ Shared Library Enterprise RHEL 8.0 Query Patch Only (Classic Engine)" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter # This is a patch-only variant but we run on mainline to pick up task history. expansions: - <<: *enterprise-rhel-8-64-bit-dynamic-sbe-expansions + <<: *enterprise-rhel-80-64-bit-dynamic-classic-engine-expansions jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 target_resmoke_time: 30 max_sub_suites: 3 test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: false}" + --mongodSetParameters="{internalQueryForceClassicEngine: true}" --excludeWithAnyTags=resource_intensive -- <<: *enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required-template - name: enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required-query-patch-only - display_name: "~ Shared Library Enterprise RHEL 8 Query Patch Only (all feature flags)" +- <<: *enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-template + name: enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-query-patch-only + display_name: "~ Shared Library Enterprise RHEL 8.0 Query Patch Only (all feature flags)" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter # This is a patch-only variant but we run on mainline to pick up task history. expansions: - <<: *enterprise-rhel-8-64-bit-dynamic-all-feature-flags-expansions + <<: *enterprise-rhel-80-64-bit-dynamic-all-feature-flags-expansions jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 target_resmoke_time: 30 max_sub_suites: 3 test_flags: >- - --additionalFeatureFlagsFile all_feature_flags.txt + --runAllFeatureFlagTests --excludeWithAnyTags=resource_intensive --excludeWithAnyTags=incompatible_with_shard_merge - name: enterprise-ubuntu1804-64-libvoidstar display_name: ~ Enterprise Ubuntu 18.04 w/ libvoidstar cron: "0 4 * * FRI" # Every week at 0400 UTC Friday. This has to be a Friday since we run Antithesis on Fridays. + modules: + - enterprise run_on: - ubuntu1804-large stepback: false expansions: # Do not set link-model=dynamic on this build. The antithesis image builder # scripts cannot handle the shared library build - compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars CCFLAGS="-fsanitize-coverage=trace-pc-guard" LIBS="voidstar" --use-diagnostic-latches=on + compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars CCFLAGS="-fsanitize-coverage=trace-pc-guard" LIBS="voidstar" multiversion_platform: ubuntu1804 multiversion_edition: enterprise repo_edition: enterprise @@ -2992,6 +3124,7 @@ buildvariants: display_name: "~ Enterprise Windows WiredTiger develop" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter modules: + - enterprise - wtdevelop expansions: <<: *enterprise-windows-nopush-expansions-template @@ -3007,9 +3140,11 @@ buildvariants: distros: - windows-vsCurrent-large -- name: rhel8-debug-aubsan-lite_fuzzer - display_name: "{A,UB}SAN Enterprise RHEL 8 FUZZER" +- name: rhel80-debug-aubsan-lite_fuzzer + display_name: "{A,UB}SAN Enterprise RHEL 8.0 FUZZER" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -3019,8 +3154,8 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: LINKFLAGS=-nostdlib++ LIBS=stdc++ --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address,fuzzer --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on - test_flags: --excludeWithAnyTags=requires_ocsp_stapling --enableEnterpriseTests=off + compile_flags: LINKFLAGS=-nostdlib++ LIBS=stdc++ --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address,fuzzer --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) + test_flags: --excludeWithAnyTags=requires_ocsp_stapling resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false scons_cache_scope: shared diff --git a/etc/evergreen_nightly.yml b/etc/evergreen_nightly.yml index 7688719bd4c..fc3b3404a69 100644 --- a/etc/evergreen_nightly.yml +++ b/etc/evergreen_nightly.yml @@ -11,8 +11,6 @@ include: - filename: etc/evergreen_yml_components/variants/in_memory.yml ### Uncomment when using this file for a LTS or Rapid release branch. ### - filename: etc/evergreen_yml_components/variants/sanitizer.yml -### Uncomment when using this file for a LTS or Rapid release branch. ### -- filename: etc/evergreen_yml_components/variants/ninja.yml parameters: @@ -29,10 +27,10 @@ buildvariants: - rhel80-medium expansions: resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's - compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-http-client=on --link-model=dynamic --modules= + compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-free-mon=on --enable-http-client=on --link-model=dynamic scons_cache_scope: shared scons_cache_mode: all - test_flags: --excludeWithAnyTags=requires_http_client,requires_latch_analyzer --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=requires_http_client target_resmoke_time: 15 max_sub_suites: 5 num_scons_link_jobs_available: 0.99 @@ -47,9 +45,6 @@ buildvariants: - name: compile_test_and_package_parallel_dbtest_stream_TG distros: - rhel80-xlarge - - name: compile_integration_and_test_no_audit_parallel_stream_TG - distros: - - rhel80-xlarge - name: .aggregation !.encrypt - name: .auth !.audit !.multiversion - name: .batched_deletes @@ -58,6 +53,8 @@ buildvariants: - name: .clustered_collections - name: .misc_js - name: disk_wiredtiger + - name: free_monitoring + - name: .integration !.audit - name: .jscore .common - name: jsCore_txns_large_txns_format - name: json_schema @@ -89,6 +86,8 @@ buildvariants: name: enterprise-windows-required display_name: "! Enterprise Windows" cron: "0 */4 * * *" # Every 4 hours starting at midnight + modules: + - enterprise run_on: - windows-vsCurrent-small expansions: @@ -111,14 +110,11 @@ buildvariants: push_bucket: downloads.10gen.com push_name: windows push_arch: x86_64-enterprise - test_flags: --excludeWithAnyTags=incompatible_with_windows_tls,requires_latch_analyzer + test_flags: --excludeWithAnyTags=incompatible_with_windows_tls tasks: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-xlarge - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-xlarge - name: compile_build_tools_next_TG distros: - windows-vsCurrent-xlarge @@ -143,16 +139,13 @@ buildvariants: - name: jsCore_ese - name: jsCore_txns_large_txns_format - name: .jstestfuzz .common + - name: mqlrun - name: noPassthrough_gen - name: noPassthroughWithMongod_gen - - name: .replica_sets .common !.ignore_non_generated_replica_sets_jscore_passthrough - - name: .replica_sets .multi_oplog !.ignore_non_generated_replica_sets_jscore_passthrough - - name: replica_sets_jscore_passthrough - distros: - - windows-vsCurrent-large + - name: .replica_sets .common + - name: .replica_sets .multi_oplog - name: replica_sets_ese_gen - name: sasl - - name: sasl_windows_cyrussasl - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG - name: .sharding .txns @@ -162,13 +155,15 @@ buildvariants: - name: snmp - name: unittest_shell_hang_analyzer_gen -- &enterprise-rhel-8-64-bit-dynamic-required-template - name: enterprise-rhel-8-64-bit-dynamic-required - display_name: "! Shared Library Enterprise RHEL 8" +- &enterprise-rhel-80-64-bit-dynamic-required-template + name: enterprise-rhel-80-64-bit-dynamic-required + display_name: "! Shared Library Enterprise RHEL 8.0" cron: "0 */4 * * *" # Every 4 hours starting at midnight + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel-8-64-bit-dynamic-required-expansions + expansions: &enterprise-rhel-80-64-bit-dynamic-required-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" @@ -184,8 +179,8 @@ buildvariants: idle_timeout_factor: 1.5 exec_timeout_factor: 1.5 large_distro_name: rhel80-medium + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion num_scons_link_jobs_available: 0.99 - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_parallel_core_stream_TG distros: @@ -199,9 +194,6 @@ buildvariants: - name: compile_test_and_package_parallel_dbtest_stream_TG distros: - rhel80-xlarge - - name: compile_integration_and_test_parallel_stream_TG - distros: - - rhel80-xlarge - name: .lint - name: lint_fuzzer_sanity_patch - name: test_api_version_compatibility @@ -212,6 +204,7 @@ buildvariants: - name: .aggregation - name: audit - name: .auth + - name: burn_in_tags_gen - name: buildscripts_test - name: resmoke_end2end_tests - name: unittest_shell_hang_analyzer_gen @@ -238,6 +231,9 @@ buildvariants: - name: .encrypt - name: idl_tests - name: initial_sync_fuzzer_gen + - name: .integration + distros: + - rhel80-medium - name: jsCore distros: - rhel80-xlarge @@ -248,14 +244,11 @@ buildvariants: - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - name: .multiversion_sanity_check + - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -281,7 +274,6 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt @@ -289,6 +281,7 @@ buildvariants: - name: .sharding .txns - name: .sharding .common - name: snmp + - name: .stitch - name: .crypt distros: - rhel80-xlarge @@ -304,13 +297,12 @@ buildvariants: - name: server_selection_json_test_TG distros: - rhel80-xlarge - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - name: enterprise-ubuntu-dynamic-1804-clang-tidy-required display_name: "! Enterprise Clang Tidy" cron: "0 */4 * * *" # Every 4 hours starting at midnight + modules: + - enterprise run_on: - ubuntu1804-xlarge expansions: @@ -332,6 +324,8 @@ buildvariants: name: ubuntu1804-debug-aubsan-lite-required display_name: "! Shared Library {A,UB}SAN Enterprise Ubuntu 18.04 DEBUG" cron: "0 */4 * * *" # Every 4 hours starting at midnight + modules: + - enterprise run_on: - ubuntu1804-build stepback: true @@ -342,7 +336,7 @@ buildvariants: # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined,address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic - test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_latch_analyzer + test_flags: --excludeWithAnyTags=requires_ocsp_stapling resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false scons_cache_scope: shared @@ -364,36 +358,13 @@ buildvariants: - name: jsCore_txns - name: unittest_shell_hang_analyzer_gen -- <<: *enterprise-rhel-8-64-bit-dynamic-required-template - name: commit-queue - display_name: "~ Commit Queue" - cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter - stepback: false - tasks: - - name: compile_test_and_package_parallel_core_stream_TG - distros: - - rhel80-xlarge-commitqueue - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel80-xlarge-commitqueue - - name: compile_test_and_package_parallel_dbtest_stream_TG - distros: - - rhel80-xlarge-commitqueue - - name: jsCore - distros: - - rhel80-xlarge-commitqueue - - name: .lint - - name: test_api_version_compatibility - - name: validate_commit_message - - name: check_feature_flag_tags - # Temporary variant to stress a component and gather more data. -- <<: *enterprise-rhel-8-64-bit-dynamic-required-template - name: enterprise-rhel-8-64-bit-dynamic-stress-initialsyncid # For BF-25011 - display_name: "~ Shared Library Enterprise RHEL 8 (initialSyncId stress test)" +- <<: *enterprise-rhel-80-64-bit-dynamic-required-template + name: enterprise-rhel-80-64-bit-dynamic-stress-initialsyncid # For BF-25011 + display_name: "~ Shared Library Enterprise RHEL 8.0 (initialSyncId stress test)" cron: "0 */1 * * *" # Every hour starting at midnight expansions: - <<: *enterprise-rhel-8-64-bit-dynamic-required-expansions + <<: *enterprise-rhel-80-64-bit-dynamic-required-expansions max_sub_suites: 10 # Set a low target time to run only 1 test per generated Evergreen task. Increase timeout. target_resmoke_time: 1 @@ -407,193 +378,3 @@ buildvariants: distros: - rhel80-xlarge - name: .sharding - -- name: enterprise-rhel-8-64-bit-large-txns-format - display_name: "Enterprise RHEL 8 (large transactions format)" - run_on: - - rhel80-small - cron: "0 4 * * 0" - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel80 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - --link-model=dynamic - multiversion_platform: rhel80 - multiversion_edition: enterprise - repo_edition: enterprise - scons_cache_mode: all - scons_cache_scope: shared - num_scons_link_jobs_available: 0.99 - large_distro_name: rhel80-medium - test_flags: >- - --mongodSetParameters="{maxNumberOfTransactionOperationsInSingleOplogEntry: 2}" - --excludeWithAnyTags=exclude_from_large_txns,requires_latch_analyzer - tasks: - - name: compile_test_and_package_parallel_core_stream_TG - distros: - - rhel80-large - - name: auth_gen - - name: auth_audit_gen - - name: causally_consistent_jscore_txns_passthrough - - name: change_streams - - name: change_streams_whole_db_passthrough - - name: change_streams_whole_cluster_passthrough - - name: concurrency_replication_gen - - name: concurrency_replication_multi_stmt_txn_gen - - name: concurrency_sharded_replication_gen - - name: concurrency_sharded_replication_with_balancer_gen - - name: concurrency_sharded_clusterwide_ops_add_remove_shards_gen - - name: concurrency_sharded_local_read_write_multi_stmt_txn_gen - - name: concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer_gen - - name: concurrency_sharded_multi_stmt_txn_gen - - name: concurrency_sharded_multi_stmt_txn_with_balancer_gen - - name: concurrency_sharded_multi_stmt_txn_with_stepdowns_gen - - name: concurrency_sharded_with_stepdowns_gen - - name: concurrency_sharded_with_stepdowns_and_balancer_gen - - name: initial_sync_fuzzer_gen - - name: jsCore - - name: jsCore_txns - - name: .logical_session_cache .repl - - name: .multi_shard - - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - - name: multiversion_auth_gen - - name: multiversion_gen - - name: noPassthrough_gen - - name: .replica_sets !.multi_oplog !.large - - name: .replica_sets !.multi_oplog .large - distros: - - rhel80-medium - - name: .resharding_fuzzer - - name: .rollbackfuzzer - - name: .sharding .txns - - name: sharding_gen - - name: sharding_auth_gen - - name: sharding_auth_audit_gen - - name: sharding_ese_gen - - name: sharding_ese_gcm_gen - - name: sharding_csrs_continuous_config_stepdown_gen - - name: sharded_multi_stmt_txn_jscore_passthrough - distros: - - rhel80-medium - - name: generate_buildid_to_debug_symbols_mapping - -- &enterprise-rhel-8-64-bit-dynamic-sbe - name: enterprise-rhel-8-64-bit-dynamic-sbe - display_name: "Shared Library Enterprise RHEL 8 (SBE)" - cron: "0 4 * * 0" - run_on: - - rhel80-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --link-model=dynamic - multiversion_platform: rhel80 - multiversion_edition: enterprise - has_packages: false - scons_cache_scope: shared - scons_cache_mode: all - jstestfuzz_num_generated_files: 40 - jstestfuzz_concurrent_num_files: 10 - target_resmoke_time: 10 - max_sub_suites: 5 - large_distro_name: rhel80-medium - num_scons_link_jobs_available: 0.99 - test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: false}" - --excludeWithAnyTags=requires_latch_analyzer - tasks: - - name: .aggfuzzer - - name: .aggregation - - name: .auth - - name: .causally_consistent !.sharding - - name: .change_stream_fuzzer - - name: .change_streams - - name: .concurrency !.large !.ubsan !.no_txns !.debug_only - - name: .concurrency .large !.ubsan !.no_txns !.debug_only - distros: - - rhel80-medium - - name: .encrypt - - name: .jscore .common !jsCore - - name: .jstestfuzz !.flow_control - - name: .misc_js - - name: .multi_shard - - name: .query_fuzzer - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - - name: .random_multiversion_ds - - name: .read_only - - name: .read_write_concern !.large - - name: .read_write_concern .large - distros: - - rhel80-medium - - name: .replica_sets !.encrypt !.auth - distros: - - rhel80-xlarge - - name: .rollbackfuzzer - - name: .sharding .common - - name: .sharding .jscore !.wo_snapshot !.multi_stmt - - name: .sharding .txns - - name: .serverless - distros: - - rhel80-xlarge - - name: .updatefuzzer - - name: audit - - name: compile_test_and_package_parallel_core_stream_TG - distros: - - rhel80-xlarge - - name: compile_integration_and_test_parallel_stream_TG - distros: - - rhel80-xlarge - - name: disk_wiredtiger - - name: initial_sync_fuzzer_gen - - name: jsCore - distros: - - rhel80-xlarge - - name: jsCore_minimum_batch_size - - name: jsCore_txns_large_txns_format - - name: json_schema - - name: lint_fuzzer_sanity_patch - - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - - name: multiversion_gen - - name: .multiversion_sanity_check - - name: replica_sets_api_version_jscore_passthrough_gen - - name: replica_sets_reconfig_jscore_passthrough_gen - - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen - distros: - - rhel80-xlarge - - name: replica_sets_reconfig_kill_primary_jscore_passthrough_gen - distros: - - rhel80-xlarge - - name: retryable_writes_jscore_passthrough_gen - - name: retryable_writes_jscore_stepdown_passthrough_gen - - name: sasl - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: secondary_reads_passthrough_gen - - name: session_jscore_passthrough - - name: sharding_api_version_jscore_passthrough_gen - - name: snmp - - name: test_api_version_compatibility - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - -########################################### -# Copybara buildvariants # -########################################### - -- name: ©bara-sync-between-repos copybara-sync-between-repos - display_name: "* Copybara Sync Between Repos" - activate: true - run_on: - - ubuntu2204-small - stepback: false - tasks: - - name: sync_repo_with_copybara - priority: 50 diff --git a/etc/evergreen_timeouts.yml b/etc/evergreen_timeouts.yml index 6f979850021..850085f56fa 100644 --- a/etc/evergreen_timeouts.yml +++ b/etc/evergreen_timeouts.yml @@ -10,59 +10,112 @@ # in alphabetical order. overrides: - enterprise-macos: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours + - task: run_unittests + exec_timeout: 60 enterprise-macos-arm64: + - task: run_unittests + exec_timeout: 60 + + enterprise-macos-cxx20: + - task: run_unittests + exec_timeout: 60 + + enterprise-macos-rosetta-2: + - task: run_unittests + exec_timeout: 60 + + enterprise-rhel-80-64-bit-coverage: - task: replica_sets_jscore_passthrough - exec_timeout: 150 # 2.5 hours + exec_timeout: 150 # 2.5 hours. + + enterprise-rhel-80-64-bit-dynamic-required: + - task: replica_sets_large_txns_format + exec_timeout: 120 # 2 hours. + + enterprise-rhel80-debug-tsan: + - task: run_unittests + exec_timeout: 24 + + enterprise-windows: + - task: replica_sets_jscore_passthrough + exec_timeout: 180 # 3 hours. - enterprise-rhel-8-64-bit-coverage: + enterprise-windows-all-feature-flags-suggested: - task: replica_sets_jscore_passthrough + exec_timeout: 180 # 3 hours. + - task: replica_sets_update_v1_oplog_jscore_passthrough exec_timeout: 150 # 2.5 hours. - enterprise-rhel8-debug-complete-tsan: - - task: aggregation_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + enterprise-windows-inmem: + - task: replica_sets_jscore_passthrough + exec_timeout: 180 # 3 hours. - enterprise-rhel8-debug-tsan: - - task: aggregation_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + enterprise-windows-required: + - task: replica_sets_jscore_passthrough + exec_timeout: 180 # 3 hours. + - task: replica_sets_update_v1_oplog_jscore_passthrough + exec_timeout: 150 # 2.5 hours. + + linux-64-debug: + - task: auth + exec_timeout: 60 # 1 hour. + + linux-64-debug-repeated-execution: + - task: run_unittests + exec_timeout: 120 # 2 hours. macos: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours + - task: run_unittests + exec_timeout: 60 + + macos-debug-suggested: + - task: run_unittests + exec_timeout: 60 macos-arm64: - - task: replica_sets_jscore_passthrough - exec_timeout: 150 # 2.5 hours + - task: run_unittests + exec_timeout: 60 - rhel8-asan: - - task: aggregation_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + rhel80-asan: + - task: run_unittests + exec_timeout: 24 - rhel8-debug-ubsan: - - task: update_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + rhel80-debug-asan: + - task: run_unittests + exec_timeout: 24 + + rhel80-debug-aubsan-lite: + - task: run_unittests + exec_timeout: 24 + + rhel80-debug-ubsan: + - task: run_unittests + exec_timeout: 24 - rhel8-debug-suggested: + rhel80-debug-suggested: - task: replica_sets_jscore_passthrough exec_timeout: 180 # 3 hours. - rhel8-debug-ubsan-all-feature-flags: - - task: update_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours - - rhel8-debug-ubsan-sbe-engine: - - task: update_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + rhel80-ubsan: + - task: run_unittests + exec_timeout: 24 - rhel8-debug-v4ubsan-all-feature-flags-experimental: - - task: update_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + windows: + - task: replica_sets + exec_timeout: 180 # 3 hours. + - task: replica_sets_jscore_passthrough + exec_timeout: 150 # 2.5 hours. - ubuntu1804-asan: - - task: aggregation_timeseries_fuzzer - exec_timeout: 150 # 2.5 hours + windows-debug-suggested: + - task: replica_sets_initsync_jscore_passthrough + exec_timeout: 150 # 2.5 hours. + - task: replica_sets_jscore_passthrough + exec_timeout: 180 # 3 hours. + - task: replica_sets_update_v1_oplog_jscore_passthrough + exec_timeout: 150 # 2.5 hours. diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 109ad1a923d..b0db6165d9b 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -14,6 +14,22 @@ pre_error_fails_task: true stepback: true +# Files that match an ignore-list pattern will not trigger a build, if they're the only modified +# files in the patch. +ignore: +- ".*" +- "!.clang-format" +- "!.eslintrc.yml" +- "*.md" +- "*.rst" +- "*.txt" +- "/distsrc/**" +- "/docs/**" +- "/etc/*.yml" +- "!/etc/evergreen.yml" +- "README" + + ## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds) parameters: - key: patch_compile_flags @@ -52,6 +68,11 @@ parameters: # if a module is added and to be added to the manifest # be sure to add the module to git.get_project revisions parameter modules: +- name: enterprise + repo: git@github.com:10gen/mongo-enterprise-modules.git + prefix: src/mongo/db/modules + branch: v6.0 + - name: wtdevelop repo: git@github.com:wiredtiger/wiredtiger.git prefix: src/third_party @@ -86,19 +107,6 @@ variables: vars: resmoke_args: --help -- &gen_burn_in_task_template - name: gen_burn_in_task_template - depends_on: - - name: version_gen - variant: generate-tasks-for-version - - name: version_burn_in_gen - variant: generate-tasks-for-version - - name: archive_dist_test_debug - commands: - - func: "generate resmoke tasks" - vars: - resmoke_args: --help - - &benchmark_template name: benchmark_template depends_on: @@ -110,7 +118,7 @@ variables: resmoke_args: --help resmoke_jobs_max: 1 - func: "send benchmark results" -# - func: "analyze benchmark results" + - func: "analyze benchmark results" vars: suite: benchmark_suite @@ -142,12 +150,7 @@ variables: is_jstestfuzz: true num_files: 15 num_tasks: 5 # Upperbound by `max_sub_suites` specified through the variant with each task still running `num_files` files. - # It is error prone to require each fuzzer-related Evergreen task to need to override the - # resmoke_args variable. However, the resmoke_args variable must be defined as a string in the - # task generation configuration to satisfy mongodb/mongo-task-generator. We therefore specify an - # empty string for the variable to reflect there are no additional arguments provided to resmoke - # by default for the fuzzer-related tasks. - resmoke_args: "" + resmoke_args: --help # resmoke_args needs to be overridden to specify one of the jstestfuzz suites resmoke_jobs_max: 1 should_shuffle: false continue_on_failure: false @@ -247,8 +250,8 @@ variables: - func: "run hang analyzer" - func: "wait for resmoke to shutdown" -- &stitch_support_task_group_template_community_only - name: stitch_support_task_group_template_community_only +- &stitch_support_task_group_template + name: stitch_support_task_group_template setup_task: - func: "apply compile expansions" - func: "set task expansion macros" @@ -279,8 +282,6 @@ variables: - mongocryptd_variants: &mongocryptd_variants - enterprise-amazon2 - enterprise-amazon2-arm64 - - enterprise-amazon2023 - - enterprise-amazon2023-arm64 - enterprise-debian10-64 - enterprise-debian11-64 - enterprise-linux-64-amazon-ami @@ -289,26 +290,21 @@ variables: - enterprise-rhel-70-64-bit # - enterprise-rhel-70-64-bit-kitchen-sink - enterprise-rhel-70-64-bit-no-libunwind - - enterprise-rhel-71-ppc64le - enterprise-rhel-81-ppc64le - enterprise-rhel-72-s390x - - enterprise-rhel-8-64-bit - - enterprise-rhel-8-64-bit-coverage - - enterprise-rhel-8-64-bit-inmem - - enterprise-rhel-8-64-bit-multiversion - - enterprise-rhel-8-64-bit-suggested - - enterprise-rhel-8-arm64 + - enterprise-rhel-80-64-bit + - enterprise-rhel-80-64-bit-coverage + - enterprise-rhel-80-64-bit-inmem + - enterprise-rhel-80-64-bit-multiversion + - enterprise-rhel-80-64-bit-suggested + - enterprise-rhel-82-arm64 - enterprise-rhel-83-s390x - - enterprise-rhel-90-64-bit - - enterprise-rhel-90-arm64 - enterprise-suse12-64 - enterprise-suse15-64 - enterprise-ubuntu1804-64 - enterprise-ubuntu1804-arm64 - enterprise-ubuntu2004-arm64 - - enterprise-ubuntu2204-arm64 - enterprise-ubuntu2004-64 - - enterprise-ubuntu2204-64 - enterprise-windows - enterprise-windows-debug-unoptimized - enterprise-windows-inmem @@ -325,17 +321,12 @@ variables: - mh_variants: &mh_variants - enterprise-macos - enterprise-macos-arm64 - - enterprise-rhel-8-64-bit - - enterprise-rhel-8-64-bit-dynamic-required + - enterprise-rhel-80-64-bit + - enterprise-rhel-80-64-bit-dynamic-required - enterprise-rhel-70-64-bit - - enterprise-rhel-8-arm64 - - enterprise-rhel-90-64-bit - - enterprise-rhel-90-arm64 + - enterprise-rhel-82-arm64 - enterprise-amazon2-arm64 - - enterprise-amazon2023 - - enterprise-amazon2023-arm64 - enterprise-ubuntu1804-64 - - enterprise-ubuntu2204-64 - enterprise-windows - enterprise-windows-required @@ -344,12 +335,8 @@ variables: - amazon - enterprise-linux-64-amazon-ami - amazon2 - - amazon2023 - - amazon2023-arm64 - enterprise-amazon2 - enterprise-amazon2-arm64 - - enterprise-amazon2023 - - enterprise-amazon2023-arm64 - debian10 - debian11 - enterprise-debian10-64 @@ -357,20 +344,15 @@ variables: - rhel70 - rhel76_compile_rhel70 - enterprise-rhel-70-64-bit - - enterprise-rhel-71-ppc64le - enterprise-rhel-81-ppc64le - enterprise-rhel-72-s390x - ubi8 - - rhel8 - - rhel-8-arm64 - - rhel90 - - rhel90-arm64 - - enterprise-rhel-8-64-bit - - enterprise-rhel-8-64-bit-suggested # For testing selinux. - - enterprise-rhel-8-arm64 + - rhel80 + - rhel-82-arm64 + - enterprise-rhel-80-64-bit + - enterprise-rhel-80-64-bit-suggested # For testing selinux. + - enterprise-rhel-82-arm64 - enterprise-rhel-83-s390x - - enterprise-rhel-90-64-bit - - enterprise-rhel-90-arm64 - suse12 - enterprise-suse12-64 - suse15 @@ -378,13 +360,10 @@ variables: - ubuntu1804 - ubuntu1804-arm64 - ubuntu2004-arm64 - - ubuntu2204-arm64 - ubuntu2004 - - ubuntu2204 - enterprise-ubuntu1804-64 - enterprise-ubuntu1804-arm64 - enterprise-ubuntu2004-arm64 - - enterprise-ubuntu2204-arm64 - enterprise-ubuntu2004-64 - enterprise-windows - windows @@ -531,23 +510,12 @@ functions: args: - "src/evergreen/functions/binaries_extract.sh" - "get version expansions": &get_version_expansions - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${project}/${version_id}/${bv_future_git_tag|version}_expansions.yml - bucket: mciuploads - local_file: src/version_expansions.yml - - "apply version expansions": &apply_version_expansions - command: expansions.update + "check binary version": &check_binary_version + command: subprocess.exec params: - file: src/version_expansions.yml - - "get and apply version expansions": - - *get_version_expansions - - *apply_version_expansions + binary: bash + args: + - "src/evergreen/functions/binary_version_check.sh" "fetch benchmarks": &fetch_benchmarks command: s3.get @@ -616,7 +584,7 @@ functions: "get buildnumber": &get_buildnumber command: keyval.inc params: - key: "${build_variant}_${project}" + key: "${build_variant}_v6.0" destination: "builder_num" "run diskstats": &run_diskstats @@ -827,10 +795,15 @@ functions: remote_file: ${project}/${build_variant}/${revision}/pip-requirements-${task_id}-${execution}.txt bucket: mciuploads permissions: public-read - content_type: text/plain + content_type: atext-plain display_name: Pip Requirements "send benchmark results": + - command: json.send + params: + name: perf + file: src/perf.json + - command: perf.send params: aws_key: ${aws_key} @@ -873,6 +846,7 @@ functions: - *adjust_venv - *fetch_binaries - *extract_binaries + - *check_binary_version - *get_buildnumber - *f_expansions_write - *set_up_credentials @@ -985,14 +959,6 @@ functions: args: - "./src/evergreen/powercycle_sentinel_run.sh" - "sign macos dev binaries": &sign_macos_dev_binaries - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/run_python_script.sh" - - "buildscripts/sign_macos_binaries_for_testing.py" - "execute resmoke tests": &execute_resmoke_tests command: subprocess.exec type: test @@ -1092,7 +1058,7 @@ functions: files: - src/generated_resmoke_config/*.json - "generate version burn in": + "generate burn in tags": - *f_expansions_write - *configure_evergreen_api_credentials - command: subprocess.exec @@ -1100,42 +1066,33 @@ functions: params: binary: bash args: - - "./src/evergreen/generate_version_burn_in.sh" + - "./src/evergreen/burn_in_tests_generate.sh" - command: archive.targz_pack params: - target: generate_tasks_config.tgz - source_dir: src/generated_resmoke_config + target: burn_in_tags_gen.tgz + source_dir: src/generated_burn_in_tags_config include: - "*" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: generate_tasks_config.tgz - remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz + local_file: burn_in_tags_gen.tgz + remote_file: ${project}/${build_variant}/${revision}/burn_in_tags_gen/burn_in_tags_gen-${build_id}.tgz bucket: mciuploads permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Generated Burn In Task Config - Execution ${execution} - optional: true + content_type: application/gzip + display_name: Burn_in_tags Task Config - Execution ${execution} - command: generate.tasks params: - optional: true files: - - src/generated_resmoke_config/*.json - - "initialize multiversion tasks": &initialize_multiversion_tasks - - command: shell.exec - params: - script: "echo 'noop'" + - src/generated_burn_in_tags_config/burn_in_tags_gen.json "generate resmoke tasks": - *fetch_artifacts - *f_expansions_write - *kill_processes - *cleanup_environment - - *get_version_expansions - - *apply_version_expansions - *fetch_venv - *adjust_venv - *f_expansions_write @@ -1185,7 +1142,6 @@ functions: - *update_task_timeout_expansions - *update_task_timeout - *f_expansions_write - - *sign_macos_dev_binaries - command: subprocess.exec params: binary: bash @@ -1236,7 +1192,6 @@ functions: - *update_resmoke_jobs_expansions - *f_expansions_write - *configure_evergreen_api_credentials - - *sign_macos_dev_binaries - command: subprocess.exec params: binary: bash @@ -1269,8 +1224,6 @@ functions: - "src/evergreen/scons_lint.sh" "scons compile": - - *get_version_expansions - - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -1278,43 +1231,6 @@ functions: binary: bash args: - "src/evergreen/scons_compile.sh" - - command: expansions.update - params: - updates: - - key: install_dir - value: ${install_dir} - - "ninja compile": - - *f_expansions_write - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "./src/evergreen/ninja_compile.sh" - - "generate version expansions": &generate_version_expansions - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/functions/version_expansions_generate.sh" - - "upload version expansions": &upload_version_expansions - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/version_expansions.yml - remote_file: ${project}/${version_id}/${bv_future_git_tag|version}_expansions.yml - bucket: mciuploads - permissions: public-read - content_type: application/x-yaml - display_name: version expansions - - "generate and upload version expansions": - - *generate_version_expansions - - *upload_version_expansions "generate compile expansions": - *f_expansions_write @@ -1459,16 +1375,6 @@ functions: args: - "./src/evergreen/lint_fuzzer_sanity_all.sh" - "sync repo with copybara": - - *f_expansions_write - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "src/evergreen/run_python_script.sh" - - "buildscripts/sync_repo_with_copybara.py --expansion-file ../expansions.yml" - # Used by generator "run jstestfuzz": - *f_expansions_write @@ -1600,8 +1506,6 @@ functions: - "./src/evergreen/powercycle_exit.sh" "run packager.py": - - *get_version_expansions - - *apply_version_expansions - *f_expansions_write - command: subprocess.exec params: @@ -1843,25 +1747,11 @@ functions: display_name: Jepsen Test Results - ${execution} optional: true - "upload jepsen_log": &upload_jepsen_log - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/jepsen-mongodb/jepsen_${task_name}_${execution}.log - remote_file: ${project}/${build_variant}/${revision}/jepsen_${task_name}_${execution}.log - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Jepsen Test Log - "save jepsen artifacts": - *tar_jepsen_logs - *archive_jepsen_logs - *tar_jepsen_results - *archive_jepsen_results - - *upload_jepsen_log ### Process & archive mongo coredumps ### "gather mongo coredumps": &gather_mongo_coredumps @@ -2186,16 +2076,9 @@ functions: remote_file: ${project}/${build_variant}/${revision}/local-resmoke-invocation-${task_id}-${execution}.txt bucket: mciuploads permissions: public-read - content_type: text/plain + content_type: atext-plain display_name: Resmoke.py Invocation for Local Usage - "cleanup external auth test artifacts": - - *f_expansions_write - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/sasl_windows_cyrussasl_teardown.sh" # Pre task steps pre: @@ -2226,7 +2109,6 @@ post: - func: "cleanup FUSE watchdog" - func: "cleanup environment" - func: "cleanup jepsen docker test" -- func: "cleanup external auth test artifacts" # Timeout steps timeout: @@ -2234,6 +2116,7 @@ timeout: - func: "run hang analyzer" - func: "wait for resmoke to shutdown" + ####################################### # Tasks # ####################################### @@ -2242,9 +2125,7 @@ tasks: ## compile - build all scons targets except unittests ## - name: compile_dist_test tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version + depends_on: [] commands: - func: "scons compile" vars: @@ -2253,38 +2134,6 @@ tasks: ${additional_compile_targets|} task_compile_flags: >- PREFIX=dist-test - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/record_mongodb_server_version.sh" - - "./build/install/dist-test/bin/mongod" - - "./version_info_mongod.txt" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/version_info_mongod.txt - remote_file: ${project}/${build_variant}/${version_id}/version-info-mongod-${task_id}.txt - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: mongod --version output - -## compile - build all scons targets except unittests ## -- name: compile_dist_test_half - tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - targets: >- - compile_first_half_non_test_source - ${additional_compile_targets|} - task_compile_flags: >- - PREFIX=dist-test - name: determine_patch_tests commands: @@ -2301,20 +2150,13 @@ tasks: depends_on: - name: compile_dist_test commands: + - *f_expansions_write - func: "scons compile" vars: targets: >- archive-dist-test task_compile_flags: >- PREFIX=dist-test - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - args: - - "src/evergreen/run_python_script.sh" - - "evergreen/macos_notary.py" - - "mongodb-binaries.${ext|tgz}" - command: s3.put params: @@ -2363,8 +2205,8 @@ tasks: - "./etc/pip/**" - "./etc/repo_config.yaml" - "./etc/scons/**" - - "./etc/macos_dev_entitlements.xml" - "buildscripts/**" + - "compile_expansions.yml" - "all_feature_flags.txt" # Must correspond to the definition in buildscripts/idl/lib.py. - "jstests/**" - "patch_files.txt" @@ -2429,120 +2271,47 @@ tasks: - name: compile_ninja tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - generating_for_ninja: true - separate_debug: off - task_compile_flags: >- - --ninja - - *f_expansions_write - - func: "ninja compile" - vars: - ninja_file: "build.ninja" - -- name: compile_ninja_default_profile - tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: generating_for_ninja: true separate_debug: off + task_install_action: + default task_compile_flags: >- - --build-profile=default - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --ninja + targets: + generate-ninja - *f_expansions_write - - func: "ninja compile" - vars: - ninja_file: "build.ninja" - -- name: compile_ninja_opt_profile - tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - generating_for_ninja: true - separate_debug: off - task_compile_flags: >- - --build-profile=opt - CCACHE= - ICECC= - - *f_expansions_write - - func: "ninja compile" - vars: - ninja_file: "opt.ninja" - -- name: compile_ninja_fast_profile - tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - generating_for_ninja: true - separate_debug: off - task_compile_flags: >- - --build-profile=fast - CCACHE= - ICECC= - - *f_expansions_write - - func: "ninja compile" - vars: - ninja_file: "fast.ninja" - -- name: compile_ninja_san_profile - tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - generating_for_ninja: true - separate_debug: off - task_compile_flags: >- - --build-profile=san - CCACHE= - ICECC= - - *f_expansions_write - - func: "ninja compile" - vars: - ninja_file: "san.ninja" + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/ninja_compile.sh" - name: compile_ninja_next tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: generating_for_ninja: true separate_debug: off + task_install_action: + default task_compile_flags: >- --build-tools=next --ninja + targets: + generate-ninja - *f_expansions_write - - func: "ninja compile" - vars: - ninja_file: "build.ninja" + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/ninja_compile.sh" - name: compile_build_tools_next tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2553,9 +2322,6 @@ tasks: - name: libdeps_graph_linting tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - *f_expansions_write - command: subprocess.exec @@ -2611,108 +2377,14 @@ tasks: targets: install-all-meta compiling_for_test: true -- name: compile_all_but_not_unittests - tags: [] - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: install-all-meta-but-not-unittests - compiling_for_test: true - -- name: compile_and_run_unittests_first_quarter - exec_timeout_secs: 21600 # 6 hours - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: install-first-quarter-unittests install-first-quarter-unittests-debug - compiling_for_test: true - - func: "f_expansions_write" - - func: "run diskstats" - - func: "f_expansions_write" - - func: "monitor process threads" - - func: "collect system resource info" - - func: "run tests" - vars: - suite: unittests_first_quarter - install_dir: build/install/bin - exec_timeout_secs: 21600 # 6 hours - -- name: compile_and_run_unittests_second_quarter - exec_timeout_secs: 21600 # 6 hours - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: install-second-quarter-unittests install-second-quarter-unittests-debug - compiling_for_test: true - - func: "f_expansions_write" - - func: "run diskstats" - - func: "f_expansions_write" - - func: "monitor process threads" - - func: "collect system resource info" - - func: "run tests" - vars: - suite: unittests_second_quarter - install_dir: build/install/bin - exec_timeout_secs: 21600 # 6 hours - -- name: compile_and_run_unittests_third_quarter - exec_timeout_secs: 21600 # 6 hours - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: install-third-quarter-unittests install-third-quarter-unittests-debug - compiling_for_test: true - - func: "f_expansions_write" - - func: "run diskstats" - - func: "f_expansions_write" - - func: "monitor process threads" - - func: "collect system resource info" - - func: "run tests" - vars: - suite: unittests_third_quarter - install_dir: build/install/bin - exec_timeout_secs: 21600 # 6 hours - -- name: compile_and_run_unittests_fourth_quarter - exec_timeout_secs: 21600 # 6 hours - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: install-fourth-quarter-unittests install-fourth-quarter-unittests-debug - compiling_for_test: true - - func: "f_expansions_write" - - func: "run diskstats" - - func: "f_expansions_write" - - func: "monitor process threads" - - func: "collect system resource info" - - func: "run tests" - vars: - suite: unittests_fourth_quarter - install_dir: build/install/bin - exec_timeout_secs: 21600 # 6 hours - ## clang_tidy - run clang_tidy - name: clang_tidy tags: [] exec_timeout_secs: 3600 # 1 hour timeout for the task overall - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: + - *f_expansions_write - func: "scons compile" vars: - task_compile_flags: >- - --build-profile=compiledb targets: generated-sources compiledb compiling_for_test: true - command: subprocess.exec @@ -2757,7 +2429,7 @@ tasks: ## run_unittests with UndoDB live-record ## #- name: run_unittests_with_recording # depends_on: -# - name: compile_unittests_for_recorded_unittest +# - name: compile_unittests_for_recorded_unittest # commands: # - *f_expansions_write # - func: "run diskstats" @@ -2786,9 +2458,6 @@ tasks: ##compile_and_archive_libfuzzertests - build libfuzzertests ## - name: compile_and_archive_libfuzzertests tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2819,9 +2488,6 @@ tasks: - name: server_discovery_and_monitoring_json_test tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2834,9 +2500,6 @@ tasks: - name: server_selection_json_test tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2882,21 +2545,8 @@ tasks: targets: archive-dbtest archive-dbtest-debug compiling_for_test: true -- name: compile_integration_test - tags: [] - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: install-integration-tests - compiling_for_test: true - - name: compile_visibility_test tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2924,12 +2574,7 @@ tasks: - name: embedded_sdk_build_cdriver tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - - *get_version_expansions - - *apply_version_expansions - func: f_expansions_write - command: subprocess.exec params: @@ -2948,6 +2593,7 @@ tasks: task_compile_flags: &embedded_sdk_compile_flags >- --allocator=system --dbg=off + --enable-free-mon=off --enable-http-client=off --js-engine=none --opt=size @@ -2963,8 +2609,6 @@ tasks: depends_on: - name: embedded_sdk_install_dev commands: - - *get_version_expansions - - *apply_version_expansions - *f_expansions_write - command: subprocess.exec params: @@ -3008,8 +2652,6 @@ tasks: depends_on: - name: embedded_sdk_install_tests commands: - - *get_version_expansions - - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -3036,8 +2678,6 @@ tasks: depends_on: - name: embedded_sdk_install_tests commands: - - *get_version_expansions - - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -3056,8 +2696,6 @@ tasks: depends_on: - name: embedded_sdk_run_tests commands: - - *get_version_expansions - - *apply_version_expansions # A second put, this time to -latest, to give devs a reasonable # way to get the most recent build. - command: s3.put @@ -3077,8 +2715,6 @@ tasks: depends_on: - name: embedded_sdk_run_tests commands: - - *get_version_expansions - - *apply_version_expansions # A second put, this time to -latest, to give devs a reasonable # way to get the most recent build. - command: s3.put @@ -3093,11 +2729,8 @@ tasks: permissions: public-read content_type: ${content_type|application/x-gzip} -- name: stitch_support_create_lib_community_only +- name: stitch_support_create_lib tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -3105,6 +2738,7 @@ tasks: targets: install-stitch-support install-stitch-support-debug install-stitch-support-dev task_compile_flags: >- --link-model=dynamic-sdk + --enable-free-mon=off --ssl=off --enable-http-client=off --modules= @@ -3126,27 +2760,27 @@ tasks: content_type: application/tar display_name: "Stitch Support Library" -- name: stitch_support_install_tests_community_only +- name: stitch_support_install_tests tags: [] depends_on: - - name: stitch_support_create_lib_community_only + - name: stitch_support_create_lib commands: - func: "scons compile" vars: targets: install-stitch-support-test compiling_for_test: true task_compile_flags: >- + --enable-free-mon=off --ssl=off --enable-http-client=off --modules= DESTDIR='$BUILD_ROOT/stitch-support-lib-$MONGO_VERSION' -- name: stitch_support_run_tests_community_only +- name: stitch_support_run_tests tags: [] depends_on: - - name: stitch_support_install_tests_community_only + - name: stitch_support_install_tests commands: - - func: "get and apply version expansions" - *f_expansions_write - command: subprocess.exec type: test @@ -3157,9 +2791,6 @@ tasks: - name: crypt_create_lib tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -3167,7 +2798,8 @@ tasks: targets: archive-mongo-crypt-dev task_compile_flags: >- --allocator=system - --enterprise-features=fle,search,vector_search + --enable-free-mon=off + --enterprise-features=fle --js-engine=none --link-model=dynamic-sdk --enable-http-client=off @@ -3192,9 +2824,6 @@ tasks: - name: crypt_create_debug_lib tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -3204,7 +2833,8 @@ tasks: --dbg=on --opt=off --allocator=system - --enterprise-features=fle,search,vector_search + --enable-free-mon=off + --enterprise-features=fle --js-engine=none --link-model=dynamic-sdk DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' @@ -3239,13 +2869,15 @@ tasks: vars: targets: archive-mongo-crypt-shlib-test compiling_for_test: true + task_install_action: + default task_compile_flags: >- --allocator=system - --enterprise-features=fle,search,vector_search + --enable-free-mon=off + --enterprise-features=fle --js-engine=none --link-model=static DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' - RPATH='$$RPATH_ESCAPED_DOLLAR_ORIGIN/../lib' - command: s3.put params: aws_key: ${aws_key} @@ -3262,7 +2894,6 @@ tasks: depends_on: - name: crypt_install_tests commands: - - func: "get and apply version expansions" - *f_expansions_write - command: subprocess.exec type: test @@ -3273,9 +2904,7 @@ tasks: - name: compile_benchmarks tags: [] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version + depends_on: [] commands: - command: manifest.load - func: "git get project and add git tag" @@ -3465,26 +3094,46 @@ tasks: args: - "src/evergreen/check_idl_compat.sh" -- <<: *gen_burn_in_task_template - name: burn_in_tests_gen +- name: burn_in_tests_gen tags: [] - patch_only: true commands: - - func: "generate resmoke tasks" - -- <<: *gen_burn_in_task_template - name: burn_in_tags_gen - tags: [] - patch_only: true - commands: - - func: "generate resmoke tasks" + - command: manifest.load + - func: "git get project and add git tag" + - *f_expansions_write + - *kill_processes + - *cleanup_environment + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/burn_in_tests.sh" + env: + BURN_IN_TESTS: ${burn_in_tests} + - command: archive.targz_pack + params: + target: src/burn_in_tests_gen.tgz + source_dir: src + include: + - burn_in_tests_gen.json -- <<: *gen_burn_in_task_template - name: burn_in_tasks_gen - tags: [] - patch_only: true - commands: - - func: "generate resmoke tasks" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/burn_in_tests_gen.tgz + remote_file: ${project}/${build_variant}/${revision}/burn_in_tests_gen/burn_in_tests_gen-${build_id}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/gzip + display_name: Burn_in_tests Task Config - Execution ${execution} + - command: generate.tasks + params: + files: + - src/burn_in_tests_gen.json - <<: *benchmark_template name: benchmarks_orphaned @@ -3497,7 +3146,7 @@ tasks: exec_timeout_secs: 14400 # 4 hour timeout. resmoke_jobs_max: 1 - func: "send benchmark results" -# - func: "analyze benchmark results" + - func: "analyze benchmark results" vars: suite: benchmarks @@ -3511,7 +3160,7 @@ tasks: suite: benchmarks_sharding resmoke_jobs_max: 1 - func: "send benchmark results" -# - func: "analyze benchmark results" + - func: "analyze benchmark results" # Disabled under SERVER-64949. # - <<: *benchmark_template @@ -3617,18 +3266,14 @@ tasks: name: initial_sync_multiversion_fuzzer_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - initial_sync_fuzzer_last_lts: last_lts - initial_sync_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 10 num_tasks: 5 npm_command: initsync-fuzzer + suite: initial_sync_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - run_no_feature_flag_tests: "true" ## initial sync generational fuzzer ## - <<: *jstestfuzz_template @@ -3649,36 +3294,28 @@ tasks: name: aggregation_multiversion_fuzzer_gen tags: ["aggfuzzer", "common", "multiversion", "require_npm", "random_name", "future_git_tag_incompatible"] commands: - - func: "initialize multiversion tasks" - vars: - aggregation_multiversion_fuzzer_last_lts: last_lts - aggregation_multiversion_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 + suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: agg-fuzzer - run_no_feature_flag_tests: "true" ## Standalone generational fuzzer for multiversion aggregation expressions ## - <<: *jstestfuzz_template name: aggregation_expression_multiversion_fuzzer_gen tags: ["aggfuzzer", "multiversion", "require_npm", "random_name"] commands: - - func: "initialize multiversion tasks" - vars: - aggregation_expression_multiversion_fuzzer_last_lts: last_lts - aggregation_expression_multiversion_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 + suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: agg-expr-fuzzer - run_no_feature_flag_tests: "true" ## Standalone generational fuzzer for checking optimized and unoptimized expression equivalence - <<: *jstestfuzz_template @@ -3790,18 +3427,14 @@ tasks: name: update_fuzzer_gen tags: ["updatefuzzer", "require_npm", "random_name", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - update_fuzzer_last_lts: last_lts - update_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 npm_command: update-fuzzer + suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - run_no_feature_flag_tests: "true" ## jstestfuzz standalone update time-series generational fuzzer ## - <<: *jstestfuzz_template @@ -3823,38 +3456,30 @@ tasks: name: update_fuzzer_replication_gen tags: ["updatefuzzer", "require_npm", "random_name", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - update_fuzzer_replication_last_lts: last_lts - update_fuzzer_replication_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 npm_command: update-fuzzer + suite: generational_fuzzer_replication resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - run_no_feature_flag_tests: "true" ## rollback multiversion fuzzer ## - <<: *jstestfuzz_template name: rollback_multiversion_fuzzer_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - rollback_fuzzer_last_lts: last_lts - rollback_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 3 num_tasks: 5 npm_command: rollback-fuzzer + suite: rollback_fuzzer # Rollback suites create indexes with majority of nodes not available for replication. So, disabling # index build commit quorum. resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}, enableIndexBuildCommitQuorum: false}'" - run_no_feature_flag_tests: "true" ## rollback generational fuzzer ## - <<: *jstestfuzz_template @@ -4160,21 +3785,13 @@ tasks: name: jstestfuzz_replication_multiversion_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - jstestfuzz_replication_last_continuous_new_new_old: last_continuous - jstestfuzz_replication_last_continuous_new_old_new: last_continuous - jstestfuzz_replication_last_continuous_old_new_new: last_continuous - jstestfuzz_replication_last_lts_new_new_old: last_lts - jstestfuzz_replication_last_lts_new_old_new: last_lts - jstestfuzz_replication_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars jstestfuzz_vars: --jsTestsDir ../jstests resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" + suite: jstestfuzz_replication npm_command: jstestfuzz - run_no_feature_flag_tests: "true" ## jstestfuzz initial sync replica set ## - <<: *jstestfuzz_template @@ -4234,17 +3851,13 @@ tasks: name: jstestfuzz_sharded_multiversion_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - jstestfuzz_sharded_last_continuous_new_old_old_new: last_continuous - jstestfuzz_sharded_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars jstestfuzz_vars: --jsTestsDir ../jstests resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" + suite: jstestfuzz_sharded npm_command: jstestfuzz - run_no_feature_flag_tests: "true" ## jstestfuzz sharded cluster causal consistency ## - <<: *jstestfuzz_template @@ -4370,33 +3983,15 @@ tasks: name: multiversion_sanity_check_gen tags: ["multiversion", "multiversion_sanity_check"] commands: - - func: "initialize multiversion tasks" - vars: - multiversion_sanity_check_last_continuous_new_new_old: last_continuous - multiversion_sanity_check_last_continuous_new_old_new: last_continuous - multiversion_sanity_check_last_continuous_old_new_new: last_continuous - multiversion_sanity_check_last_lts_new_new_old: last_lts - multiversion_sanity_check_last_lts_new_old_new: last_lts - multiversion_sanity_check_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" - vars: - run_no_feature_flag_tests: "true" - <<: *gen_task_template name: replica_sets_jscore_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: - - func: "initialize multiversion tasks" - vars: - replica_sets_jscore_passthrough_last_continuous_new_new_old: last_continuous - replica_sets_jscore_passthrough_last_continuous_new_old_new: last_continuous - replica_sets_jscore_passthrough_last_continuous_old_new_new: last_continuous - replica_sets_jscore_passthrough_last_lts_new_new_old: last_lts - replica_sets_jscore_passthrough_last_lts_new_old_new: last_lts - replica_sets_jscore_passthrough_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: replica_sets_jscore_passthrough # Check that the mutational fuzzer can parse JS files modified in a patch build. @@ -4427,20 +4022,6 @@ tasks: - func: "setup jstestfuzz" - func: "lint fuzzer sanity all" - -- name: sync_repo_with_copybara - tags: [] - commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - *kill_processes - - *cleanup_environment - - *set_up_venv - - func: "configure evergreen api credentials" - - *f_expansions_write - - func: "sync repo with copybara" - ## integration test suites ## - <<: *task_template @@ -4564,31 +4145,27 @@ tasks: commands: - func: "generate resmoke tasks" -- name: version_gen - commands: - - command: manifest.load - - *git_get_project - - *f_expansions_write - - *add_git_tag - - *kill_processes - - *cleanup_environment - - func: "set up venv" - - func: "upload pip requirements" - - func: "generate version" - -- name: version_burn_in_gen +- name: burn_in_tags_gen + tags: [] + depends_on: + - name: archive_dist_test commands: - command: manifest.load - - *git_get_project + - func: "git get project and add git tag" - *f_expansions_write - - *add_git_tag - *kill_processes - *cleanup_environment - func: "set up venv" - func: "upload pip requirements" - - func: "generate version burn in" + - func: "configure evergreen api credentials" + - func: "generate burn in tags" + vars: + max_revisions: 25 + repeat_tests_secs: 600 + repeat_tests_min: 2 + repeat_tests_max: 1000 -- name: version_expansions_gen +- name: version_gen commands: - command: manifest.load - *git_get_project @@ -4598,7 +4175,7 @@ tasks: - *cleanup_environment - func: "set up venv" - func: "upload pip requirements" - - func: "generate and upload version expansions" + - func: "generate version" - name: selected_tests_gen tags: [] @@ -4636,29 +4213,15 @@ tasks: name: change_streams_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: - - func: "initialize multiversion tasks" - vars: - change_streams_last_continuous_new_new_old: last_continuous - change_streams_last_continuous_new_old_new: last_continuous - change_streams_last_continuous_old_new_new: last_continuous - change_streams_last_lts_new_new_old: last_lts - change_streams_last_lts_new_old_new: last_lts - change_streams_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: change_streams - <<: *gen_task_template name: change_streams_downgrade_gen tags: ["multiversion_passthrough", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - change_streams_downgrade_last_continuous_new_old_old_new: last_continuous - change_streams_downgrade_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" - vars: - run_no_feature_flag_tests: "true" - <<: *task_template name: change_streams_update_v1_oplog @@ -4716,17 +4279,13 @@ tasks: name: change_streams_sharded_collections_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - change_streams_sharded_collections_passthrough_last_continuous_new_old_old_new: last_continuous - change_streams_sharded_collections_passthrough_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: change_streams_sharded_collections_passthrough - <<: *gen_task_template name: multiversion_future_git_tag_gen - tags: ["multiversion", "no_multiversion_generate_tasks", "multiversion_future_git_tag"] + tags: ["multiversion", "no_version_combination", "multiversion_future_git_tag"] commands: - func: "generate resmoke tasks" vars: @@ -4734,7 +4293,7 @@ tasks: - <<: *gen_task_template name: multiversion_auth_future_git_tag_gen - tags: ["auth", "multiversion", "no_multiversion_generate_tasks", "multiversion_future_git_tag"] + tags: ["auth", "multiversion", "no_version_combination", "multiversion_future_git_tag"] commands: - func: "generate resmoke tasks" vars: @@ -4878,77 +4437,93 @@ tasks: - <<: *task_template name: integration_tests_standalone tags: ["integration", "standalone"] - depends_on: - - name: compile_integration_test commands: - - *f_expansions_write - - *run_diskstats - - *monitor_process_threads - - *collect_system_resource_info - - *collect_ulimit_info - - func: "run tests" + - command: manifest.load + - func: "git get project and add git tag" + - func: "do setup" + - func: "set up win mount script" + - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. + - func: "apply compile expansions" + - func: "scons compile" vars: - install_dir: build/install/bin + targets: install-integration-tests + compiling_for_test: true + - func: "attach scons logs" + - func: "send scons cedar report" + - func: "run tests" - <<: *task_template name: integration_tests_standalone_audit tags: ["integration", "audit"] - depends_on: - - name: compile_integration_test commands: - - *f_expansions_write - - *run_diskstats - - *monitor_process_threads - - *collect_system_resource_info - - *collect_ulimit_info - - func: "run tests" + - command: manifest.load + - func: "git get project and add git tag" + - func: "do setup" + - func: "set up win mount script" + - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. + - func: "apply compile expansions" + - func: "scons compile" vars: - install_dir: build/install/bin + targets: install-integration-tests + compiling_for_test: true + - func: "attach scons logs" + - func: "send scons cedar report" + - func: "run tests" - <<: *task_template name: integration_tests_replset tags: ["integration"] - depends_on: - - name: compile_integration_test commands: - - *f_expansions_write - - *run_diskstats - - *monitor_process_threads - - *collect_system_resource_info - - *collect_ulimit_info - - func: "run tests" + - command: manifest.load + - func: "git get project and add git tag" + - func: "do setup" + - func: "set up win mount script" + - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. + - func: "apply compile expansions" + - func: "scons compile" vars: - install_dir: build/install/bin + targets: install-integration-tests + compiling_for_test: true + - func: "attach scons logs" + - func: "send scons cedar report" + - func: "run tests" - <<: *task_template name: integration_tests_replset_ssl_auth tags: ["integration"] - depends_on: - - name: compile_integration_test commands: - - *f_expansions_write - - *run_diskstats - - *monitor_process_threads - - *collect_system_resource_info - - *collect_ulimit_info - - func: "run tests" + - command: manifest.load + - func: "git get project and add git tag" + - func: "do setup" + - func: "set up win mount script" + - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. + - func: "apply compile expansions" + - func: "scons compile" vars: - install_dir: build/install/bin + targets: install-integration-tests + compiling_for_test: true + - func: "attach scons logs" + - func: "send scons cedar report" + - func: "run tests" - <<: *task_template name: integration_tests_sharded tags: ["integration", "sharded"] - depends_on: - - name: compile_integration_test commands: + - command: manifest.load + - func: "git get project and add git tag" - *f_expansions_write - - *run_diskstats - - *monitor_process_threads - - *collect_system_resource_info - - *collect_ulimit_info - - func: "run tests" + - func: "do setup" + - func: "set up win mount script" + - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. + - func: "apply compile expansions" + - func: "scons compile" vars: - install_dir: build/install/bin + targets: install-integration-tests + compiling_for_test: true + - func: "attach scons logs" + - func: "send scons cedar report" + - func: "run tests" - <<: *task_template name: external_auth @@ -5001,20 +4576,6 @@ tasks: resmoke_args: --includeWithAnyTags=requires_domain_controller - <<: *task_template - name: sasl_windows_cyrussasl - tags: ["assigned_to_jira_team_server_security", "sasl", "experimental"] - commands: - - func: "f_expansions_write" - - func: "do setup" - - func: "f_expansions_write" - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/sasl_windows_cyrussasl_setup.sh" - - func: "run tests" - -- <<: *task_template name: jsCore tags: ["jscore", "common"] commands: @@ -5031,9 +4592,7 @@ tasks: - func: "run tests" vars: suite: core - resmoke_args: >- - --fuzzMongodConfigs - --excludeWithAnyTags=does_not_support_config_fuzzer + resmoke_args: --fuzzMongodConfigs - <<: *task_template name: config_fuzzer_concurrency @@ -5043,9 +4602,7 @@ tasks: - func: "run tests" vars: suite: concurrency - resmoke_args: >- - --fuzzMongodConfigs - --excludeWithAnyTags=does_not_support_config_fuzzer + resmoke_args: --fuzzMongodConfigs - <<: *task_template name: config_fuzzer_simulate_crash_concurrency_replication @@ -5055,9 +4612,7 @@ tasks: - func: "run tests" vars: suite: simulate_crash_concurrency_replication - resmoke_args: >- - --fuzzMongodConfigs - --excludeWithAnyTags=does_not_support_config_fuzzer + resmoke_args: --fuzzMongodConfigs - <<: *task_template name: config_fuzzer_concurrency_replication @@ -5067,9 +4622,7 @@ tasks: - func: "run tests" vars: suite: concurrency_replication - resmoke_args: >- - --fuzzMongodConfigs - --excludeWithAnyTags=does_not_support_config_fuzzer + resmoke_args: --fuzzMongodConfigs - <<: *task_template name: config_fuzzer_replica_sets_jscore_passthrough @@ -5079,9 +4632,7 @@ tasks: - func: "run tests" vars: suite: replica_sets_jscore_passthrough - resmoke_args: >- - --fuzzMongodConfigs - --excludeWithAnyTags=does_not_support_config_fuzzer + resmoke_args: --fuzzMongodConfigs - <<: *task_template name: jsCore_ese @@ -5348,13 +4899,13 @@ tasks: - <<: *gen_task_template name: multiversion_auth_gen - tags: ["auth", "multiversion", "no_multiversion_generate_tasks", "future_git_tag_incompatible"] + tags: ["auth", "multiversion", "no_version_combination", "future_git_tag_incompatible"] commands: - func: "generate resmoke tasks" - <<: *gen_task_template name: multiversion_gen - tags: ["multiversion", "no_multiversion_generate_tasks", "future_git_tag_incompatible"] + tags: ["multiversion", "no_version_combination", "future_git_tag_incompatible"] commands: - func: "generate resmoke tasks" @@ -5363,7 +4914,7 @@ tasks: # build variants that enable this task. - <<: *gen_task_template name: feature_flag_multiversion_gen - tags: ["multiversion", "no_multiversion_generate_tasks"] + tags: ["multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" @@ -5377,13 +4928,7 @@ tasks: - <<: *gen_task_template name: noPassthrough_gen - tags: - [ - "incompatible_mac", - "incompatible_ppc", - "incompatible_s390x", - "misc_js" - ] + tags: ["misc_js"] commands: - func: "generate resmoke tasks" vars: @@ -5428,23 +4973,20 @@ tasks: vars: resmoke_jobs_max: 1 -- <<: *gen_task_template - name: sharded_collections_jscore_passthrough_gen +- <<: *task_template + name: sharded_collections_jscore_passthrough tags: ["sharding", "jscore"] commands: - - func: "generate resmoke tasks" + - func: "do setup" + - func: "run tests" - <<: *gen_task_template name: sharded_collections_jscore_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - sharded_collections_jscore_passthrough_last_continuous_new_old_old_new: last_continuous - sharded_collections_jscore_passthrough_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: sharded_collections_jscore_passthrough - <<: *task_template name: sharding_jscore_passthrough @@ -5457,13 +4999,9 @@ tasks: name: sharding_jscore_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - sharding_jscore_passthrough_last_lts_new_old_old_new: last_lts - sharding_jscore_passthrough_last_continuous_new_old_old_new: last_continuous - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: sharding_jscore_passthrough - <<: *gen_task_template name: sharding_api_version_jscore_passthrough_gen @@ -5528,34 +5066,6 @@ tasks: vars: use_large_distro: "true" -- <<: *task_template - name: query_stats_passthrough - tags: [] - commands: - - func: "do setup" - - func: "run tests" - -- <<: *task_template - name: query_stats_aggregation_passthrough - tags: [] - commands: - - func: "do setup" - - func: "run tests" - -- <<: *task_template - name: query_stats_mongos_passthrough - tags: [] - commands: - - func: "do setup" - - func: "run tests" - -- <<: *task_template - name: query_stats_mongos_aggregation_passthrough - tags: [] - commands: - - func: "do setup" - - func: "run tests" - - <<: *gen_task_template name: tenant_migration_jscore_passthrough_gen tags: ["serverless"] @@ -5647,6 +5157,14 @@ tasks: use_large_distro: "true" - <<: *gen_task_template + name: sharding_batched_deletes_passthrough_gen + tags: ["sharding", "large", "batched_deletes"] + commands: + - func: "generate resmoke tasks" + vars: + use_large_distro: "true" + +- <<: *gen_task_template name: change_streams_batched_deletes_passthrough_gen tags: ["change_streams", "large", "batched_deletes"] commands: @@ -5706,17 +5224,9 @@ tasks: name: concurrency_replication_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: - - func: "initialize multiversion tasks" - vars: - concurrency_replication_last_continuous_new_new_old: last_continuous - concurrency_replication_last_continuous_new_old_new: last_continuous - concurrency_replication_last_continuous_old_new_new: last_continuous - concurrency_replication_last_lts_new_new_old: last_lts - concurrency_replication_last_lts_new_old_new: last_lts - concurrency_replication_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: concurrency_replication - <<: *gen_task_template name: concurrency_replication_causal_consistency_gen @@ -5792,13 +5302,9 @@ tasks: name: concurrency_sharded_replication_multiversion_gen tags: ["multiversion_passthrough", "sharded", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - concurrency_sharded_replication_last_continuous_new_old_old_new: last_continuous - concurrency_sharded_replication_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: concurrency_sharded_replication - <<: *gen_task_template name: concurrency_sharded_replication_with_balancer_gen @@ -6006,13 +5512,12 @@ tasks: vars: use_large_distro: "true" -- <<: *gen_task_template - name: write_concern_majority_passthrough_gen +- <<: *task_template + name: write_concern_majority_passthrough tags: ["read_write_concern", "large", "write"] commands: - - func: "generate resmoke tasks" - vars: - use_large_distro: "true" + - func: "do setup" + - func: "run tests" - <<: *task_template name: cwrwc_passthrough @@ -6097,13 +5602,9 @@ tasks: name: replica_sets_multiversion_gen tags: ["random_multiversion_ds", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - replica_sets_last_lts: last_lts - replica_sets_last_continuous: last_continuous - func: "generate resmoke tasks" vars: - run_no_feature_flag_tests: "true" + suite: replica_sets - <<: *task_template name: sasl @@ -6131,14 +5632,10 @@ tasks: name: sharding_multiversion_gen tags: ["random_multiversion_ds", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - sharding_last_continuous: last_continuous - sharding_last_lts: last_lts - func: "generate resmoke tasks" vars: use_large_distro: "true" - run_no_feature_flag_tests: "true" + suite: sharding - <<: *gen_task_template name: sharding_max_mirroring_gen @@ -6199,16 +5696,12 @@ tasks: resmoke_jobs_max: 1 - <<: *gen_task_template - name: sharding_mongos_and_mixed_shards_gen - tags: ["sharding", "common", "multiversion"] + name: sharding_last_lts_mongos_and_mixed_shards_gen + tags: ["sharding", "common", "multiversion", "no_version_combination"] commands: - - func: "initialize multiversion tasks" - vars: - sharding_last_lts_mongos_and_mixed_shards: last_lts - func: "generate resmoke tasks" vars: use_large_distro: "true" - run_no_feature_flag_tests: "true" - <<: *gen_task_template name: sharding_update_v1_oplog_gen @@ -6261,15 +5754,6 @@ tasks: vars: suite: ssl_x509 -- <<: *gen_task_template - name: ssl_linear_gen - tags: ["encrypt", "ssl"] - commands: - - func: "generate resmoke tasks" - vars: - suite: ssl_linear - resmoke_jobs_max: 1 - - <<: *task_template name: jsCore_decimal tags: ["jscore", "common", "decimal"] @@ -6338,28 +5822,25 @@ tasks: vars: use_large_distro: "true" +# Use explicit task definitions for retryable_writes_downgrade suites to avoid running +# with all Repl multiversion combinations. - <<: *gen_task_template - name: retryable_writes_downgrade_gen - tags: ["multiversion_passthrough", "multiversion"] + name: retryable_writes_downgrade_last_continuous_gen + tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] + commands: + - func: "generate resmoke tasks" + +- <<: *gen_task_template + name: retryable_writes_downgrade_last_lts_gen + tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] commands: - - func: "initialize multiversion tasks" - vars: - retryable_writes_downgrade_last_lts: last_lts - func: "generate resmoke tasks" - vars: - run_no_feature_flag_tests: "true" - <<: *gen_task_template name: sharded_retryable_writes_downgrade_gen tags: ["multiversion_passthrough", "multiversion"] commands: - - func: "initialize multiversion tasks" - vars: - sharded_retryable_writes_downgrade_last_lts_new_old_old_new: last_lts - sharded_retryable_writes_downgrade_last_continuous_new_old_old_new: last_continuous - func: "generate resmoke tasks" - vars: - run_no_feature_flag_tests: "true" - <<: *gen_task_template name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen @@ -6472,6 +5953,15 @@ tasks: resmoke_jobs_max: 1 - <<: *task_template + name: free_monitoring + tags: [] + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_jobs_max: 1 + +- <<: *task_template name: client_encrypt tags: ["ssl", "encrypt"] commands: @@ -6588,11 +6078,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle @@ -6606,11 +6092,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle_kill_mongod @@ -6624,11 +6106,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle_last_lts_fcv @@ -6642,11 +6120,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle_replication @@ -6660,11 +6134,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle_replication_smalloplog @@ -6678,11 +6148,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle_syncdelay @@ -6696,11 +6162,7 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output - name: powercycle_write_concern_majority @@ -6714,13 +6176,10 @@ tasks: vars: <<: *powercycle_remote_credentials - func: "set up EC2 instance" - vars: - <<: *powercycle_remote_credentials - func: "run powercycle test" - vars: - <<: *powercycle_remote_credentials timeout_secs: 1800 # 30 minute timeout for no output + - name: selinux_rhel8_org tags: [] depends_on: @@ -6757,42 +6216,6 @@ tasks: distro: rhel80-selinux test_list: jstests/selinux/*.js src/mongo/db/modules/enterprise/jstests/selinux/*.js -- name: selinux_rhel9_org - tags: [] - depends_on: - - name: archive_dist_test - - name: package - commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - func: "set up venv" - - func: "fetch packages" - - func: "fetch binaries" - - func: "extract binaries" - - func: "run selinux tests" - vars: - distro: rhel90-selinux - test_list: jstests/selinux/*.js -- name: selinux_rhel9_enterprise - tags: [] - depends_on: - - name: archive_dist_test - - name: package - commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - func: "set up venv" - - func: "fetch packages" - - func: "fetch binaries" - - func: "extract binaries" - - func: "run selinux tests" - vars: - distro: rhel90-selinux - test_list: jstests/selinux/*.js src/mongo/db/modules/enterprise/jstests/selinux/*.js - - - name: selinux_rhel7_org tags: [] depends_on: @@ -6845,6 +6268,26 @@ tasks: - func: "upload pip requirements" - func: "run idl tests" +- name: blackduck_scanner + tags: [] + patchable: false + commands: + - *f_expansions_write + - func: "do non-compile setup" + - command: subprocess.exec + type: setup + params: + binary: bash + silent: true + args: + - "src/evergreen/blackduck_setup.sh" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/blackduck_hub.sh" + - name: tla_plus tags: [] commands: @@ -6880,6 +6323,26 @@ tasks: - func: "do setup" - func: "run tests" +- name: test_packages + tags: [] + depends_on: + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - *f_expansions_write + - *kill_processes + - *cleanup_environment + - *set_up_venv + - func: "fetch packages" + - func: "set up remote credentials" + vars: + private_key_file: ~/.ssh/kitchen.pem + private_key_remote: ${kitchen_private_key} + aws_key_remote: ${kitchen_aws_key} + aws_secret_remote: ${kitchen_aws_secret} + - func: "run kitchen" + - name: package tags: [] depends_on: @@ -6889,6 +6352,7 @@ tasks: - func: "scons compile" vars: targets: >- + distsrc-${ext|tgz} archive-dist archive-dist-debug archive-jstestshell @@ -6896,14 +6360,6 @@ tasks: ${additional_package_targets|} task_compile_flags: >- --legacy-tarball - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - args: - - "src/evergreen/run_python_script.sh" - - "evergreen/macos_notary.py" - - "mongodb-dist.${ext|tgz}" - func: "f_expansions_write" - command: subprocess.exec params: @@ -6912,6 +6368,19 @@ tasks: - "src/evergreen/package.sh" - command: s3.put params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/distsrc.${ext|tgz} + remote_file: ${project}/${build_variant}/${revision}/sources/mongo-src-${build_id}.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Source tarball + # We only need to upload the source tarball from one of the build variants + # because it should be the same everywhere, so just use rhel70/windows. + build_variants: [rhel70, windows] + - command: s3.put + params: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} @@ -7018,7 +6487,6 @@ tasks: - name: publish_packages - run_on: rhel80-small tags: ["publish"] # This should prevent this task from running in patch builds, where we # don't want to publish packages. @@ -7033,7 +6501,6 @@ tasks: commands: - command: manifest.load - func: "git get project and add git tag" - - func: "get and apply version expansions" - func: "f_expansions_write" - func: "kill processes" - func: "cleanup environment" @@ -7054,7 +6521,6 @@ tasks: - "./src/evergreen/packages_publish.sh" - name: push - run_on: rhel8.7-small tags: ["publish"] patchable: false depends_on: @@ -7066,7 +6532,6 @@ tasks: commands: - command: manifest.load - func: "git get project and add git tag" - - func: "get and apply version expansions" - func: "f_expansions_write" - func: "kill processes" - func: "cleanup environment" @@ -7092,6 +6557,14 @@ tasks: local_file: src/mh.tgz build_variants: *mh_variants # Fetch the sources (on relevant variants only) + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${project}/${build_variant}/${revision}/sources/mongo-src-${build_id}.${ext|tgz} + bucket: mciuploads + local_file: src/distsrc.${ext|tgz} + build_variants: [rhel70, windows] - func: "generate compile expansions" - func: "apply compile expansions" - func: "fetch dist debugsymbols" @@ -7100,29 +6573,13 @@ tasks: aws_key_remote: ${repo_aws_key} aws_secret_remote: ${repo_aws_secret} - func: "f_expansions_write" - - # login to container registry - - command: shell.exec - params: - shell: bash - silent: true - script: | - set -oe - echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry} - - # signing windows artifacts - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/garasign_jsign_sign.sh" - - # signing linux artifacts + - func: "set up notary client credentials" - command: subprocess.exec + type: test params: binary: bash args: - - "./src/evergreen/garasign_gpg_sign.sh" + - "./src/evergreen/notary_client_run.sh" # Put the binaries tarball/zipfile - command: s3.put @@ -7156,6 +6613,17 @@ tasks: content_type: ${content_type|application/gzip} remote_file: ${push_path}-STAGE/${push_name}/mh-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz} build_variants: *mh_variants + # Put the source tarball + - command: s3.put + params: + aws_secret: ${aws_secret} + local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz} + aws_key: ${aws_key} + bucket: build-push-testing + permissions: public-read + content_type: ${content_type|application/gzip} + remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz} + build_variants: [rhel70, windows] # Put the debug symbols # push directly to repo due to limitations in file size SERVER-63432 @@ -7193,6 +6661,18 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sig build_variants: *mongocryptd_variants + # Put the source tarball signature + - command: s3.put + params: + aws_secret: ${aws_secret} + local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig + aws_key: ${aws_key} + bucket: build-push-testing + permissions: public-read + content_type: ${content_type|application/gzip} + remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sig + build_variants: [rhel70, windows] + # Put the debug symbols signature - command: s3.put params: @@ -7212,7 +6692,7 @@ tasks: aws_secret: ${aws_secret} permissions: public-read build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi bucket: build-push-testing content_type: application/x-msi remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi @@ -7240,6 +6720,18 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1 build_variants: *mongocryptd_variants + # Put the source tarball sha1 + - command: s3.put + params: + aws_secret: ${aws_secret} + local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1 + aws_key: ${aws_key} + permissions: public-read + bucket: build-push-testing + content_type: text/plain + remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha1 + build_variants: [rhel70, windows] + # Put the debug symbols sha1 - command: s3.put params: @@ -7259,7 +6751,7 @@ tasks: aws_secret: ${aws_secret} permissions: public-read build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha1 + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1 bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha1 @@ -7287,6 +6779,18 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha256 build_variants: *mongocryptd_variants + # Put the source tarball sha256 + - command: s3.put + params: + aws_secret: ${aws_secret} + local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256 + permissions: public-read + aws_key: ${aws_key} + bucket: build-push-testing + content_type: text/plain + remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha256 + build_variants: [rhel70, windows] + # Put the debug symbols sha256 - command: s3.put params: @@ -7305,7 +6809,7 @@ tasks: aws_key: ${aws_key} aws_secret: ${aws_secret} build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha256 + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256 bucket: build-push-testing permissions: public-read remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha256 @@ -7334,6 +6838,18 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.md5 build_variants: *mongocryptd_variants + # Put the source tarball md5 + - command: s3.put + params: + aws_secret: ${aws_secret} + local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5 + aws_key: ${aws_key} + bucket: build-push-testing + permissions: public-read + content_type: text/plain + remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.md5 + build_variants: [rhel70, windows] + # Put the debug symbols md5 - command: s3.put params: @@ -7352,7 +6868,7 @@ tasks: aws_key: ${aws_key} aws_secret: ${aws_secret} build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.md5 + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5 bucket: build-push-testing permissions: public-read content_type: text/plain @@ -7377,6 +6893,11 @@ tasks: 'destination': {'path': '${push_path}/mh-${push_name}-${push_arch}-${suffix}.${ext|tgz}', 'bucket': '${push_bucket}'}, 'build_variants': *mh_variants} + #Source tarball + - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}', 'bucket': 'build-push-testing'}, + 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}', 'bucket': '${push_bucket}'}, + 'build_variants': ['rhel70', 'windows']} + #MSI (Windows only) - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi', 'bucket': '${push_bucket}'}, @@ -7391,6 +6912,11 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} + #Source tarball signature + - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sig', 'bucket': 'build-push-testing'}, + 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig', 'bucket': '${push_bucket}'}, + 'build_variants': ['rhel70', 'windows']} + #SHA1 for binaries - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}} @@ -7400,6 +6926,11 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} + #SHA1 for source tarball + - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha1', 'bucket': 'build-push-testing'}, + 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1', 'bucket': '${push_bucket}'}, + 'build_variants': ['rhel70', 'windows']} + #SHA1 for MSI - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1', 'bucket': '${push_bucket}'}, @@ -7414,6 +6945,11 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} + #SHA256 for source tarball + - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha256', 'bucket': 'build-push-testing'}, + 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256', 'bucket': '${push_bucket}'}, + 'build_variants': ['rhel70', 'windows']} + #SHA256 for MSI files - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256', 'bucket': '${push_bucket}'}, @@ -7428,6 +6964,11 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} + #MD5 for source tarball + - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.md5', 'bucket': 'build-push-testing'}, + 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5', 'bucket': '${push_bucket}'}, + 'build_variants': ['rhel70', 'windows']} + #MD5 for MSIs - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5', 'bucket': '${push_bucket}'}, @@ -7458,7 +6999,6 @@ tasks: 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}} - name: crypt_push - run_on: rhel8.7-small tags: ["publish_crypt"] patchable: false stepback: false @@ -7476,8 +7016,6 @@ tasks: - func: "upload pip requirements" - func: "get buildnumber" - func: "f_expansions_write" - - func: "get and apply version expansions" - - func: "f_expansions_write" - func: "generate compile expansions" - func: "apply compile expansions" - func: "f_expansions_write" @@ -7495,19 +7033,13 @@ tasks: aws_key_remote: ${repo_aws_key} aws_secret_remote: ${repo_aws_secret} - func: "f_expansions_write" - # login to container registry - - command: shell.exec - params: - shell: bash - silent: true - script: | - set -oe - echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry} + - func: "set up notary client credentials" - command: subprocess.exec + type: test params: binary: bash args: - - "./src/evergreen/garasign_gpg_crypt_sign.sh" + - "./src/evergreen/notary_client_crypt_run.sh" # Put the crypt tarball/zipfile - command: s3.put params: @@ -7595,15 +7127,6 @@ tasks: resmoke_jobs_max: 1 - <<: *task_template - name: search_pinned_connections_auth - tags: [] - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_jobs_max: 1 - -- <<: *task_template name: search_ssl tags: [] commands: @@ -7613,33 +7136,6 @@ tasks: resmoke_jobs_max: 1 - <<: *task_template - name: vector_search - tags: [] - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_jobs_max: 1 - -- <<: *task_template - name: vector_search_auth - tags: [] - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_jobs_max: 1 - -- <<: *task_template - name: vector_search_ssl - tags: [] - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_jobs_max: 1 - -- <<: *task_template name: cqf tags: [] commands: @@ -7722,6 +7218,13 @@ tasks: args: - "./src/evergreen/todos_check.sh" +- <<: *task_template + name: mqlrun + tags: [] + commands: + - func: "do setup" + - func: "run tests" + - name: check_feature_flag_tags tags: [] patch_only: true @@ -7861,26 +7364,6 @@ task_groups: display_name: build.ninja - <<: *compile_task_group_template - name: compile_ninja_default_profile_TG - tasks: - - compile_ninja_default_profile - -- <<: *compile_task_group_template - name: compile_ninja_opt_profile_TG - tasks: - - compile_ninja_opt_profile - -- <<: *compile_task_group_template - name: compile_ninja_san_profile_TG - tasks: - - compile_ninja_san_profile - -- <<: *compile_task_group_template - name: compile_ninja_fast_profile_TG - tasks: - - compile_ninja_fast_profile - -- <<: *compile_task_group_template name: server_discovery_and_monitoring_json_test_TG tasks: - server_discovery_and_monitoring_json_test @@ -7909,28 +7392,23 @@ task_groups: - compile_dist_test - archive_dist_test - archive_dist_test_debug + - compile_unittests + - run_unittests - compile_dbtest - run_dbtest - archive_dbtest - - compile_all_but_not_unittests + - compile_all - package -# SERVER-76006 -# This is a compile stream meant for non-cached and/or underpowered systems. -# It joins most of the compile tasks together under a single host spread out -# across different tasks. - <<: *compile_task_group_template - name: small_compile_test_and_package_serial_TG + name: compile_and_test_TG tasks: - - compile_dist_test_half - compile_dist_test - - archive_dist_test - - archive_dist_test_debug + - compile_unittests + - run_unittests - compile_dbtest - run_dbtest - - archive_dbtest - - package - + - compile_all # These `parallel` task groups are only appropriate for builders that # use --link-model=dynamic, and have scons_cache_scope: shared and @@ -7948,17 +7426,14 @@ task_groups: - determine_patch_tests - archive_dist_test - archive_dist_test_debug - - compile_all_but_not_unittests + - compile_all - package - <<: *compile_task_group_template name: compile_test_and_package_parallel_unittest_stream_TG - max_hosts: -1 tasks: - - compile_and_run_unittests_first_quarter - - compile_and_run_unittests_second_quarter - - compile_and_run_unittests_third_quarter - - compile_and_run_unittests_fourth_quarter + - compile_unittests + - run_unittests #- <<: *compile_task_group_template # name: compile_test_and_package_parallel_unittest_stream_with_recording_TG @@ -7973,25 +7448,6 @@ task_groups: - run_dbtest - archive_dbtest -- <<: *compile_task_group_template - name: compile_integration_and_test_parallel_stream_TG - tasks: - - compile_integration_test - - integration_tests_standalone - - integration_tests_standalone_audit - - integration_tests_replset - - integration_tests_replset_ssl_auth - - integration_tests_sharded - -- <<: *compile_task_group_template - name: compile_integration_and_test_no_audit_parallel_stream_TG - tasks: - - compile_integration_test - - integration_tests_standalone - - integration_tests_replset - - integration_tests_replset_ssl_auth - - integration_tests_sharded - - name: clang_tidy_TG setup_group_can_fail_task: true setup_group: @@ -8083,19 +7539,18 @@ task_groups: - "embedded_sdk_s3_put_latest" - "embedded_sdk_tests_s3_put_latest" -- <<: *stitch_support_task_group_template_community_only - name: stitch_support_lib_build_and_archive_community_only +- <<: *stitch_support_task_group_template + name: stitch_support_lib_build_and_archive tags: ["stitch"] tasks: - - "stitch_support_create_lib_community_only" - -- <<: *stitch_support_task_group_template_community_only - name: stitch_support_lib_build_and_test_community_only + - "stitch_support_create_lib" +- <<: *stitch_support_task_group_template + name: stitch_support_lib_build_and_test tags: ["stitch"] max_hosts: 1 tasks: - - "stitch_support_install_tests_community_only" - - "stitch_support_run_tests_community_only" + - "stitch_support_install_tests" + - "stitch_support_run_tests" - name: crypt_build setup_task: diff --git a/etc/evergreen_yml_components/variants/atlas.yml b/etc/evergreen_yml_components/variants/atlas.yml index e032fb38e10..982aca93761 100644 --- a/etc/evergreen_yml_components/variants/atlas.yml +++ b/etc/evergreen_yml_components/variants/atlas.yml @@ -4,6 +4,8 @@ buildvariants: - name: enterprise-rhel-70-64-bit display_name: "Enterprise RHEL 7.0" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel70-small expansions: @@ -22,17 +24,13 @@ buildvariants: packager_distro: rhel70 repo_edition: enterprise scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - - rhel70-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel70-large + - rhel70 - name: compile_build_tools_next_TG distros: - - rhel70-large + - rhel70 - name: .aggfuzzer - name: audit - name: auth_audit_gen @@ -64,17 +62,19 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: sharding_auth_audit_gen - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt - name: unittest_shell_hang_analyzer_gen - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl + - name: test_packages + distros: + - ubuntu2004-package - name: selinux_rhel7_enterprise - name: .publish + distros: + - rhel70-small - name: generate_buildid_to_debug_symbols_mapping diff --git a/etc/evergreen_yml_components/variants/ibm.yml b/etc/evergreen_yml_components/variants/ibm.yml index 9feca21fad5..e31896a58ce 100644 --- a/etc/evergreen_yml_components/variants/ibm.yml +++ b/etc/evergreen_yml_components/variants/ibm.yml @@ -3,6 +3,8 @@ buildvariants: - name: enterprise-rhel-81-ppc64le display_name: Enterprise RHEL 8.1 PPC64LE + modules: + - enterprise run_on: - rhel81-power8-small cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -25,77 +27,18 @@ buildvariants: multiversion_platform: rhel81 multiversion_architecture: ppc64le multiversion_edition: enterprise - test_flags: >- - --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer,ppc64le_incompatible tasks: - name: compile_test_and_package_serial_TG distros: - rhel81-power8-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel81-power8-large - - name: compile_integration_and_test_no_audit_parallel_stream_TG - distros: - - rhel81-power8-large - name: .aggregation .common - name: audit - name: .auth !.multiversion !.jscore - name: .misc_js - name: .encrypt - - name: .jscore .common !.auth - - name: .read_write_concern - - name: replica_sets_gen - - name: .replica_sets .common - - name: sasl - - name: .sharding .jscore !.wo_snapshot !.multi_stmt - - name: .sharding .common !.multiversion - - name: snmp - - name: .crypt - - name: .publish_crypt - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-rhel-71-ppc64le - display_name: Enterprise RHEL 7.1 PPC64LE - run_on: - - rhel71-power8-small - cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter - stepback: false - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - # We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down - compile_flags: --ssl MONGO_DISTMOD=rhel71 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - resmoke_jobs_factor: 0.25 - has_packages: true - packager_script: packager_enterprise.py - packager_arch: ppc64le - packager_distro: rhel71 - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: ppc64le-enterprise-rhel71 - repo_edition: enterprise - multiversion_platform: rhel71 - multiversion_architecture: ppc64le - multiversion_edition: enterprise - test_flags: >- - --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer,ppc64le_incompatible - tasks: - - name: compile_test_and_package_serial_TG - distros: - - rhel71-power8-large - - name: compile_test_and_package_parallel_unittest_stream_TG + - name: .integration !.audit distros: - - rhel71-power8-large - - name: compile_integration_and_test_no_audit_parallel_stream_TG - distros: - - rhel71-power8-large - - name: .aggregation .common - - name: audit - - name: .auth !.multiversion !.jscore - - name: .misc_js - - name: .encrypt + - rhel81-power8-large - name: .jscore .common !.auth - name: .read_write_concern - name: replica_sets_gen @@ -104,16 +47,23 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.multiversion - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt + distros: + - rhel70-small - name: .publish + distros: + - rhel70-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-rhel-72-s390x display_name: Enterprise RHEL 7.2 s390x + modules: + - enterprise run_on: - rhel72-zseries-test - cron: "0 4 * * 0" + cron: "0 0 1 1 *" # Every year at Jan 1st. 0000 UTC stepback: false expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug @@ -133,16 +83,8 @@ buildvariants: multiversion_platform: rhel72 multiversion_architecture: s390x multiversion_edition: enterprise - test_flags: >- - --excludeWithAnyTags=incompatible_with_s390x,requires_increased_memlock_limits,requires_latch_analyzer tasks: - - name: small_compile_test_and_package_serial_TG - distros: - - rhel72-zseries-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel72-zseries-build - - name: compile_integration_and_test_no_audit_parallel_stream_TG + - name: compile_test_and_package_serial_TG distros: - rhel72-zseries-build - name: .aggregation .common @@ -150,6 +92,9 @@ buildvariants: - name: .auth !.multiversion !.jscore - name: .misc_js - name: .encrypt + - name: .integration !.audit + distros: + - rhel72-zseries-build - name: .jscore .common !.auth - name: .read_write_concern - name: replica_sets_gen @@ -158,16 +103,23 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.multiversion - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt + distros: + - rhel70-small - name: .publish + distros: + - rhel70-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-rhel-83-s390x display_name: Enterprise RHEL 8.3 s390x + modules: + - enterprise run_on: - rhel83-zseries-small - cron: "0 4 * * 0" + cron: "0 0 1 1 *" # Every year at Jan 1st. 0000 UTC stepback: false expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug @@ -187,20 +139,18 @@ buildvariants: multiversion_platform: rhel83 multiversion_architecture: s390x multiversion_edition: enterprise - test_flags: >- - --excludeWithAnyTags=incompatible_with_s390x,requires_increased_memlock_limits,requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - rhel83-zseries-large - - name: compile_integration_and_test_no_audit_parallel_stream_TG - distros: - - rhel83-zseries-large - name: .aggregation .common - name: audit - name: .auth !.multiversion !.jscore - name: .misc_js - name: .encrypt + - name: .integration !.audit + distros: + - rhel83-zseries-large - name: .jscore .common !.auth - name: .read_write_concern - name: replica_sets_gen @@ -209,7 +159,12 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.multiversion - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt + distros: + - rhel80-small - name: .publish + distros: + - rhel80-small - name: generate_buildid_to_debug_symbols_mapping diff --git a/etc/evergreen_yml_components/variants/in_memory.yml b/etc/evergreen_yml_components/variants/in_memory.yml index 9f3797c8e7b..9cbd9a9065f 100644 --- a/etc/evergreen_yml_components/variants/in_memory.yml +++ b/etc/evergreen_yml_components/variants/in_memory.yml @@ -1,15 +1,17 @@ # InMemory build variants that need to be run on LTS but not Rapid releases. buildvariants: -- name: enterprise-rhel-8-64-bit-inmem - display_name: Enterprise RHEL 8 (inMemory) +- name: enterprise-rhel-80-64-bit-inmem + display_name: Enterprise RHEL 8.0 (inMemory) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel80 multiversion_edition: enterprise scons_cache_scope: shared @@ -18,12 +20,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-xlarge - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel80-xlarge - - name: compile_integration_and_test_no_audit_parallel_stream_TG - distros: - - rhel80-xlarge - name: .aggfuzzer .common - name: .aggregation !.unwind !.encrypt - name: audit @@ -36,6 +32,9 @@ buildvariants: distros: - rhel80-medium # Some workloads require a lot of memory, use a bigger machine for this suite. - name: initial_sync_fuzzer_gen + - name: .integration !.audit + distros: + - rhel80-medium - name: .jscore .common !.decimal - name: jsCore_txns_large_txns_format - name: .jstestfuzz !.initsync @@ -68,13 +67,15 @@ buildvariants: - name: enterprise-windows-inmem display_name: Enterprise Windows (inMemory) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - windows-vsCurrent-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug msi exe: ".exe" content_type: application/zip - compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --use-diagnostic-latches=on + compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 num_scons_link_jobs_available: 0.25 python: '/cygdrive/c/python/python37/python.exe' test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,incompatible_with_windows_tls @@ -87,9 +88,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-large - name: .aggfuzzer .common - name: audit - name: auth_audit_gen @@ -103,7 +101,7 @@ buildvariants: - name: .jstestfuzz .common !.flow_control # Flow control jstestfuzz take longer. - name: .read_write_concern .linearize !.durable_history - name: replica_sets_auth_gen - - name: replica_sets_jscore_passthrough_gen + - name: replica_sets_jscore_passthrough - name: replica_sets_multi_stmt_txn_jscore_passthrough - name: sasl - name: .sharding .txns diff --git a/etc/evergreen_yml_components/variants/misc_release.yml b/etc/evergreen_yml_components/variants/misc_release.yml index ce385594026..7d01683ecfb 100644 --- a/etc/evergreen_yml_components/variants/misc_release.yml +++ b/etc/evergreen_yml_components/variants/misc_release.yml @@ -1,19 +1,127 @@ # All remaining release variants not included in other files. buildvariants: +- name: amazon + display_name: Amazon Linux + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + run_on: + - amazon1-2018-test + expansions: + push_path: linux + push_bucket: downloads.mongodb.org + push_name: linux + push_arch: x86_64-amazon + compile_flags: --ssl MONGO_DISTMOD=amazon -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux + multiversion_platform: amazon + multiversion_edition: targeted + has_packages: true + packager_script: packager.py + packager_arch: x86_64 + packager_distro: amazon + repo_edition: org + scons_cache_scope: shared + virtualenv: /opt/mongodbtoolchain/v3/bin/virtualenv + large_distro_name: amazon1-2018-build + tasks: + - name: compile_test_and_package_serial_TG + distros: + - amazon1-2018-build + - name: .aggfuzzer .common + - name: aggregation + - name: .auth !.audit !.multiversion + - name: causally_consistent_jscore_txns_passthrough + - name: .misc_js + - name: .concurrency .common + - name: concurrency_replication_causal_consistency_gen + - name: disk_wiredtiger + - name: free_monitoring + - name: .jscore .common + - name: .jstestfuzz .common + - name: libunwind_tests + - name: .logical_session_cache .one_sec + - name: multiversion_gen + - name: replica_sets_gen + - name: .replica_sets .common + - name: .sharding .jscore !.wo_snapshot !.multi_stmt + - name: .sharding .common !.csrs + - name: .sharding .txns + - name: .ssl + - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package + - name: generate_buildid_to_debug_symbols_mapping + +- name: enterprise-linux-64-amazon-ami + display_name: "Enterprise Amazon Linux" + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise + run_on: + - amazon1-2018-test + expansions: + additional_package_targets: archive-mongocryptd archive-mongocryptd-debug + push_path: linux + push_bucket: downloads.10gen.com + push_name: linux + push_arch: x86_64-enterprise-amzn64 + compile_flags: --ssl MONGO_DISTMOD=amzn64 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" + test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux + multiversion_platform: amzn64 + multiversion_edition: enterprise + has_packages: true + packager_script: packager_enterprise.py + packager_arch: x86_64 + packager_distro: amazon + repo_edition: enterprise + scons_cache_scope: shared + virtualenv: /opt/mongodbtoolchain/v3/bin/virtualenv + tasks: + - name: compile_test_and_package_serial_TG + distros: + - amazon1-2018-build + - name: .aggfuzzer .common + - name: aggregation + - name: .auth !.multiversion + - name: audit + - name: causally_consistent_jscore_txns_passthrough + - name: .encrypt !.aggregation + - name: .jscore .common + - name: .jstestfuzz .common + - name: libunwind_tests + - name: .logical_session_cache .one_sec + - name: noPassthrough_gen + - name: noPassthroughWithMongod_gen + - name: .powercycle + - name: .replica_sets .common + - name: sasl + - name: serial_run + - name: .sharding .jscore !.wo_snapshot !.multi_stmt + - name: .sharding .txns + - name: slow1_gen + - name: snmp + - name: .stitch + - name: .crypt + - name: test_packages + distros: + - ubuntu2004-package + - name: generate_buildid_to_debug_symbols_mapping + - name: amazon2 display_name: Amazon Linux 2 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - amazon2-latest-small + - amazon2-test expansions: test_flags: >- - --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_latch_analyzer --enableEnterpriseTests=off + --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux push_path: linux push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-amazon2 - compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: amazon2 multiversion_edition: targeted has_packages: true @@ -22,14 +130,11 @@ buildvariants: packager_distro: amazon2 repo_edition: org scons_cache_scope: shared - large_distro_name: amazon2-latest-large + large_distro_name: amazon2-large tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-latest-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - amazon2-latest-large + - amazon2-build - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -38,6 +143,7 @@ buildvariants: - name: .concurrency .common - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common - name: .jstestfuzz .common - name: libunwind_tests @@ -50,19 +156,26 @@ buildvariants: - name: .sharding .txns - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - amazon2-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-amazon2 display_name: "Enterprise Amazon Linux 2" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - - amazon2-latest-small + - amazon2-test expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug # TODO BUILD-13887 should fix uses_pykmip incompatibility. test_flags: >- - --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,uses_pykmip,requires_latch_analyzer + --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,uses_pykmip push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -80,13 +193,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-latest-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - amazon2-latest-large - - name: compile_integration_and_test_parallel_stream_TG - distros: - - amazon2-latest-large + - amazon2-build - name: .aggfuzzer .common - name: aggregation - name: audit @@ -106,46 +213,50 @@ buildvariants: - name: .sharding .txns !.csrs - name: slow1_gen - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - amazon2-small - name: generate_buildid_to_debug_symbols_mapping - name: amazon2-arm64 display_name: Amazon Linux 2 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - amazon2-arm64-latest-small + - amazon2-arm64-small expansions: push_path: linux push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-amazon2 - compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_latch_analyzer --enableEnterpriseTests=off + compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux has_packages: true packager_script: packager.py packager_arch: aarch64 packager_distro: amazon2 repo_edition: org scons_cache_scope: shared - large_distro_name: amazon2-arm64-latest-large + large_distro_name: amazon2-arm64-large tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-arm64-latest-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough - name: .misc_js - name: .concurrency .common distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -155,14 +266,21 @@ buildvariants: - name: .sharding .common !.csrs !.multiversion - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - rhel80-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-amazon2-arm64 display_name: "Enterprise Amazon Linux 2 arm64" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - - amazon2-arm64-latest-small + - amazon2-arm64-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: linux @@ -172,7 +290,7 @@ buildvariants: compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" # TODO BUILD-13887 should fix uses_pykmip incompatibility. - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,uses_pykmip,requires_v4_0,requires_latch_analyzer + test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,uses_pykmip,requires_v4_0 has_packages: true multiversion_platform: amazon2 multiversion_edition: enterprise @@ -187,13 +305,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-arm64-latest-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - amazon2-arm64-latest-large - - name: compile_integration_and_test_parallel_stream_TG - distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: test_api_version_compatibility - name: .aggfuzzer - name: .aggregation @@ -206,57 +318,56 @@ buildvariants: - name: .concurrency !.large !.ubsan !.no_txns !.debug_only - name: .concurrency .large !.ubsan !.no_txns !.debug_only distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: config_fuzzer_concurrency - name: config_fuzzer_simulate_crash_concurrency_replication distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: config_fuzzer_concurrency_replication distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: config_fuzzer_jsCore - name: config_fuzzer_replica_sets_jscore_passthrough distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: disk_wiredtiger - name: .encrypt - name: idl_tests - name: initial_sync_fuzzer_gen + - name: .integration + distros: + - amazon2-arm64-large - name: jsCore distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: .jscore .common !jsCore - name: jsCore_minimum_batch_size - name: jsCore_txns_large_txns_format - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests + - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: .query_fuzzer - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: .read_write_concern .large distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: .read_write_concern !.large - name: .replica_sets !.encrypt !.auth distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: replica_sets_api_version_jscore_passthrough_gen - name: replica_sets_reconfig_jscore_passthrough_gen - name: retryable_writes_jscore_passthrough_gen - name: retryable_writes_jscore_stepdown_passthrough_gen distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: .read_only - name: .rollbackfuzzer - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt @@ -264,218 +375,27 @@ buildvariants: - name: .sharding .txns - name: .sharding .common - name: snmp + - name: .stitch - name: .crypt distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: .publish_crypt + distros: + - amazon2-arm64-large - name: secondary_reads_passthrough_gen - name: server_discovery_and_monitoring_json_test_TG - name: .serverless distros: - - amazon2-arm64-latest-large + - amazon2-arm64-large - name: server_selection_json_test_TG distros: - - amazon2-arm64-latest-large - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - -- name: amazon2023 - display_name: Amazon Linux 2023 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - amazon2023.0-small - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: x86_64-amazon2023 - compile_flags: --ssl MONGO_DISTMOD=amazon2023 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --modules= - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source,requires_latch_analyzer --enableEnterpriseTests=off - has_packages: true - packager_script: packager.py - packager_arch: x86_64 - packager_distro: amazon2023 - repo_edition: org - scons_cache_scope: shared - large_distro_name: amazon2023.0-large - compile_variant: amazon2023 - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2023.0-large - - name: .aggfuzzer .common !.multiversion - - name: aggregation - - name: .auth !.audit !.multiversion - - name: causally_consistent_jscore_txns_passthrough - - name: .misc_js !.parallel - - name: .concurrency .common - distros: - - amazon2023.0-large - - name: concurrency_replication_causal_consistency_gen - - name: disk_wiredtiger - - name: .jscore .common - - name: .jstestfuzz .common - - name: libunwind_tests - - name: .logical_session_cache .one_sec - - name: replica_sets_gen - - name: .replica_sets .common - - name: .sharding .jscore !.wo_snapshot !.multi_stmt - - name: .sharding .common !.csrs !.multiversion - - name: .sharding .txns - - name: .ssl - - name: .stitch - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-amazon2023 - display_name: "Enterprise Amazon Linux 2023" - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - amazon2023.0-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: x86_64-enterprise-amazon2023 - compile_flags: --ssl MONGO_DISTMOD=amazon2023 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --enterprise-features="audit,encryptdb,fcbis,fips,fle,hot_backups,inmemory,kerberos,vector_search,ldap,live_import,mongohouse,queryable,sasl,search,serverless" - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - # TODO BUILD-13887 should fix uses_pykmip incompatibility. - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,uses_pykmip,requires_v4_0,requires_external_data_source,requires_latch_analyzer - has_packages: true - multiversion_platform: amazon2023 - multiversion_edition: enterprise - multiversion_architecture: x86_64 - packager_script: packager_enterprise.py - packager_arch: x86_64 - packager_distro: amazon2023 - repo_edition: enterprise - scons_cache_scope: shared - compile_variant: enterprise-amazon2023 - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2023.0-large - - name: .aggfuzzer .common !.multiversion - - name: aggregation - - name: audit - - name: .auth !.multiversion - - name: causally_consistent_jscore_txns_passthrough - - name: .encrypt !.aggregation - - name: .jscore .common - - name: .jstestfuzz .common - - name: libunwind_tests - - name: .logical_session_cache .one_sec - - name: noPassthrough_gen - - name: noPassthroughWithMongod_gen - - name: .replica_sets .common - - name: sasl - - name: serial_run - - name: .sharding .jscore !.wo_snapshot !.multi_stmt - - name: .sharding .txns !.csrs - - name: slow1_gen - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: amazon2023-arm64 - display_name: Amazon Linux 2023 arm64 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - amazon2023.0-arm64-small - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: aarch64-amazon2023 - compile_flags: --ssl MONGO_DISTMOD=amazon2023 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --modules= - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source,requires_latch_analyzer --enableEnterpriseTests=off - has_packages: true - packager_script: packager.py - packager_arch: aarch64 - packager_distro: amazon2023 - repo_edition: org - scons_cache_scope: shared - large_distro_name: amazon2023.0-arm64-large - compile_variant: amazon2023-arm64 - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2023.0-arm64-large - - name: aggregation - - name: .auth !.audit !.multiversion - - name: causally_consistent_jscore_txns_passthrough - - name: .misc_js !.parallel - - name: .concurrency .common + - amazon2-arm64-large + - name: test_packages distros: - - amazon2023.0-arm64-large - - name: concurrency_replication_causal_consistency_gen - - name: disk_wiredtiger - - name: .jscore .common - - name: .jstestfuzz .common - - name: .logical_session_cache .one_sec - - name: replica_sets_gen - - name: .replica_sets .common - - name: .sharding .txns - - name: .sharding .common !.csrs !.multiversion - - name: .ssl - - name: .stitch + - ubuntu2004-package - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-amazon2023-arm64 - display_name: "Enterprise Amazon Linux 2023 arm64" - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - amazon2023.0-arm64-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: aarch64-enterprise-amazon2023 - compile_flags: --ssl MONGO_DISTMOD=amazon2023 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --enterprise-features="audit,encryptdb,fcbis,fips,fle,hot_backups,inmemory,kerberos,vector_search,ldap,live_import,mongohouse,queryable,sasl,search,serverless" - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - # TODO BUILD-13887 should fix uses_pykmip incompatibility. - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,uses_pykmip,requires_v4_0,requires_external_data_source,requires_latch_analyzer - has_packages: true - multiversion_platform: amazon2023 - multiversion_edition: enterprise - multiversion_architecture: aarch64 - packager_script: packager_enterprise.py - packager_arch: aarch64 - packager_distro: amazon2023 - repo_edition: enterprise - scons_cache_scope: shared - compile_variant: enterprise-amazon2023-arm64 - tasks: - - name: compile_test_and_package_serial_TG distros: - - amazon2023.0-arm64-large - - name: .aggfuzzer !.multiversion - - name: audit - - name: auth_audit_gen - - name: auth_gen - - name: causally_consistent_jscore_txns_passthrough - - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore - - name: external_auth - - name: external_auth_aws - - name: .jscore .common !.decimal !.sharding - - name: jsCore_txns_large_txns_format - - name: .jstestfuzz .common - - name: .logical_session_cache .one_sec - - name: replica_sets_auth_gen - - name: replica_sets_jscore_passthrough - - name: .replica_sets .multi_oplog - - name: sasl - - name: search - - name: search_auth - - name: search_ssl - - name: sharding_auth_audit_gen - - name: sharding_auth_gen - - name: .publish + - rhel80-small - name: generate_buildid_to_debug_symbols_mapping - name: debian10 @@ -488,8 +408,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-debian10 - compile_flags: --ssl MONGO_DISTMOD=debian10 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + compile_flags: --ssl MONGO_DISTMOD=debian10 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: linux_x86_64 multiversion_edition: base multiversion_platform_42_or_later: debian10 @@ -505,9 +424,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - debian10-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - debian10-build - name: .aggfuzzer .common - name: aggregation - name: aggregation_auth @@ -517,6 +433,7 @@ buildvariants: - name: .concurrency .common - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common !.decimal - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -527,12 +444,19 @@ buildvariants: - name: .sharding .common !.csrs - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - debian10-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-debian10-64 display_name: Enterprise Debian 10 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - debian10-test expansions: @@ -554,18 +478,17 @@ buildvariants: repo_edition: enterprise scons_cache_scope: shared large_distro_name: debian10-build - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - debian10-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - debian10-build - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore + - name: .jepsen_docker + distros: + - debian10-large - name: .jscore .common !.decimal !.sharding - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -574,9 +497,15 @@ buildvariants: - name: sasl - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - debian10-small - name: generate_buildid_to_debug_symbols_mapping - name: debian11 @@ -589,14 +518,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-debian11 - compile_flags: --ssl MONGO_DISTMOD=debian11 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + compile_flags: --ssl MONGO_DISTMOD=debian11 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: linux_x86_64 multiversion_edition: base - multiversion_platform_42_or_later: debian10 + multiversion_platform_42_or_later: debian11 multiversion_edition_42_or_later: targeted - multiversion_platform_last_lts: debian11 - multiversion_edition_last_lts: targeted has_packages: true packager_script: packager.py packager_arch: x86_64 @@ -608,9 +534,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - debian11-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - debian11-build - name: .aggfuzzer .common - name: aggregation - name: aggregation_auth @@ -620,6 +543,7 @@ buildvariants: - name: .concurrency .common - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common !.decimal - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -630,12 +554,19 @@ buildvariants: - name: .sharding .common !.csrs - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - debian11-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-debian11-64 display_name: Enterprise Debian 11 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - debian11-small expansions: @@ -648,10 +579,8 @@ buildvariants: crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: linux_x86_64 multiversion_edition: base - multiversion_platform_42_or_later: debian10 + multiversion_platform_42_or_later: debian11 multiversion_edition_42_or_later: enterprise - multiversion_platform_last_lts: debian11 - multiversion_edition_last_lts: enterprise has_packages: true packager_script: packager_enterprise.py packager_arch: x86_64 @@ -659,18 +588,17 @@ buildvariants: repo_edition: enterprise scons_cache_scope: shared large_distro_name: debian11-large - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - debian11-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - debian11-large - name: .aggfuzzer .common !.multiversion - name: audit - name: causally_consistent_jscore_txns_passthrough - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore + - name: .jepsen_docker + distros: + - debian11-large - name: .jscore .common !.decimal !.sharding - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -679,9 +607,15 @@ buildvariants: - name: sasl - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - debian11-small - name: generate_buildid_to_debug_symbols_mapping - name: rhel70 @@ -694,8 +628,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-rhel70 - compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel70 multiversion_edition: targeted has_packages: true @@ -708,10 +641,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG distros: - - rhel70-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel70-large + - rhel70 - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -721,6 +651,7 @@ buildvariants: - rhel70 - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -731,8 +662,13 @@ buildvariants: - name: .sharding .common !.csrs - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: selinux_rhel7_org - name: .publish + distros: + - rhel70-small - name: generate_buildid_to_debug_symbols_mapping # This variant compiles on RHEL 7.0 and runs tests on RHEL 7.6 @@ -742,8 +678,7 @@ buildvariants: run_on: - rhel76-test expansions: - compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars tasks: - name: compile_and_archive_dist_test_then_package_TG distros: @@ -760,11 +695,11 @@ buildvariants: expansions: resmoke_jobs_factor: 1 disable_shared_scons_cache: true - compile_flags: MONGO_DISTMOD=rhel80 --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + compile_flags: MONGO_DISTMOD=rhel80 --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars tooltags: "" build_mongoreplay: true test_flags: >- - --excludeWithAnyTags=requires_os_access,requires_latch_analyzer --enableEnterpriseTests=off + --excludeWithAnyTags=requires_os_access tasks: - name: compile_and_archive_dist_test_then_package_TG distros: @@ -773,8 +708,8 @@ buildvariants: - name: sharding_gen - name: replica_sets_gen -- name: rhel8 - display_name: RHEL 8 +- name: rhel80 + display_name: RHEL 8.0 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -782,9 +717,8 @@ buildvariants: push_path: linux push_bucket: downloads.mongodb.org push_name: linux - push_arch: x86_64-rhel8 - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + push_arch: x86_64-rhel80 + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel80 multiversion_edition: targeted has_packages: true @@ -798,9 +732,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel80-build - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -810,6 +741,7 @@ buildvariants: - rhel80-build - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -820,21 +752,27 @@ buildvariants: - name: .sharding .common !.csrs - name: .ssl - name: .stitch - - name: selinux_rhel8_org + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - rhel80-small - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel-8-64-bit - display_name: "Enterprise RHEL 8" +- name: enterprise-rhel-80-64-bit + display_name: "Enterprise RHEL 8.0" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-small - expansions: &enterprise-rhel-8-64-bit-expansions + expansions: &enterprise-rhel-80-64-bit-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux - push_arch: x86_64-enterprise-rhel8 + push_arch: x86_64-enterprise-rhel80 compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 @@ -850,17 +788,10 @@ buildvariants: target_resmoke_time: 10 max_sub_suites: 3 large_distro_name: rhel80-medium - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - rhel80-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel80-large - - name: compile_integration_and_test_parallel_stream_TG - distros: - - rhel80-large - name: .aggfuzzer - name: .aggregation - name: audit @@ -878,20 +809,20 @@ buildvariants: - name: .encrypt - name: idl_tests - name: initial_sync_fuzzer_gen + - name: .integration + distros: + - rhel80-medium - name: .jscore .common - name: jsCore_minimum_batch_size - name: jsCore_txns_large_txns_format - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests + - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer - - name: query_stats_aggregation_passthrough - - name: query_stats_mongos_aggregation_passthrough - - name: query_stats_mongos_passthrough - - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -918,7 +849,6 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: session_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt @@ -926,6 +856,7 @@ buildvariants: - name: .sharding .txns - name: .sharding .common - name: snmp + - name: .stitch - name: .crypt distros: - rhel80-large @@ -938,15 +869,15 @@ buildvariants: - name: server_selection_json_test_TG distros: - rhel80-large + - name: test_packages + distros: + - ubuntu2004-package - name: selinux_rhel8_enterprise - name: .publish - name: generate_buildid_to_debug_symbols_mapping - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl -- name: rhel-8-arm64 - display_name: RHEL 8 arm64 +- name: rhel-82-arm64 + display_name: RHEL 8.2 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel82-arm64-small @@ -954,8 +885,8 @@ buildvariants: push_path: linux push_bucket: downloads.mongodb.org push_name: linux - push_arch: aarch64-rhel8 - compile_flags: --ssl MONGO_DISTMOD=rhel82 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + push_arch: aarch64-rhel82 + compile_flags: --ssl MONGO_DISTMOD=rhel82 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars has_packages: true packager_script: packager.py packager_arch: aarch64 @@ -963,14 +894,10 @@ buildvariants: repo_edition: org scons_cache_scope: shared large_distro_name: rhel82-arm64-large - test_flags: --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer --enableEnterpriseTests=off tasks: - name: compile_test_and_package_serial_TG distros: - rhel82-arm64-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel82-arm64-large - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -980,6 +907,7 @@ buildvariants: - rhel82-arm64-large - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -989,12 +917,19 @@ buildvariants: - name: .sharding .common !.csrs !.multiversion - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - rhel80-small - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel-8-arm64 - display_name: "Enterprise RHEL 8 arm64" +- name: enterprise-rhel-82-arm64 + display_name: "Enterprise RHEL 8.2 arm64" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel82-arm64-small expansions: @@ -1002,7 +937,7 @@ buildvariants: push_path: linux push_bucket: downloads.10gen.com push_name: linux - push_arch: aarch64-enterprise-rhel8 + push_arch: aarch64-enterprise-rhel82 compile_flags: --ssl MONGO_DISTMOD=rhel82 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" has_packages: true @@ -1011,14 +946,10 @@ buildvariants: packager_distro: rhel82 repo_edition: enterprise scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - rhel82-arm64-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel82-arm64-large - name: .aggfuzzer !.multiversion - name: audit - name: auth_audit_gen @@ -1037,254 +968,21 @@ buildvariants: - name: sasl - name: search - name: search_auth - - name: search_pinned_connections_auth - name: search_ssl - name: sharding_auth_audit_gen - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt distros: - rhel82-arm64-large - name: .publish_crypt - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - -- name: rhel90 - display_name: RHEL 9.0 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - rhel90-large - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: x86_64-rhel90 - compile_flags: --ssl MONGO_DISTMOD=rhel90 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer - multiversion_platform: rhel90 - multiversion_edition: targeted - has_packages: true - packager_script: packager.py - packager_arch: x86_64 - packager_distro: rhel90 - repo_edition: org - scons_cache_scope: shared - large_distro_name: rhel90-large - tasks: - - name: compile_test_and_package_serial_TG - distros: - - rhel90-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel90-large - - name: aggregation - - name: .auth !.audit !.multiversion - - name: causally_consistent_jscore_txns_passthrough - - name: .misc_js - - name: .concurrency .common - distros: - - rhel90-large - - name: concurrency_replication_causal_consistency_gen - - name: disk_wiredtiger - - name: .jscore .common - - name: .jstestfuzz .common !.multiversion - - name: .logical_session_cache .one_sec - - name: replica_sets_gen - - name: .replica_sets .common - - name: .sharding .txns - - name: .sharding .common !.csrs !.multiversion - - name: .ssl - - name: .stitch - - name: selinux_rhel9_org - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - - -- name: enterprise-rhel-90-64-bit - display_name: "Enterprise RHEL 9.0" - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - rhel90-small - expansions: &enterprise-rhel-90-64-bit-expansions - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: x86_64-enterprise-rhel90 - compile_flags: --ssl MONGO_DISTMOD=rhel90 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - multiversion_platform: rhel90 - multiversion_edition: enterprise - has_packages: true - packager_script: packager_enterprise.py - packager_arch: x86_64 - packager_distro: rhel90 - repo_edition: enterprise - scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer - tasks: - - name: compile_test_and_package_serial_TG - distros: - - rhel90-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel90-large - - name: compile_build_tools_next_TG - distros: - - rhel90-large - - name: .aggfuzzer !.multiversion - - name: audit - - name: auth_audit_gen - - name: auth_gen - - name: causally_consistent_jscore_txns_passthrough - - name: config_fuzzer_concurrency - - name: config_fuzzer_simulate_crash_concurrency_replication - distros: - - rhel90-large - - name: config_fuzzer_concurrency_replication - distros: - - rhel90-large - - name: config_fuzzer_jsCore - - name: config_fuzzer_replica_sets_jscore_passthrough - distros: - - rhel90-large - - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore - - name: external_auth - - name: external_auth_aws - - name: .jscore .common !.decimal !.sharding - - name: jsCore_txns_large_txns_format - - name: .jstestfuzz .common - - name: libunwind_tests - - name: .logical_session_cache .one_sec - - name: .ocsp - - name: replica_sets_auth_gen - - name: replica_sets_jscore_passthrough - - name: .replica_sets .multi_oplog - - name: sasl - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: sharding_auth_audit_gen - - name: sharding_auth_gen - - name: snmp - - name: .crypt - - name: .publish_crypt - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - - name: selinux_rhel9_enterprise - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: rhel90-arm64 - display_name: RHEL 9.0 arm64 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - rhel90-arm64-small - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: aarch64-rhel90 - compile_flags: --ssl MONGO_DISTMOD=rhel90 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer - has_packages: true - packager_script: packager.py - packager_arch: aarch64 - packager_distro: rhel90 - repo_edition: org - scons_cache_scope: shared - large_distro_name: rhel90-arm64-large - tasks: - - name: compile_test_and_package_serial_TG - distros: - - rhel90-arm64-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel90-arm64-large - - name: aggregation - - name: .auth !.audit !.multiversion - - name: causally_consistent_jscore_txns_passthrough - - name: .misc_js - - name: .concurrency .common + - name: test_packages distros: - - rhel90-arm64-large - - name: concurrency_replication_causal_consistency_gen - - name: disk_wiredtiger - - name: .jscore .common - - name: .jstestfuzz .common - - name: .logical_session_cache .one_sec - - name: replica_sets_gen - - name: .replica_sets .common - - name: .sharding .txns - - name: .sharding .common !.csrs !.multiversion - - name: .ssl - - name: .stitch + - ubuntu2004-package - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-rhel-90-arm64 - display_name: "Enterprise RHEL 9.0 arm64" - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - rhel90-arm64-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: aarch64-enterprise-rhel90 - compile_flags: --ssl MONGO_DISTMOD=rhel90 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - has_packages: true - packager_script: packager_enterprise.py - packager_arch: aarch64 - packager_distro: rhel90 - repo_edition: enterprise - scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer - tasks: - - name: compile_test_and_package_serial_TG - distros: - - rhel90-arm64-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - rhel90-arm64-large - - name: .aggfuzzer !.multiversion - - name: audit - - name: auth_audit_gen - - name: auth_gen - - name: causally_consistent_jscore_txns_passthrough - - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore - - name: external_auth - - name: external_auth_aws - - name: .jscore .common !.decimal !.sharding - - name: jsCore_txns_large_txns_format - - name: .jstestfuzz .common - - name: .logical_session_cache .one_sec - - name: replica_sets_auth_gen - - name: replica_sets_jscore_passthrough - - name: .replica_sets .multi_oplog - - name: sasl - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: sharding_auth_audit_gen - - name: sharding_auth_gen - - name: snmp - - name: .crypt distros: - - rhel90-arm64-large - - name: .publish_crypt - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - - name: .publish + - rhel80-small - name: generate_buildid_to_debug_symbols_mapping - name: suse12 @@ -1297,7 +995,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-suse12 - compile_flags: --ssl MONGO_DISTMOD=suse12 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + compile_flags: --ssl MONGO_DISTMOD=suse12 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: suse12 multiversion_edition: targeted has_packages: true @@ -1306,15 +1004,11 @@ buildvariants: packager_distro: suse12 repo_edition: org scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer --enableEnterpriseTests=off large_distro_name: suse12-sp5-large tasks: - name: compile_test_and_package_serial_TG distros: - suse12-sp5-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - suse12-sp5-large - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -1326,6 +1020,7 @@ buildvariants: - suse12-sp5-large - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common !.decimal - name: .jstestfuzz .common - name: multiversion_gen @@ -1337,11 +1032,15 @@ buildvariants: - name: .ssl - name: .stitch - name: .publish + distros: + - suse12-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-suse12-64 display_name: Enterprise SLES 12 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - suse12-sp5-small expansions: @@ -1360,15 +1059,10 @@ buildvariants: packager_distro: suse12 repo_edition: enterprise scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer - large_distro_name: suse12-sp5-large tasks: - name: compile_test_and_package_serial_TG distros: - suse12-sp5-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - suse12-sp5-large - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -1381,16 +1075,22 @@ buildvariants: - name: sasl - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt - distros: - - suse12-sp5-large - name: .publish_crypt + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - suse12-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-suse15-64 display_name: Enterprise SLES 15 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - suse15-test expansions: @@ -1407,14 +1107,10 @@ buildvariants: packager_distro: suse15 repo_edition: enterprise scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - suse15-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - suse15-build - name: .aggfuzzer .common !.multiversion - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -1427,9 +1123,12 @@ buildvariants: - name: sasl - name: sharding_auth_gen - name: snmp + - name: .stitch - name: .crypt - name: .publish_crypt - name: .publish + distros: + - suse15-small - name: generate_buildid_to_debug_symbols_mapping - name: suse15 @@ -1442,7 +1141,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-suse15 - compile_flags: --ssl MONGO_DISTMOD=suse15 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + compile_flags: --ssl MONGO_DISTMOD=suse15 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: linux_x86_64 multiversion_edition: base multiversion_platform_42_or_later: suse15 @@ -1453,15 +1152,11 @@ buildvariants: packager_distro: suse15 repo_edition: org scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer --enableEnterpriseTests=off large_distro_name: suse15-build tasks: - name: compile_test_and_package_serial_TG distros: - suse15-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - suse15-build - name: .aggfuzzer .common !.multiversion - name: aggregation - name: .auth !.audit !.multiversion @@ -1470,6 +1165,7 @@ buildvariants: - name: .concurrency .common - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common !.decimal - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -1481,6 +1177,8 @@ buildvariants: - name: .ssl - name: .stitch - name: .publish + distros: + - suse15-small - name: generate_buildid_to_debug_symbols_mapping @@ -1494,10 +1192,10 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-ubuntu1804 - compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= + compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: ubuntu1804 multiversion_edition: targeted - test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_latch_analyzer --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=requires_ocsp_stapling has_packages: true packager_script: packager.py packager_arch: x86_64 @@ -1509,9 +1207,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - ubuntu1804-build - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -1519,6 +1214,7 @@ buildvariants: - name: .concurrency .common - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jepsen distros: - ubuntu1804-build @@ -1536,12 +1232,19 @@ buildvariants: - name: watchdog_wiredtiger - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - ubuntu1804-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-ubuntu1804-64 display_name: Enterprise Ubuntu 18.04 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - ubuntu1804-test stepback: false @@ -1555,7 +1258,7 @@ buildvariants: crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: ubuntu1804 multiversion_edition: enterprise - test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_latch_analyzer + test_flags: --excludeWithAnyTags=requires_ocsp_stapling has_packages: true packager_script: packager_enterprise.py packager_arch: x86_64 @@ -1567,9 +1270,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - ubuntu1804-build - name: compile_build_tools_next_TG distros: - ubuntu1804-build @@ -1602,12 +1302,19 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .watchdog + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - ubuntu1804-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-ubuntu1804-arm64 display_name: Enterprise Ubuntu 18.04 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - ubuntu1804-arm64-build expansions: @@ -1618,7 +1325,7 @@ buildvariants: push_arch: aarch64-enterprise-ubuntu1804 compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_latch_analyzer + test_flags: --excludeWithAnyTags=requires_ocsp_stapling resmoke_jobs_max: 4 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager_enterprise.py @@ -1632,7 +1339,6 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: aggregation - name: aggregation_wildcard_fuzzer_gen - name: .auth !.audit !.multiversion !.jscore @@ -1650,9 +1356,17 @@ buildvariants: - name: sharding_gen - name: sharding_jscore_passthrough - name: .ssl + - name: .stitch - name: .crypt - name: .publish_crypt + distros: + - ubuntu1804-test + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - ubuntu1804-test - name: generate_buildid_to_debug_symbols_mapping - name: ubuntu1804-arm64 @@ -1665,8 +1379,8 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-ubuntu1804 - compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_latch_analyzer --enableEnterpriseTests=off + compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + test_flags: --excludeWithAnyTags=requires_ocsp_stapling resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager.py @@ -1680,64 +1394,17 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG + - name: free_monitoring - name: jsCore - name: replica_sets_jscore_passthrough - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - - -- name: ubuntu2204 - display_name: Ubuntu 22.04 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - ubuntu2204-small - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: x86_64-ubuntu2204 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2204 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - multiversion_platform: ubuntu2204 - multiversion_edition: targeted - has_packages: true - packager_script: packager.py - packager_arch: x86_64 - packager_distro: ubuntu2204 - repo_edition: org - scons_cache_scope: shared - large_distro_name: ubuntu2204-large - test_flags: --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer --enableEnterpriseTests=off - tasks: - - name: compile_test_and_package_serial_TG - distros: - - ubuntu2204-large - - name: compile_test_and_package_parallel_unittest_stream_TG + - name: test_packages distros: - - ubuntu2204-large - - name: .aggfuzzer .common !.multiversion - - name: aggregation - - name: .auth !.audit !.multiversion - - name: .misc_js - - name: .concurrency .common - - name: concurrency_replication_causal_consistency_gen - - name: disk_wiredtiger - - name: .jscore .common - - name: .jstestfuzz .common - - name: libunwind_tests - - name: .logical_session_cache .one_sec - # - name: multiversion_gen - - name: replica_sets_gen - - name: replica_sets_jscore_passthrough - - name: .sharding .txns - - name: sharding_gen - - name: sharding_jscore_passthrough - - name: .ssl - - name: .stitch + - ubuntu2004-package - name: .publish + distros: + - ubuntu1804-test - name: generate_buildid_to_debug_symbols_mapping - - name: ubuntu2004 display_name: Ubuntu 20.04 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. @@ -1748,8 +1415,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-ubuntu2004 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2004 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + compile_flags: --ssl MONGO_DISTMOD=ubuntu2004 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: ubuntu2004 multiversion_edition: targeted has_packages: true @@ -1763,9 +1429,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu2004-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - ubuntu2004-large - name: .aggfuzzer .common !.multiversion - name: aggregation - name: .auth !.audit !.multiversion @@ -1773,6 +1436,7 @@ buildvariants: - name: .concurrency .common - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common - name: .jstestfuzz .common - name: libunwind_tests @@ -1785,12 +1449,19 @@ buildvariants: - name: sharding_jscore_passthrough - name: .ssl - name: .stitch + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - ubuntu2004-small - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-ubuntu2004-64 display_name: Enterprise Ubuntu 20.04 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - ubuntu2004-small stepback: false @@ -1811,14 +1482,10 @@ buildvariants: repo_edition: enterprise scons_cache_scope: shared large_distro_name: ubuntu2004-large - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG distros: - ubuntu2004-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - ubuntu2004-large - name: .crypt - name: .publish_crypt - name: .aggfuzzer .common !.multiversion @@ -1838,90 +1505,19 @@ buildvariants: - name: sasl - name: sharding_auth_gen - name: snmp - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-ubuntu2204-64 - display_name: Enterprise Ubuntu 22.04 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - ubuntu2204-small - stepback: false - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: x86_64-enterprise-ubuntu2204 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2204 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - multiversion_platform: ubuntu2204 - multiversion_edition: enterprise - has_packages: true - packager_script: packager_enterprise.py - packager_arch: x86_64 - packager_distro: ubuntu2204 - repo_edition: enterprise - scons_cache_scope: shared - large_distro_name: ubuntu2204-large - test_flags: --excludeWithAnyTags=requires_latch_analyzer - tasks: - - name: compile_test_and_package_serial_TG + - name: test_packages distros: - - ubuntu2204-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - ubuntu2204-large - - name: .crypt - - name: .publish_crypt - - name: .aggfuzzer .common !.multiversion - - name: audit - - name: causally_consistent_jscore_txns_passthrough - - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore - - name: external_auth - - name: external_auth_aws - - name: .jscore .common !.decimal !.sharding - - name: jsCore_auth - - name: .jstestfuzz .common - - name: libunwind_tests - - name: .logical_session_cache .one_sec - - name: .ocsp - - name: replica_sets_auth_gen - - name: replica_sets_jscore_passthrough - - name: sasl - - name: sharding_auth_gen - - name: snmp + - ubuntu2004-package - name: .publish + distros: + - ubuntu2004-small - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-ubuntu2204-jepsen - display_name: Jepsen Tests on Enterprise Ubuntu 22.04 - tags: ["bazel_check"] - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - ubuntu2204-small - stepback: false - expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - scons_cache_scope: shared - large_distro_name: ubuntu2204-large - compile_variant: enterprise-ubuntu2204-64 - tasks: - - name: compile_test_and_package_serial_TG - distros: - - ubuntu2204-large - - name: .jepsen_docker - distros: - - ubuntu2204-large - - name: enterprise-ubuntu2004-arm64 display_name: Enterprise Ubuntu 20.04 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - ubuntu2004-arm64-large expansions: @@ -1943,10 +1539,8 @@ buildvariants: multiversion_architecture_42_or_later: aarch64 multiversion_edition: enterprise scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: aggregation - name: aggregation_wildcard_fuzzer_gen - name: .auth !.audit !.multiversion !.jscore @@ -1964,9 +1558,15 @@ buildvariants: - name: sharding_gen - name: sharding_jscore_passthrough - name: .ssl + - name: .stitch - name: .crypt - name: .publish_crypt + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - ubuntu2004-test - name: generate_buildid_to_debug_symbols_mapping - name: ubuntu2004-arm64 @@ -1979,8 +1579,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-ubuntu2004 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2004 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer + compile_flags: --ssl MONGO_DISTMOD=ubuntu2004 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager.py @@ -1994,92 +1593,15 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - - name: jsCore - - name: replica_sets_jscore_passthrough - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - - -- name: enterprise-ubuntu2204-arm64 - display_name: Enterprise Ubuntu 22.04 arm64 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - ubuntu2204-arm64-large - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: aarch64-enterprise-ubuntu2204 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2204 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - resmoke_jobs_max: 4 # Avoid starting too many mongod's on ARM test servers - has_packages: true - packager_script: packager_enterprise.py - packager_arch: arm64 - packager_distro: ubuntu2204 - repo_edition: enterprise - multiversion_platform: ubuntu2204 - multiversion_architecture: arm64 - multiversion_architecture_42_or_later: aarch64 - multiversion_edition: enterprise - scons_cache_scope: shared - test_flags: --excludeWithAnyTags=requires_latch_analyzer - tasks: - - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - - name: aggregation - - name: aggregation_wildcard_fuzzer_gen - - name: .auth !.audit !.multiversion !.jscore - - name: causally_consistent_jscore_txns_passthrough - - name: .misc_js - - name: .concurrency .common - - name: concurrency_replication_causal_consistency_gen - - name: fle - - name: .jscore .common !.auth - - name: .jstestfuzz .common - - name: .logical_session_cache .one_sec - - name: replica_sets_gen - - name: .replica_sets .common - - name: .sharding .txns - - name: sharding_gen - - name: sharding_jscore_passthrough - - name: .ssl - - name: .crypt - - name: .publish_crypt - - name: .publish - - name: generate_buildid_to_debug_symbols_mapping - -- name: ubuntu2204-arm64 - display_name: Ubuntu 22.04 arm64 - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - ubuntu2204-arm64-large - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: aarch64-ubuntu2204 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2204 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= - test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer - resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers - has_packages: true - packager_script: packager.py - packager_arch: arm64 - packager_distro: ubuntu2204 - repo_edition: org - multiversion_platform: ubuntu2204 - multiversion_architecture: arm64 - multiversion_architecture_42_or_later: aarch64 - multiversion_edition: targeted - scons_cache_scope: shared - tasks: - - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG + - name: free_monitoring - name: jsCore - name: replica_sets_jscore_passthrough + - name: test_packages + distros: + - ubuntu2004-package - name: .publish + distros: + - ubuntu2004-test - name: generate_buildid_to_debug_symbols_mapping - name: windows @@ -2099,20 +1621,17 @@ buildvariants: multiversion_platform_44_or_later: windows multiversion_edition: base content_type: application/zip - compile_flags: --ssl MONGO_DISTMOD=windows -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --modules= + compile_flags: --ssl MONGO_DISTMOD=windows -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 num_scons_link_jobs_available: 0.25 python: '/cygdrive/c/python/python37/python.exe' ext: zip scons_cache_scope: shared large_distro_name: windows-vsCurrent-large - test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls,requires_latch_analyzer --enableEnterpriseTests=off + test_flags: &windows_common_test_excludes --excludeWithAnyTags=incompatible_with_windows_tls tasks: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-large - name: .aggfuzzer - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen @@ -2125,6 +1644,7 @@ buildvariants: - windows-vsCurrent-large - name: .concurrency .common - name: disk_wiredtiger + - name: free_monitoring - name: .jscore .common !.auth - name: json_schema - name: .jstestfuzz !.initsync !.flow_control !.stepdowns @@ -2133,7 +1653,7 @@ buildvariants: - name: .query_fuzzer - name: .read_write_concern - name: replica_sets_gen - - name: replica_sets_jscore_passthrough_gen + - name: replica_sets_jscore_passthrough - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .txns - name: .sharding .common !.csrs @@ -2141,10 +1661,14 @@ buildvariants: - name: .stitch - name: .updatefuzzer - name: push + distros: + - rhel70-small - name: enterprise-windows display_name: "Enterprise Windows" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - windows-vsCurrent-small expansions: @@ -2172,9 +1696,6 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large - - name: compile_test_and_package_parallel_unittest_stream_TG - distros: - - windows-vsCurrent-large - name: .aggfuzzer - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen @@ -2190,6 +1711,8 @@ buildvariants: distros: - windows-vsCurrent-large - name: .publish_crypt + distros: + - rhel70-small - name: disk_wiredtiger - name: .jscore .common !.auth - name: json_schema @@ -2205,8 +1728,11 @@ buildvariants: - name: .sharding .txns - name: .sharding .common !.csrs - name: .ssl + - name: .stitch - name: .updatefuzzer - name: push + distros: + - rhel70-small ########################################### # macOS buildvariants # @@ -2217,29 +1743,50 @@ buildvariants: run_on: - macos-1100 expansions: - test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_latch_analyzer --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=incompatible_with_macos push_path: osx push_bucket: downloads.mongodb.org push_name: macos push_arch: x86_64 compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars --modules= + compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars resmoke_jobs_max: 6 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_build_tools_next_TG - - name: aggregation + - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen - - name: change_streams + - name: .causally_consistent !.sharding + - name: .change_streams !.secondary_reads - name: .misc_js + - name: .concurrency !.ubsan !.no_txns !.debug_only !.kill_terminate - name: disk_wiredtiger - - name: .jscore .common !.auth !.sharding + - name: free_monitoring + - name: initial_sync_fuzzer_gen + - name: .jscore .common !.auth + - name: jsCore_txns_large_txns_format + - name: .jstestfuzz .causal + - name: .jstestfuzz .interrupt + - name: .jstestfuzz .common + - name: .jstestfuzz .session - name: .logical_session_cache .one_sec + - name: .query_fuzzer + - name: .read_write_concern !.linearize - name: replica_sets_gen + - name: replica_sets_kill_secondaries_jscore_passthrough + - name: replica_sets_large_txns_format_gen - name: .replica_sets .common !.auth + - name: retryable_writes_jscore_passthrough_gen + - name: .rollbackfuzzer + - name: session_jscore_passthrough + - name: .sharding .jscore !.wo_snapshot !.multi_stmt + - name: .sharding .txns - name: .ssl + - name: .stitch + - name: unittest_shell_hang_analyzer_gen - name: push + distros: + - rhel70-small - name: macos-arm64 display_name: macOS arm64 @@ -2247,37 +1794,60 @@ buildvariants: run_on: - macos-1100-arm64 expansions: - test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_latch_analyzer --enableEnterpriseTests=off + test_flags: --excludeWithAnyTags=incompatible_with_macos push_path: osx push_bucket: downloads.mongodb.org push_name: macos push_arch: arm64 compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx_arm.vars --modules= + compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx_arm.vars resmoke_jobs_max: 6 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_build_tools_next_TG - - name: aggregation + - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen - - name: change_streams + - name: .causally_consistent !.sharding + - name: .change_streams !.secondary_reads - name: .misc_js + - name: .concurrency !.ubsan !.no_txns !.debug_only !.kill_terminate - name: disk_wiredtiger - - name: .jscore .common !.auth !.sharding + - name: free_monitoring + - name: initial_sync_fuzzer_gen + - name: .jscore .common !.auth + - name: jsCore_txns_large_txns_format + - name: .jstestfuzz .causal + - name: .jstestfuzz .interrupt + - name: .jstestfuzz .common + - name: .jstestfuzz .session - name: .logical_session_cache .one_sec + - name: .query_fuzzer + - name: .read_write_concern !.linearize - name: replica_sets_gen + - name: replica_sets_kill_secondaries_jscore_passthrough + - name: replica_sets_large_txns_format_gen - name: .replica_sets .common !.auth + - name: retryable_writes_jscore_passthrough_gen + - name: .rollbackfuzzer + - name: session_jscore_passthrough + - name: .sharding .jscore !.wo_snapshot !.multi_stmt + - name: .sharding .txns - name: .ssl + - name: .stitch + - name: unittest_shell_hang_analyzer_gen - name: push + distros: + - rhel70-small - name: enterprise-macos display_name: Enterprise macOS cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - macos-1100 expansions: - test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_latch_analyzer + test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: osx push_bucket: downloads.10gen.com @@ -2288,25 +1858,36 @@ buildvariants: resmoke_jobs_max: 6 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_build_tools_next_TG - name: audit - name: auth_audit_gen - - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore + - name: causally_consistent_jscore_txns_passthrough + # TODO: SERVER-66945 Re-enable ESE on enterprise macos + # - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore + - name: .jscore .common !.decimal !.sharding + - name: .jstestfuzz .common - name: .logical_session_cache .one_sec + - name: mqlrun - name: replica_sets_auth_gen + - name: replica_sets_jscore_passthrough - name: sasl - name: push + distros: + - rhel70-small - name: .crypt - name: .publish_crypt + distros: + - rhel70-small - name: enterprise-macos-arm64 display_name: Enterprise macOS arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - macos-1100-arm64 expansions: - test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_latch_analyzer + test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: osx push_bucket: downloads.10gen.com @@ -2317,225 +1898,22 @@ buildvariants: resmoke_jobs_max: 6 tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_build_tools_next_TG - name: audit - name: auth_audit_gen + - name: causally_consistent_jscore_txns_passthrough + - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore - name: .jscore .common !.decimal !.sharding - - name: fle - - name: fle2 + - name: .jstestfuzz .common - name: .logical_session_cache .one_sec + - name: mqlrun - name: replica_sets_auth_gen + - name: replica_sets_jscore_passthrough - name: sasl - name: push + distros: + - rhel70-small - name: .crypt - name: .publish_crypt - -- name: amazon-linux2-arm64-try-sbe-engine - display_name: Amazon Linux 2 arm64 Query Patch Only (trySbeEngine) - cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter. - run_on: - - amazon2-arm64-latest-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: aarch64-enterprise-amazon2 - # Diagnostic latches are disabled by default, and we match that behavior here. We also need to - # exclude tests with 'requires_latch_analyzer'. - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - --use-diagnostic-latches=off - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - test_flags: >- - --mongodSetParameters="{featureFlagSbeFull: true, internalQueryForceClassicEngine: false}" - --excludeWithAnyTags=resource_intensive,incompatible_with_amazon_linux,sbe_incompatible,requires_latch_analyzer - has_packages: true - multiversion_platform: amazon2 - multiversion_edition: enterprise - multiversion_architecture: arm64 - multiversion_architecture_42_or_later: aarch64 - multiversion_skip_install_for_40: "true" - packager_script: packager_enterprise.py - packager_arch: aarch64 - packager_distro: amazon2 - repo_edition: enterprise - scons_cache_scope: shared - large_distro_name: amazon2-arm64-latest-large - tasks: - - name: compile_and_archive_dist_test_then_package_TG - distros: - - amazon2-arm64-latest-large - - name: aggregation_mongos_passthrough - - name: aggregation_sharded_collections_passthrough - - name: aggregation_one_shard_sharded_collections - - name: aggregation - - name: aggregation_disabled_optimization - - name: noPassthrough_gen - - name: noPassthroughWithMongod_gen - - name: jsCore distros: - - amazon2-arm64-latest-large - - name: replica_sets_jscore_passthrough_gen - - name: sharded_collections_jscore_passthrough_gen - - name: sharding_jscore_passthrough - - name: burn_in_tests_gen - - name: concurrency_gen - - name: concurrency_replication_gen - - name: concurrency_sharded_replication_gen - -# This task is responsible for creating a AL2023 x86 server binary candidate for 10gen/mongot. -# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting -# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests. -- name: amazon-linux-2023-x86-mongot-integration-cron-only - display_name: "AL2023 x86 mongot integration tasks cron only" - cron: "0 */4 * * *" # Run these tasks every 4 hours - patchable: false - run_on: - - amazon2023.0-small - expansions: - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: x86_64-enterprise-amazon2023 - compile_flags: --ssl MONGO_DISTMOD=amazon2023 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --enterprise-features="audit,encryptdb,fcbis,fips,fle,hot_backups,inmemory,kerberos,vector_search,ldap,live_import,mongohouse,queryable,sasl,search,serverless" - # TODO BUILD-13887 should fix uses_pykmip incompatibility. - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,uses_pykmip,requires_v4_0,requires_external_data_source,requires_latch_analyzer - has_packages: false - multiversion_platform: amazon2023 - multiversion_edition: enterprise - multiversion_architecture: x86_64 - packager_script: packager_enterprise.py - packager_arch: x86_64 - packager_distro: amazon2023 - repo_edition: enterprise - scons_cache_scope: shared - compile_variant: amazon-linux-2023-x86-mongot-integration-cron-only - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2023.0-large - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - -# This task is responsible for creating a AL2023 arm64 server binary candidate for 10gen/mongot. -# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting -# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests. -- name: amazon2023-arm64-mongot-integration-cron-only - display_name: "AL2023 arm64 mongot integration tasks cron only" - cron: "0 */4 * * *" # Run these tasks every 4 hours - patchable: false - run_on: - - amazon2023.0-arm64-small - expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --enterprise-features="audit,encryptdb,fcbis,fips,fle,hot_backups,inmemory,kerberos,vector_search,ldap,live_import,mongohouse,queryable,sasl,search,serverless" - test_flags: >- - --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source,requires_latch_analyzer - --enableEnterpriseTests=off - has_packages: false - multiversion_platform: amazon2023 - multiversion_edition: enterprise - multiversion_architecture: aarch64 - packager_script: packager_enterprise.py - packager_arch: aarch64 - packager_distro: amazon2023 - repo_edition: enterprise - scons_cache_scope: shared - compile_variant: amazon2023-arm64-mongot-integration-cron-only - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2023.0-arm64-large - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - -# This task is responsible for creating a AL2 arm64 server binary candidate for 10gen/mongot. -# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting -# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests. -- name: amazon-linux2-arm64-mongot-integration-cron-only - display_name: "AL2 arm64 mongot integration tasks cron only" - cron: "0 */4 * * *" # Run these tasks every 4 hours - patchable: false - run_on: - - amazon2-arm64-small - stepback: true - expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - scons_cache_scope: shared - scons_cache_mode: all - has_packages: false - target_resmoke_time: 10 - idle_timeout_factor: 1.5 - exec_timeout_factor: 1.5 - compile_variant: amazon-linux2-arm64-mongot-integration-cron-only - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2-arm64-large - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl - -# This task is responsible for creating a AL2 x86 server binary candidate for 10gen/mongot. -# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting -# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests -- name: amazon2-x86-mongot-integration-cron-only - display_name: "AL2 x86 mongot integration tasks cron only" - cron: "0 */4 * * *" # Run these tasks every 4 hours - patchable: false - run_on: - - amazon2-large - expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source,requires_latch_analyzer - has_packages: false - multiversion_platform: amazon2 - multiversion_edition: enterprise - multiversion_architecture: x86_64 - packager_script: packager_enterprise.py - packager_arch: x86_64 - packager_distro: amazon2 - repo_edition: enterprise - scons_cache_scope: shared - compile_variant: amazon2-x86-mongot-integration-cron-only - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2-large - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl + - rhel70-small diff --git a/etc/evergreen_yml_components/variants/ninja.yml b/etc/evergreen_yml_components/variants/ninja.yml deleted file mode 100644 index 07d5be46125..00000000000 --- a/etc/evergreen_yml_components/variants/ninja.yml +++ /dev/null @@ -1,85 +0,0 @@ -buildvariants: - -- name: enterprise-windows-ninja - display_name: "Ninja Build: Enterprise Windows" - cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds - expansions: - compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 - tasks: - - name: compile_ninja_next_TG - distros: - - windows-vsCurrent-large - - name: compile_ninja_TG - distros: - - windows-vsCurrent-large - -- name: macos-enterprise-ninja - display_name: "Ninja Build: macOS Enterprise" - cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds - run_on: - - macos-1100 - expansions: - compile_env: DEVELOPER_DIR=/Applications/Xcode13.app - compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars - tasks: - - name: compile_ninja_next_TG - - name: compile_ninja_TG - -- name: ubuntu1804-ninja-build-profiles - display_name: "Ninja Build Profiles: Ubuntu 18.04" - cron: "0 4 * * 0" # # Run once a week to ensure no failures introduced to build profiles - run_on: - - ubuntu1804-small - stepback: false - expansions: - compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --modules= - test_flags: --enableEnterpriseTests=off - repo_edition: enterprise - has_packages: false - tasks: - - name: compile_ninja_default_profile_TG - distros: - - ubuntu1804-xlarge - - name: compile_ninja_opt_profile_TG - distros: - - ubuntu1804-xlarge - - name: compile_ninja_san_profile_TG - distros: - - ubuntu1804-xlarge - - name: compile_ninja_fast_profile_TG - distros: - - ubuntu1804-xlarge - -- name: enterprise-rhel-8-64-bit-dynamic-required-ninja - display_name: "Ninja Build: Enterprise RHEL 8" - cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds - run_on: - - rhel80-small - expansions: - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic - has_packages: false - tasks: - - name: compile_ninja_next_TG - distros: - - rhel80-xlarge - - name: compile_ninja_TG - distros: - - rhel80-xlarge - -- name: enterprise-rhel8-ninja - display_name: "Ninja Build: RHEL 8" - cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds - run_on: - - rhel80-small - stepback: false - expansions: - compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic - repo_edition: enterprise - has_packages: false - tasks: - - name: compile_ninja_next_TG - distros: - - rhel80-build - - name: compile_ninja_TG - distros: - - rhel80-build diff --git a/etc/evergreen_yml_components/variants/sanitizer.yml b/etc/evergreen_yml_components/variants/sanitizer.yml index bdc478003e8..ec0da723723 100644 --- a/etc/evergreen_yml_components/variants/sanitizer.yml +++ b/etc/evergreen_yml_components/variants/sanitizer.yml @@ -3,8 +3,8 @@ buildvariants: -- name: rhel8-asan - display_name: ~ ASAN RHEL 8 +- name: rhel80-asan + display_name: ~ ASAN RHEL 8.0 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -12,7 +12,7 @@ buildvariants: expansions: lang_environment: LANG=C san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --use-diagnostic-latches=on + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. @@ -22,13 +22,15 @@ buildvariants: separate_debug: off tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: .aggfuzzer .common + - name: free_monitoring - name: .jstestfuzz !.initsync -- name: rhel8-debug-asan - display_name: ~ ASAN Enterprise RHEL 8 DEBUG +- name: rhel80-debug-asan + display_name: ~ ASAN Enterprise RHEL 8.0 DEBUG cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -38,8 +40,8 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) - test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling,requires_increased_memlock_limits,requires_latch_analyzer + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) + test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. @@ -50,9 +52,7 @@ buildvariants: large_distro_name: rhel80-build tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_benchmarks - - name: compile_integration_and_test_parallel_stream_TG - name: .aggregation - name: .auth - name: audit @@ -62,9 +62,11 @@ buildvariants: - name: .misc_js - name: .concurrency !.ubsan !.no_txns !.kill_terminate - name: .encrypt + - name: free_monitoring - name: external_auth - name: external_auth_aws - name: initial_sync_fuzzer_gen + - name: .integration !.standalone !.audit - name: .jscore .common - name: jsCore_minimum_batch_size - name: jsCore_txns_large_txns_format @@ -92,15 +94,18 @@ buildvariants: - name: .sharding .common !.csrs - name: snmp - name: .watchdog + - name: .stitch - name: .serverless - name: unittest_shell_hang_analyzer_gen - name: .updatefuzzer - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG -- name: rhel8-debug-ubsan - display_name: ~ UBSAN Enterprise RHEL 8 DEBUG +- name: rhel80-debug-ubsan + display_name: ~ UBSAN Enterprise RHEL 8.0 DEBUG cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + modules: + - enterprise run_on: - rhel80-build stepback: false @@ -110,8 +115,8 @@ buildvariants: # If you add anything to san_options, make sure the appropriate changes are # also made to SConstruct. san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" - compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --use-diagnostic-latches=on - test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits + compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off --enable-free-mon=on -j$(grep -c ^processor /proc/cpuinfo) + test_flags: --excludeWithAnyTags=requires_ocsp_stapling multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. @@ -120,9 +125,7 @@ buildvariants: large_distro_name: rhel80-build tasks: - name: compile_test_and_package_serial_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - name: compile_benchmarks - - name: compile_integration_and_test_parallel_stream_TG - name: .aggregation - name: .auth - name: audit @@ -130,10 +133,12 @@ buildvariants: - name: .causally_consistent !.wo_snapshot - name: .change_streams - name: .misc_js - - name: .concurrency !.no_txns !.kill_terminate + - name: .concurrency !.no_txns !.repl !.kill_terminate - name: disk_wiredtiger - name: .encrypt + - name: free_monitoring - name: initial_sync_fuzzer_gen + - name: .integration !.audit - name: .jscore .common - name: jsCore_minimum_batch_size - name: jsCore_txns_large_txns_format @@ -158,6 +163,7 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot - name: .sharding .common !.csrs - name: snmp + - name: .stitch - name: .updatefuzzer - name: .serverless - name: watchdog_wiredtiger diff --git a/etc/evergreen_yml_components/variants/task_generation.yml b/etc/evergreen_yml_components/variants/task_generation.yml index fa010996291..9c09890e566 100644 --- a/etc/evergreen_yml_components/variants/task_generation.yml +++ b/etc/evergreen_yml_components/variants/task_generation.yml @@ -3,13 +3,9 @@ buildvariants: - name: generate-tasks-for-version display_name: "Generate tasks for evergreen version" + modules: + - enterprise run_on: - rhel80-medium tasks: - name: version_gen - distros: - - ubuntu2004-medium - - name: version_burn_in_gen - distros: - - ubuntu2004-medium - - name: version_expansions_gen diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml index abc72b2141b..2ddbe84a0c6 100644 --- a/etc/expansions.default.yml +++ b/etc/expansions.default.yml @@ -37,4 +37,5 @@ install_directory: /data/mongo-install-directory num_scons_link_jobs_available: "0.25" scons_cache_mode: nolinked show_scons_timings: "true" +task_install_action: hardlink separate_debug: "on" diff --git a/etc/generate_subtasks_config.yml b/etc/generate_subtasks_config.yml index 7e8384e5e6e..2a2255148b8 100644 --- a/etc/generate_subtasks_config.yml +++ b/etc/generate_subtasks_config.yml @@ -1,16 +1,10 @@ build_variant_large_distro_exceptions: - amazon - amazon2 - - amazon2023 - - amazon2023-arm64 - debian10 - debian11 - enterprise-amazon2 - - enterprise-amazon2-openssl111 - enterprise-amazon2-arm64 - - enterprise-amazon2-arm64-openssl111 - - enterprise-amazon2023 - - enterprise-amazon2023-arm64 - enterprise-debian10-64 - enterprise-debian11-64 - enterprise-linux-64-amazon-ami @@ -25,13 +19,11 @@ build_variant_large_distro_exceptions: - enterprise-rhel-71-ppc64le-inmem - enterprise-rhel-72-s390x - enterprise-rhel-72-s390x-inmem - - enterprise-rhel-8-64-bit - - enterprise-rhel-8-64-bit-coverage + - enterprise-rhel-80-64-bit + - enterprise-rhel-80-64-bit-coverage - enterprise-rhel-81-ppc64le - - enterprise-rhel-8-arm64 + - enterprise-rhel-82-arm64 - enterprise-rhel-83-s390x - - enterprise-rhel-90-64-bit - - enterprise-rhel-90-arm64 - enterprise-suse12-64 - enterprise-suse12-s390x - enterprise-suse15-64 @@ -39,17 +31,14 @@ build_variant_large_distro_exceptions: - enterprise-ubuntu1804-ppc64le - enterprise-ubuntu1804-s390x - enterprise-ubuntu2004-arm64 - - enterprise-ubuntu2204-arm64 - hot_backups-rhel-70-64-bit - linux-64-ephemeralForTest - macos - macos-arm64 - macos-debug-suggested - rhel70 - - rhel8 - - rhel-8-arm64 - - rhel90 - - rhel90-arm64 + - rhel80 + - rhel-82-arm64 - suse12 - suse15 - ubi8 @@ -58,8 +47,8 @@ build_variant_large_distro_exceptions: - ubuntu1604-debug - ubuntu1804-debug-asan - ubuntu1804-debug-asan-all-feature-flags - - ubuntu1804-debug-asan-sbe-engine + - ubuntu1804-debug-asan-classic-engine - ubuntu1804-debug-aubsan-lite-required - ubuntu1804-debug-ubsan - ubuntu1804-debug-ubsan-all-feature-flags - - ubuntu1804-debug-ubsan-sbe-engine + - ubuntu1804-debug-ubsan-classic-engine diff --git a/etc/lsan.suppressions b/etc/lsan.suppressions index 6eb6406e5db..80d58180cae 100644 --- a/etc/lsan.suppressions +++ b/etc/lsan.suppressions @@ -9,19 +9,4 @@ leak:mongo::Interruptible::installWaitListener # The singleton must live throughout the lifetime of all SSL threads leak::mongo::SSLThreadInfo::ThreadIDManager::idManager -# Thread names leak from threads that are never terminated. -leak:mongo::setThreadName -leak:mongo::getThreadName -leak:__cxa_thread_atexit_impl - -# Threads leak from thread pools not joined at shutdown. -leak:mongo::ThreadPool::Impl::_startWorkerThread_inlock() - leak:glob64 - -# Allocations from this function in libresolv are stored in thread-locals. -# Threads that are not joined by the time the leak-detector runs (like those spawned by ASIO) may -# still have pointers to these allocated objects but LSAN isn't able to identify them and add them -# to do the root-set, resulting in false-positive leak reports. -# TODO SERVER-65936: Re-evaluate if this is needed with a clean shutdown that joins all threads. -leak:__res_context_send diff --git a/etc/macos_dev_entitlements.xml b/etc/macos_dev_entitlements.xml deleted file mode 100644 index 6835b0eb04c..00000000000 --- a/etc/macos_dev_entitlements.xml +++ /dev/null @@ -1,9 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> - <dict> - <key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/> - <!--The get-task-allow entitlement is insecure and must only be used in dev environments--> - <key>com.apple.security.get-task-allow</key><true/> - </dict> -</plist> diff --git a/etc/macos_entitlements.xml b/etc/macos_entitlements.xml deleted file mode 100644 index a7e59c8d96c..00000000000 --- a/etc/macos_entitlements.xml +++ /dev/null @@ -1,8 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> - <dict> - <key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/> - </dict> -</plist> - diff --git a/etc/macos_mongodb.plist b/etc/macos_mongodb.plist deleted file mode 100644 index e57e7d9be3e..00000000000 --- a/etc/macos_mongodb.plist +++ /dev/null @@ -1,34 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>Label</key> - <string>#{plist_name}</string> - <key>ProgramArguments</key> - <array> - <string>#{opt_bin}/mongod</string> - <string>--config</string> - <string>#{etc}/mongod.conf</string> - </array> - <key>RunAtLoad</key> - <true/> - <key>KeepAlive</key> - <false/> - <key>WorkingDirectory</key> - <string>#{HOMEBREW_PREFIX}</string> - <key>StandardErrorPath</key> - <string>#{var}/log/mongodb/output.log</string> - <key>StandardOutPath</key> - <string>#{var}/log/mongodb/output.log</string> - <key>HardResourceLimits</key> - <dict> - <key>NumberOfFiles</key> - <integer>64000</integer> - </dict> - <key>SoftResourceLimits</key> - <dict> - <key>NumberOfFiles</key> - <integer>64000</integer> - </dict> -</dict> -</plist> diff --git a/etc/perf.yml b/etc/perf.yml new file mode 100644 index 00000000000..87459fb219b --- /dev/null +++ b/etc/perf.yml @@ -0,0 +1,848 @@ +command_type: system +stepback: false + +## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds) +parameters: + - key: patch_compile_flags + description: "Additional SCons flags to be applied during scons compile invocations in this patch" + +variables: + ### + # Leave this section uncommented to enable compile. + _real_remote_file: &_remote_file + ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile-variant|}-${version_id}.tar.gz + _real_compile: &_compile + - variant: linux-wt-standalone + name: compile + _real_expansions: &_expansion_updates + [] + ### + + ### + # **Or**: Leave this section uncommented to bypass/skip compile. + # This file ↓ came from a microbenchmarks waterfall run. + # https://evergreen.mongodb.com/version/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551 + # Artifacts eventually expire. If this fails, grab the compile artifacts url and update this. +# _skip_remote_file: &_remote_file +# /perf/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551/c0a8cbe58cc46253a94130d2cb64cdd8089b3551/linux/mongodb-performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551.tar.gz +# _skip_compile: &_compile +# [] +# _skip_expansions: &_expansion_updates +# - key: mdb_binary_for_server +# value: https://mciuploads.s3.amazonaws.com/perf/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551/c0a8cbe58cc46253a94130d2cb64cdd8089b3551/linux/mongodb-performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551.tar.gz +# - key: mdb_binary_for_client +# value: https://mciuploads.s3.amazonaws.com/perf/performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551/c0a8cbe58cc46253a94130d2cb64cdd8089b3551/linux/mongodb-performance_c0a8cbe58cc46253a94130d2cb64cdd8089b3551.tar.gz + ### + + _src_dir: &src_dir src/mongo + _modules: &modules + - enterprise + - mongo-tools + - dsi + - genny + - signal-processing + - workloads + - linkbench + - linkbench2 + - mongo-perf + - YCSB + - benchmarks + - py-tpcc + +modules: + ### + # Same in every DSI project. Ensure that this block is synchronized with + # evergreen-dsitest.yml, atlas/system_perf_atlas.yml, and src/dsi/onboarding.py + # (search update-repos-here) in this repo, and etc/system_perf.yml and + # etc/perf.yml in mongodb/mongo + - name: dsi + repo: git@github.com:10gen/dsi.git + prefix: ${workdir}/src + branch: master + - name: genny + repo: git@github.com:10gen/genny.git + prefix: ${workdir}/src + branch: microbenchmarks-stable + - name: signal-processing + repo: git@github.com:10gen/signal-processing.git + prefix: ${workdir}/src + branch: master + - name: workloads + repo: git@github.com:10gen/workloads.git + prefix: ${workdir}/src + branch: master + - name: linkbench + repo: git@github.com:10gen/linkbench.git + prefix: ${workdir}/src + branch: master + - name: linkbench2 + repo: git@github.com:10gen/linkbench2.git + prefix: ${workdir}/src + branch: master + - name: mongo-perf + repo: git@github.com:mongodb/mongo-perf.git + prefix: ${workdir}/src + branch: master + - name: YCSB + repo: git@github.com:mongodb-labs/YCSB.git + prefix: ${workdir}/src + branch: production + - name: benchmarks + repo: git@github.com:mongodb-labs/benchmarks.git + prefix: ${workdir}/src + branch: master + - name: py-tpcc + repo: git@github.com:mongodb-labs/py-tpcc.git + prefix: ${workdir}/src + branch: production + + ### +# - name: mongo +# repo: git@github.com:mongodb/mongo.git +# prefix: ${workdir}/src +# branch: master + - name: enterprise + repo: git@github.com:10gen/mongo-enterprise-modules.git + prefix: src/mongo/db/modules + branch: v6.0 + - name: mongo-tools + repo: git@github.com:mongodb/mongo-tools.git + prefix: mongo-tools/src/github.com/mongodb + branch: master + + +### +# Same in every DSI project +pre: + - func: f_other_pre_ops + - func: f_dsi_pre_run +post: + - func: f_dsi_post_run + - func: f_other_post_ops +timeout: + - func: f_dsi_timeout + - func: f_other_timeout +### + +functions: + ### + # Same in every DSI project + f_dsi_pre_run: + - command: manifest.load + - command: expansions.update + params: + updates: *_expansion_updates + f_run_dsi_workload: + - command: git.get_project + params: + directory: *src_dir + revisions: + dsi: ${dsi_rev} + genny: ${genny_rev} + signal-processing: ${signal-processing_rev} + linkbench: ${linkbench_rev} + linkbench2: ${linkbench2_rev} + workloads: ${workloads_rev} + mongo-perf: ${mongo-perf_rev} + YCSB: ${YCSB_rev} + benchmarks: ${benchmarks_rev} + py-tpcc: ${py-tpcc_rev} + - command: expansions.write + params: + file: ./expansions.yml + - command: shell.exec + params: + script: ./src/dsi/run-dsi run_workload + - command: shell.exec + type: system + params: + script: ./src/dsi/run-dsi determine_failure -m SYSTEM + - command: shell.exec + type: setup + params: + script: ./src/dsi/run-dsi determine_failure -m SETUP + - command: shell.exec + type: test + params: + script: ./src/dsi/run-dsi determine_failure -m TEST + f_dsi_post_run: + - command: shell.exec + params: + script: ./src/dsi/run-dsi post_run + - command: perf.send + params: + file: ./build/CedarReports/cedar_report.json + aws_key: ${terraform_key} + aws_secret: ${terraform_secret} + bucket: genny-metrics + region: us-east-1 + prefix: ${task_id}_${execution} + - command: attach.results + params: + file_location: ./build/EvergreenResultsJson/results.json + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ./build/Artifacts/DSIArtifacts.tgz + remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/x-gzip + display_name: DSI Artifacts - Execution ${execution} + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ./build/Documentation/index.html + remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html + bucket: mciuploads + permissions: public-read + content_type: text/html + display_name: Documentation + f_dsi_timeout: + - command: shell.exec + params: + script: ./src/dsi/run-dsi on_timeout + ### + + f_other_post_ops: + - command: shell.exec + params: + working_dir: src + script: | + # removes files from the (local) scons cache when it's over a + # threshold, to the $prune_ratio percentage. Ideally override + # these default values in the distro config in evergreen. + + if [ -d "${scons_cache_path}" ]; then + /opt/mongodbtoolchain/v3/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8} + fi + f_other_pre_ops: + - &f_other_pre_ops + command: shell.exec + params: + silent: true + script: | + for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do + pkill -9 "$PS" + done + f_other_timeout: + # Can't be empty so just `echo`. + - command: shell.exec + params: {script: "echo"} + + ### + # Compile + compile mongodb: + # We create a virtual environment with the Python dependencies for compiling the server + # installed. + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + /opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python3 "${workdir}/compile_venv" + source "${workdir}/compile_venv/bin/activate" + + python -m pip install -r etc/pip/compile-requirements.txt + + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + mkdir -p mongodb/bin + + # We get the raw version string (r1.2.3-45-gabcdef) from git + MONGO_VERSION=$(git describe --abbrev=7) + + # If this is a patch build, we add the patch version id to the version string so we know + # this build was a patch, and which evergreen task it came from + if [ "${is_patch|false}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" + fi + + # This script converts the generated version string into a sanitized version string for + # use by scons and uploading artifacts as well as information about for the scons cache. + source "${workdir}/compile_venv/bin/activate" + MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} python buildscripts/generate_compile_expansions.py --out compile_expansions.yml + - command: expansions.update + params: + file: src/compile_expansions.yml + - command: shell.exec + params: + working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools + script: | + set -o verbose + set -o errexit + + # make sure newlines in the scripts are handled correctly by windows + if [ "Windows_NT" = "$OS" ]; then + set -o igncr + fi; + + # set_goenv provides set_goenv(), print_ldflags() and print_tags() used below + . ./set_goenv.sh + GOROOT="" set_goenv || exit + go version + + build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop" + if [ "${build_mongoreplay}" = "true" ]; then + build_tools="$build_tools mongoreplay" + fi + for i in $build_tools; do + go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go + "../../../../../mongodb/bin/$i${exe|}" --version + done + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + source "${workdir}/compile_venv/bin/activate" + python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src + mkdir -p mongodb/feature_flags + cp ./all_feature_flags.txt mongodb/feature_flags + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + source "${workdir}/compile_venv/bin/activate" + python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|} + mkdir -p mongodb/jstests/hooks + if [ -d jstests/hooks ] + then + echo "Fetching JS test DB correctness checks from directory jstests" + cp -a jstests/* mongodb/jstests + + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF + fi + tar czf mongodb${compile-variant|}.tar.gz mongodb + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb${compile-variant|}.tar.gz + remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile-variant|}-${version_id}.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: mongodb${compile-variant|}.tar.gz + ### + + ## Schedule Tasks ## + f_schedule_tasks: + - command: git.get_project + params: + directory: *src_dir + revisions: + dsi: ${dsi_rev} + genny: ${genny_rev} + signal-processing: ${signal-processing_rev} + linkbench: ${linkbench_rev} + linkbench2: ${linkbench2_rev} + workloads: ${workloads_rev} + mongo-perf: ${mongo-perf_rev} + YCSB: ${YCSB_rev} + benchmarks: ${benchmarks_rev} + py-tpcc: ${py-tpcc_rev} + - command: expansions.write + params: + file: ./expansions.yml + - command: shell.exec + params: + script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks} + - command: generate.tasks + params: + files: + - build/TaskJSON/Tasks.json + + +tasks: + ### + # Same in every DSI project + - name: schedule_global_auto_tasks + priority: 5 + commands: + - func: f_schedule_tasks + vars: + tasks: all_tasks + - name: schedule_variant_auto_tasks + priority: 5 + commands: + - func: f_schedule_tasks + vars: + tasks: variant_tasks + - name: schedule_patch_auto_tasks + priority: 5 + commands: + - func: f_schedule_tasks + vars: + tasks: patch_tasks + - name: smoke_test + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + - name: smoke_test_ssl + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: replica-ssl + infrastructure_provisioning: replica + - name: smoke_test_standalone_auth + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: standalone-auth + infrastructure_provisioning: single + - name: smoke_test_replset_auth + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: replica-auth + infrastructure_provisioning: replica + - name: smoke_test_shard_lite_auth + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: shard-lite-auth + infrastructure_provisioning: shard-lite + - name: dsi_integ_test_run_command_simple + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: run_command_simple + ### + + - name: compile + commands: + - command: manifest.load + - command: git.get_project + params: + directory: src + revisions: + enterprise: ${enterprise_rev} + mongo-tools: ${mongo-tools_rev} + - func: "compile mongodb" + + + - name: genny_execution_UserAcquisition + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: auto_genny_workload + auto_workload_path: ./src/genny/dist/etc/genny/workloads/execution/UserAcquisition.yml + - name: genny_scale_InsertRemove + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: auto_genny_workload + auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/InsertRemove.yml + - name: query_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: query, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: query_read_commands_large_dataset + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: query_large_dataset, + include_filter_2: regression, + exclude_filter: none, + threads: "1 4", + read_cmd: 'true', + share_dataset: 'true'} + - name: big_collection + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: query, + include_filter_2: getmore, + exclude_filter: none, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: views-query + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: query_identityview, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: views-aggregation + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: aggregation_identityview, + include_filter_2: regression, + exclude_filter: none, + threads: "1", + read_cmd: 'true'} + - name: where_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: where, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: update_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: update, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: insert_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: insert, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: wildcard-index-read_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: wildcard_read, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: wildcard-index-write_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: wildcard_write, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: geo_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: geo, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: misc_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: command multi remove mixed, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: misc_custom_filter_default_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + mongodb_setup: microbenchmarks_standalone_custom_filter_default + test_control_params: | + {include_filter_1: command multi remove mixed, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: misc_custom_filter_slow_or_sample_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + mongodb_setup: microbenchmarks_standalone_custom_filter_slow_or_sample + test_control_params: | + {include_filter_1: command multi remove mixed, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: misc_custom_filter_complex_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + mongodb_setup: microbenchmarks_standalone_custom_filter_complex + test_control_params: | + {include_filter_1: command multi remove mixed, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: misc_custom_filter_whole_doc_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + mongodb_setup: microbenchmarks_standalone_custom_filter_whole_doc + test_control_params: | + {include_filter_1: command multi remove mixed, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: misc_slowms_everything_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + mongodb_setup: microbenchmarks_standalone_slowms_everything + test_control_params: | + {include_filter_1: command multi remove mixed, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: singleThreaded_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: single_threaded, + include_filter_2: core regression, + exclude_filter: none, + threads: "1", + read_cmd: 'true'} + - name: aggregation_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: aggregation, + include_filter_2: regression, + exclude_filter: js, + threads: "1", + read_cmd: 'true'} + - name: aggregation_read_commands_large_dataset + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: aggregation_large_dataset, + include_filter_2: regression, + exclude_filter: js, + threads: "1 4", + read_cmd: 'true', + share_dataset: 'true'} + - name: agg-query-comparison_read_commands + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: agg_query_comparison, + include_filter_2: core regression, + exclude_filter: single_threaded, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: pipeline-updates + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: pipeline-updates, + include_filter_2: regression, + exclude_filter: none, + threads: "1 2 4 8", + read_cmd: 'true'} + - name: javascript + depends_on: *_compile + commands: + - func: f_run_dsi_workload + vars: + test_control: microbenchmarks + test_control_params: | + {include_filter_1: js, + include_filter_2: aggregation, + exclude_filter: none, + threads: "1 2 4 8", + read_cmd: 'true'} + + +microbenchmark-buildvariants: + _linux-wt-standalone: &linux-wt-standalone + name: linux-wt-standalone + display_name: Standalone Linux inMemory + batchtime: 240 # 4 hours + modules: *modules + expansions: + # We are explicitly tracking the rhel62 variant compile options from evergreen.yml for + # microbenchmarks, since they run on the centos6 boxes. If we can get proper artifacts directly + # from that project, we should do that and remove the compile tasks. + compile_flags: --ssl --separate-debug MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + mongod_exec_wrapper: &exec_wrapper "numactl --physcpubind=4,5,6,7 -i 1" + perf_exec_wrapper: &perf_wrapper "numactl --physcpubind=1,2,3 -i 0" + use_scons_cache: true + platform: linux + infrastructure_provisioning: microbenchmarks + mongodb_setup: microbenchmarks_standalone + canaries: none + storageEngine: inMemory + project_dir: &project_dir perf + run_on: + - "centos6-perf" + tasks: + - name: compile + distros: + - rhel62-large + - name: big_collection + - name: genny_scale_InsertRemove + - name: genny_execution_UserAcquisition + - name: aggregation_read_commands + - name: aggregation_read_commands_large_dataset + - name: agg-query-comparison_read_commands + - name: query_read_commands + - name: query_read_commands_large_dataset + - name: views-aggregation + - name: views-query + - name: where_read_commands + - name: update_read_commands + - name: insert_read_commands + - name: wildcard-index-read_read_commands + - name: wildcard-index-write_read_commands + - name: geo_read_commands + - name: misc_read_commands + - name: misc_custom_filter_default_read_commands + - name: misc_custom_filter_slow_or_sample_read_commands + - name: misc_custom_filter_complex_read_commands + - name: misc_custom_filter_whole_doc_read_commands + - name: misc_slowms_everything_read_commands + - name: singleThreaded_read_commands + - name: pipeline-updates + - name: javascript + + _linux-wt-repl: &linux-wt-repl + name: linux-wt-repl + display_name: 1-Node ReplSet Linux inMemory + batchtime: 240 # 4 hours + modules: *modules + expansions: + mongod_exec_wrapper: *exec_wrapper + perf_exec_wrapper: *perf_wrapper + platform: linux + infrastructure_provisioning: microbenchmarks + mongodb_setup: microbenchmarks_replica + canaries: none + storageEngine: inMemory + project_dir: *project_dir + run_on: + - "centos6-perf" + tasks: + - name: genny_scale_InsertRemove + - name: update_read_commands + - name: insert_read_commands + - name: misc_read_commands + - name: singleThreaded_read_commands + - name: wildcard-index-write_read_commands + - name: pipeline-updates + +buildvariants: + - *linux-wt-standalone + + - *linux-wt-repl + + - <<: *linux-wt-standalone + name: linux-wt-standalone-all-feature-flags + display_name: Standalone Linux inMemory (all feature flags) + batchtime: 1440 # 24 hours + expansions: + mongodb_setup: microbenchmarks_standalone-all-feature-flags + + - <<: *linux-wt-repl + name: linux-wt-repl-all-feature-flags + display_name: 1-Node ReplSet Linux inMemory (all feature flags) + batchtime: 1440 # 24 hours + expansions: + mongodb_setup: microbenchmarks_replica-all-feature-flags diff --git a/etc/pip/components/compile.req b/etc/pip/components/compile.req index e26295d38b3..1ef0f56a10d 100644 --- a/etc/pip/components/compile.req +++ b/etc/pip/components/compile.req @@ -4,4 +4,3 @@ packaging <= 21.3 regex <= 2021.11.10 requirements_parser <= 0.3.1 setuptools -ninja>=1.10.0 diff --git a/etc/pip/components/core.req b/etc/pip/components/core.req index 50cfb87d646..d73d60dcd9b 100644 --- a/etc/pip/components/core.req +++ b/etc/pip/components/core.req @@ -1,6 +1,6 @@ # Core (we need these for most buildscripts) psutil <= 5.8.0 pymongo >= 3.9, < 4.0 -PyYAML == 5.3.1 # Pinned due to SERVER-79126 and linked BF +PyYAML >= 3.0.0, <= 6.0.0 types-PyYAML ~= 6.0.5 requests >= 2.0.0, <= 2.26.0 diff --git a/etc/pip/components/idl.req b/etc/pip/components/idl.req index 13ad323f0cd..0b0ef8924b2 100644 --- a/etc/pip/components/idl.req +++ b/etc/pip/components/idl.req @@ -1,2 +1,3 @@ unittest-xml-reporting >= 2.2.0, <= 3.0.4 +typing <= 3.7.4.3 packaging <= 21.3 diff --git a/etc/pip/components/jiraclient.req b/etc/pip/components/jiraclient.req index 106e78f9de3..760a5c46684 100644 --- a/etc/pip/components/jiraclient.req +++ b/etc/pip/components/jiraclient.req @@ -1,5 +1,5 @@ # TIG jira integration jira <= 3.1.1 requests-oauth <= 0.4.1 -PyJWT == 2.4.0 +PyJWT <= 2.3.0 # https://github.com/pycontribs/jira/issues/247 # We are omitting pycrypto based on https://github.com/pycontribs/jira/pull/629 diff --git a/etc/pip/components/lint.req b/etc/pip/components/lint.req index 80ddffa2c2a..9e04fbdea6c 100644 --- a/etc/pip/components/lint.req +++ b/etc/pip/components/lint.req @@ -5,6 +5,7 @@ mypy ~= 0.942 pydocstyle == 6.1.1 pylint == 2.7.2 structlog ~= 19.2.0 +typing <= 3.7.4.3 yamllint == 1.15.0 yapf == 0.26.0 evergreen-lint == 0.1.3 diff --git a/etc/pip/components/platform.req b/etc/pip/components/platform.req index ec5aabae7e5..10bb44fb08f 100644 --- a/etc/pip/components/platform.req +++ b/etc/pip/components/platform.req @@ -3,4 +3,3 @@ pypiwin32>=223; sys_platform == "win32" and python_version > "3" pywin32>=225; sys_platform == "win32" and python_version > "3" cryptography == 2.3; platform_machine == "s390x" or platform_machine == "ppc64le" # Needed for oauthlib to use RSAAlgorithm # Version locked - see SERVER-36618 -cryptography == 36.0.2; platform_machine != "s390x" and platform_machine != "ppc64le"
\ No newline at end of file diff --git a/etc/pip/components/testing.req b/etc/pip/components/testing.req index e2d8e1170e2..120dbbfd183 100644 --- a/etc/pip/components/testing.req +++ b/etc/pip/components/testing.req @@ -11,7 +11,7 @@ itsdangerous == 2.0.0 ocspbuilder == 0.10.2 Werkzeug == 2.0.3 blackduck == 1.0.1 -PyGithub == 1.58 +PyGithub == 1.53 urllib3 >= 1.26.0, <= 1.26.7 distro == 1.5.0 dnspython == 2.1.0 @@ -20,4 +20,3 @@ pkce == 1.0.3 oauthlib == 3.1.1 requests-oauthlib == 1.3.0 packaging <= 21.3 -retry == 0.9.2 diff --git a/etc/repo_config.yaml b/etc/repo_config.yaml index 3579077c2ea..0fa0e5719b5 100644 --- a/etc/repo_config.yaml +++ b/etc/repo_config.yaml @@ -105,14 +105,6 @@ repos: repos: - yum/redhat/8/mongodb-org - yum/redhat/8Server/mongodb-org - - name: rhel90 - type: rpm - edition: org - bucket: repo.mongodb.org - repos: - - yum/redhat/9/mongodb-org - - yum/redhat/9Server/mongodb-org - - name: amazon type: rpm @@ -128,13 +120,6 @@ repos: repos: - yum/amazon/2/mongodb-org - - name: amazon2023 - type: rpm - edition: org - bucket: repo.mongodb.org - repos: - - yum/amazon/2023/mongodb-org - - name: suse11 type: rpm edition: org @@ -264,18 +249,6 @@ repos: repos: - apt/ubuntu/dists/focal/mongodb-org - - name: ubuntu2204 - type: deb - code_name: "jammy" - edition: org - bucket: repo.mongodb.org - component: multiverse - architectures: - - amd64 - - arm64 - repos: - - apt/ubuntu/dists/jammy/mongodb-org - #################### # # Enterprise Repos: @@ -354,14 +327,6 @@ repos: - yum/redhat/8/mongodb-enterprise - yum/redhat/8Server/mongodb-enterprise - - name: rhel90 - type: rpm - edition: enterprise - bucket: repo.mongodb.com - repos: - - yum/redhat/9/mongodb-enterprise - - yum/redhat/9Server/mongodb-enterprise - - name: amazon type: rpm edition: enterprise @@ -376,13 +341,6 @@ repos: repos: - yum/amazon/2/mongodb-enterprise - - name: amazon2023 - type: rpm - edition: enterprise - bucket: repo.mongodb.com - repos: - - yum/amazon/2023/mongodb-enterprise - - name: suse11 type: rpm edition: enterprise @@ -521,15 +479,3 @@ repos: - arm64 repos: - apt/ubuntu/dists/focal/mongodb-enterprise - - - name: ubuntu2204 - type: deb - code_name: "jammy" - edition: enterprise - bucket: repo.mongodb.com - component: multiverse - architectures: - - amd64 - - arm64 - repos: - - apt/ubuntu/dists/jammy/mongodb-enterprise diff --git a/etc/scons/experimental_unified_ninja.vars b/etc/scons/experimental_unified_ninja.vars index fe1f1febe89..21e03e71296 100644 --- a/etc/scons/experimental_unified_ninja.vars +++ b/etc/scons/experimental_unified_ninja.vars @@ -1,9 +1,6 @@ -# This file is now empty and the settings have been made the default. -# This file exists only to prevent breakage when used with existing command line invocations. - # Configures the build for building with a unified ninja # Each configuration will share a ninja log # This allows the output binaries of each configuration to share a common directory -# NINJA_BUILDDIR="$BUILD_ROOT/ninja" -# DESTDIR="$BUILD_ROOT/install" +NINJA_BUILDDIR="$BUILD_ROOT/ninja" +DESTDIR="$BUILD_ROOT/install" diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 7664bf74e89..7fa2948994f 100755 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -1,117 +1,133 @@ command_type: system stepback: false -exec_timeout_secs: &exec_timeout_secs 21600 -timeout_secs: &timeout_secs 7200 -include: - - filename: etc/system_perf_yml_components/tasks.yml - - filename: etc/system_perf_yml_components/variants.yml ## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds) parameters: - key: patch_compile_flags description: "Additional SCons flags to be applied during scons compile invocations in this patch" - # see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects - - key: reuse_compile_from - description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_6.0_90c65f9cc8fc4e6664a5848230abaa9b3f3b02f7" - - key: exec_timeout_secs_override - value: *exec_timeout_secs - description: "Overrides the default exec_timeout_secs defined at the top of this file for the f_dsi_run_workload task" - - key: timeout_secs_override - value: *timeout_secs - description: "Overrides the default timeout_secs defined at the top of this file for the f_dsi_run_workload task" variables: + ### + # Leave this section uncommented to enable compile. + _real_compile_amazon2: &_compile_amazon2 + - name: compile + variant: compile-amazon2 + - name: schedule_global_auto_tasks + variant: task_generation + _real_compile_rhel70: &_compile_rhel70 + - name: compile + variant: compile-rhel70 + - name: schedule_global_auto_tasks + variant: task_generation + _real_expansions: &_expansion_updates + [] + ### + +### +# **Or**: Leave this section uncommented to bypass/skip compile. +# _skip_compile_amazon2: &_compile_amazon2 +# - name: schedule_global_auto_tasks +# variant: task_generation +# _skip_compile_rhel70: &_compile_rhel70 +# - name: schedule_global_auto_tasks +# variant: task_generation +# _skip_expansions: &_expansion_updates +# # This is the normal (amazon2) "compile" artifact from https://evergreen.mongodb.com/version/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9 +# - key: mdb_binary_for_client +# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz +# - key: mdb_binary_for_server +# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz +### + _src_dir: &src_dir src/mongo _modules: &modules + - enterprise - mongo-tools + # - mongo - dsi - genny + - signal-processing - workloads - linkbench - linkbench2 - tsbs - mongo-perf - YCSB + - benchmarks - py-tpcc - PrivateWorkloads - - flamegraph - _shared_compile_expansions: &shared_compile_expansions - platform: linux - project_dir: dsi - scons_cache_scope: shared - scons_cache_mode: all - has_packages: false - -### -# Same in every DSI project. Ensure that this block is synchronized with -# evergreen-dsitest.yml, and src/dsi/onboarding.py (search update-repos-here) -# in the dsi repo, and etc/system_perf.yml in mongodb/mongo. modules: + ### + # Same in every DSI project. Ensure that this block is synchronized with + # evergreen-dsitest.yml, atlas/system_perf_atlas.yml, and src/dsi/onboarding.py + # (search update-repos-here) in this repo, and etc/system_perf.yml and + # etc/perf.yml in mongodb/mongo - name: dsi - owner: 10gen - repo: dsi + repo: git@github.com:10gen/dsi.git prefix: ${workdir}/src branch: master - name: genny - owner: mongodb - repo: genny + repo: git@github.com:10gen/genny.git + prefix: ${workdir}/src + branch: master + - name: signal-processing + repo: git@github.com:10gen/signal-processing.git prefix: ${workdir}/src branch: master - name: workloads - owner: 10gen - repo: workloads + repo: git@github.com:10gen/workloads.git prefix: ${workdir}/src branch: master - name: linkbench - owner: 10gen - repo: linkbench + repo: git@github.com:10gen/linkbench.git prefix: ${workdir}/src branch: master - name: linkbench2 - owner: 10gen - repo: linkbench2 + repo: git@github.com:10gen/linkbench2.git prefix: ${workdir}/src branch: master - name: tsbs - owner: mongodb-forks - repo: tsbs + repo: git@github.com:gregorynoma/tsbs.git prefix: ${workdir}/src - branch: main + branch: master + ref: 225e713010463582712dca30ffcfc70c7d661663 - name: mongo-perf - owner: mongodb - repo: mongo-perf + repo: git@github.com:mongodb/mongo-perf.git prefix: ${workdir}/src branch: master - name: YCSB - owner: mongodb-labs - repo: YCSB + repo: git@github.com:mongodb-labs/YCSB.git prefix: ${workdir}/src branch: production + - name: benchmarks + repo: git@github.com:mongodb-labs/benchmarks.git + prefix: ${workdir}/src + branch: master - name: py-tpcc - owner: mongodb-labs - repo: py-tpcc + repo: git@github.com:mongodb-labs/py-tpcc.git prefix: ${workdir}/src branch: production - - name: flamegraph - owner: mongodb-forks - repo: flamegraph - prefix: ${workdir}/src - branch: master +# - name: mongo +# repo: git@github.com:mongodb/mongo.git +# prefix: ${workdir}/src +# branch: master + ### + - name: enterprise + repo: git@github.com:10gen/mongo-enterprise-modules.git + prefix: src/mongo/db/modules + branch: v6.0 - name: mongo-tools - owner: mongodb - repo: mongo-tools + repo: git@github.com:mongodb/mongo-tools.git prefix: mongo-tools/src/github.com/mongodb branch: master - name: PrivateWorkloads - owner: 10gen - repo: PrivateWorkloads + repo: git@github.com:10gen/PrivateWorkloads.git prefix: ${workdir}/src branch: production -### ### # Same in every DSI project @@ -132,6 +148,45 @@ functions: # Same in every DSI project f_dsi_pre_run: - command: manifest.load + - command: expansions.update + params: + updates: *_expansion_updates + f_run_dsi_workload: + - command: git.get_project + params: + directory: *src_dir + revisions: + dsi: ${dsi_rev} + genny: ${genny_rev} + signal-processing: ${signal-processing_rev} + linkbench: ${linkbench_rev} + linkbench2: ${linkbench2_rev} + tsbs: ${tsbs_rev} + workloads: ${workloads_rev} + mongo-perf: ${mongo-perf_rev} + YCSB: ${YCSB_rev} + benchmarks: ${benchmarks_rev} + py-tpcc: ${py-tpcc_rev} + # mongo: ${mongo_rev} + PrivateWorkloads: ${PrivateWorkloads_rev} + - command: expansions.write + params: + file: ./expansions.yml + - command: shell.exec + params: + script: ./src/dsi/run-dsi run_workload + - command: shell.exec + type: system + params: + script: ./src/dsi/run-dsi determine_failure -m SYSTEM + - command: shell.exec + type: setup + params: + script: ./src/dsi/run-dsi determine_failure -m SETUP + - command: shell.exec + type: test + params: + script: ./src/dsi/run-dsi determine_failure -m TEST f_dsi_post_run: - command: shell.exec params: @@ -167,16 +222,6 @@ functions: permissions: public-read content_type: text/html display_name: Documentation - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: bootstrap.yml - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/bootstrap-${task_name}-${build_id}-${execution}.yml - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Task Bootstrap Config f_dsi_timeout: - command: shell.exec params: @@ -221,63 +266,33 @@ functions: set -o errexit set -o verbose - mkdir -p mongodb/bin - /opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python3 "${workdir}/compile_venv" source "${workdir}/compile_venv/bin/activate" python -m pip install -r etc/pip/compile-requirements.txt - - command: expansions.write - params: - file: expansions.yml - redacted: true + - command: shell.exec params: working_dir: src script: | - if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then - set -o errexit - set -o verbose - - source "${workdir}/compile_venv/bin/activate" + set -o errexit + set -o verbose - # We get the raw version string (r1.2.3-45-gabcdef) from git - # And append "-sys-perf" at the end - export MONGO_VERSION=$(git describe --abbrev=7) - MONGO_VERSION="$MONGO_VERSION-sys-perf" + mkdir -p mongodb/bin - # If this is a patch build, we add the patch version id to the version string so we know - # this build was a patch, and which evergreen task it came from - if [ "${is_patch|false}" = "true" ]; then - MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" - fi + # We get the raw version string (r1.2.3-45-gabcdef) from git + MONGO_VERSION=$(git describe --abbrev=7) - # This script handles sanitizing the version string for use during SCons build - # and when pushing artifacts up to S3. - IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \ - buildscripts/generate_version_expansions.py --out version_expansions.yml - else - touch version_expansions.yml - fi - - command: expansions.update - params: - file: src/version_expansions.yml - - command: shell.exec - params: - working_dir: src - script: | - if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then - set -o errexit - set -o verbose - - # This script handles whether the SCons cache should be used - source "${workdir}/compile_venv/bin/activate" - SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \ - IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \ - python buildscripts/generate_compile_expansions.py --out compile_expansions.yml - else - touch compile_expansions.yml + # If this is a patch build, we add the patch version id to the version string so we know + # this build was a patch, and which evergreen task it came from + if [ "${is_patch|false}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" fi + + # This script converts the generated version string into a sanitized version string for + # use by scons and uploading artifacts as well as information about for the scons cache. + source "${workdir}/compile_venv/bin/activate" + MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} python buildscripts/generate_compile_expansions.py --out compile_expansions.yml - command: expansions.update params: file: src/compile_expansions.yml @@ -285,118 +300,170 @@ functions: params: working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools script: | - if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then - set -o verbose - set -o errexit - - # make sure newlines in the scripts are handled correctly by windows - if [ "Windows_NT" = "$OS" ]; then - set -o igncr - fi; - - # set_goenv provides set_goenv() - . ./set_goenv.sh - GOROOT="" set_goenv || exit - go version - - build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop" - if [ "${build_mongoreplay}" = "true" ]; then - build_tools="$build_tools mongoreplay" - fi - for i in $build_tools; do - go build -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go - "../../../../../mongodb/bin/$i${exe|}" --version - done - fi - - command: shell.exec - params: - working_dir: src - script: | - if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then - set -o errexit - set -o verbose - source "${workdir}/compile_venv/bin/activate" - python ./buildscripts/idl/gen_all_feature_flag_list.py - mkdir -p mongodb/feature_flags - cp ./all_feature_flags.txt mongodb/feature_flags - fi - - command: shell.exec - params: - working_dir: src - script: | - if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then - set -o errexit - set -o verbose - source "${workdir}/compile_venv/bin/activate" - python ./buildscripts/idl/gen_all_server_params_list.py - mkdir -p mongodb/server_params - cp ./all_server_params.txt mongodb/server_params + set -o verbose + set -o errexit + + # make sure newlines in the scripts are handled correctly by windows + if [ "Windows_NT" = "$OS" ]; then + set -o igncr + fi; + + # set_goenv provides set_goenv(), print_ldflags() and print_tags() used below + . ./set_goenv.sh + GOROOT="" set_goenv || exit + go version + + build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop" + if [ "${build_mongoreplay}" = "true" ]; then + build_tools="$build_tools mongoreplay" fi + for i in $build_tools; do + go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go + "../../../../../mongodb/bin/$i${exe|}" --version + done - command: shell.exec params: working_dir: src script: | - if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then - set -o errexit - set -o verbose - source "${workdir}/compile_venv/bin/activate" - python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|} - mkdir -p mongodb/jstests/hooks - if [ -d jstests/hooks ] - then - echo "Fetching JS test DB correctness checks from directory jstests" - cp -a jstests/* mongodb/jstests - - echo "Now adding our own special run_validate_collections.js wrapper" - mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js - - cat << EOF > mongodb/jstests/hooks/run_validate_collections.js - print("NOTE: run_validate_collections.js will skip the oplog!"); - TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; - load('jstests/hooks/run_validate_collections.actual.js'); - EOF - fi - tar czf mongodb${compile_variant|}.tar.gz mongodb - # Put all matching mongo debug from the build directory in an archive in the same location - # as the library archive (i.e. mongodb/bin). - tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:' - fi + set -o errexit + set -o verbose + source "${workdir}/compile_venv/bin/activate" + python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src + mkdir -p mongodb/feature_flags + cp ./all_feature_flags.txt mongodb/feature_flags - command: shell.exec params: working_dir: src script: | - if [ -n "${reuse_compile_from}" ] && [ "${is_patch|false}" = "true" ]; then - set -o errexit - set -o verbose + set -o errexit + set -o verbose + source "${workdir}/compile_venv/bin/activate" + python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|} + mkdir -p mongodb/jstests/hooks + if [ -d jstests/hooks ] + then + echo "Fetching JS test DB correctness checks from directory jstests" + cp -a jstests/* mongodb/jstests + + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js - source "${workdir}/compile_venv/bin/activate" - python buildscripts/download_sys_perf_binaries.py -v ${reuse_compile_from} -b ${build_variant} -u ${evergreen_api_user} -k ${evergreen_api_key} - mv binary.tar.gz mongodb${compile_variant|}.tar.gz + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF fi + tar czf mongodb${compile-variant|}.tar.gz mongodb - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb${compile_variant|}.tar.gz - remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}.tar.gz + local_file: src/mongodb${compile-variant|}.tar.gz + remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile-variant|}-${version_id}.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} - display_name: mongodb${compile_variant|}.tar.gz - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb${compile_variant|}-debugsymbols.tar.gz - remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}-debugsymbols.tar.gz - bucket: mciuploads - permissions: public-read - optional: true - content_type: ${content_type|application/x-gzip} - display_name: mongo-debugsymbols.tgz + display_name: mongodb${compile-variant|}.tar.gz ### + ## Schedule Tasks ## + f_schedule_tasks: + - command: git.get_project + params: + directory: *src_dir + revisions: + dsi: ${dsi_rev} + genny: ${genny_rev} + signal-processing: ${signal-processing_rev} + linkbench: ${linkbench_rev} + linkbench2: ${linkbench2_rev} + tsbs: ${tsbs_rev} + workloads: ${workloads_rev} + mongo-perf: ${mongo-perf_rev} + YCSB: ${YCSB_rev} + benchmarks: ${benchmarks_rev} + py-tpcc: ${py-tpcc_rev} + PrivateWorkloads: ${PrivateWorkloads_rev} + - command: expansions.write + params: + file: ./expansions.yml + - command: shell.exec + params: + script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks} + - command: generate.tasks + params: + files: + - build/TaskJSON/Tasks.json + + tasks: + ### + # Same in every DSI project + - name: schedule_global_auto_tasks + priority: 5 + commands: + - func: f_schedule_tasks + vars: + tasks: all_tasks + - name: schedule_variant_auto_tasks + priority: 5 + commands: + - func: f_schedule_tasks + vars: + tasks: variant_tasks + - name: schedule_patch_auto_tasks + priority: 5 + commands: + - func: f_schedule_tasks + vars: + tasks: patch_tasks + - name: smoke_test + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + - name: smoke_test_ssl + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: replica-ssl + infrastructure_provisioning: replica + - name: smoke_test_standalone_auth + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: standalone-auth + infrastructure_provisioning: single + - name: smoke_test_replset_auth + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: replica-auth + infrastructure_provisioning: replica + - name: smoke_test_shard_lite_auth + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: short + mongodb_setup: shard-lite-auth + infrastructure_provisioning: shard-lite + - name: dsi_integ_test_run_command_simple + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: run_command_simple + ### + - name: compile commands: - command: manifest.load @@ -404,13 +471,507 @@ tasks: params: directory: src revisions: + enterprise: ${enterprise_rev} mongo-tools: ${mongo-tools_rev} - func: "compile mongodb" + - name: linkbench + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "linkbench" + + - name: linkbench_stepdowns + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "linkbench_stepdowns" + + - name: linkbench_rolling_restarts + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "linkbench_rolling_restarts" + + - name: linkbench_non_retryable_writes_stepdowns + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "linkbench_non_retryable_writes_stepdowns" + + - name: linkbench_non_retryable_writes_rolling_restarts + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "linkbench_non_retryable_writes_rolling_restarts" + + - name: linkbench2 + priority: 5 + exec_timeout_secs: 43200 # 12 hours + commands: + - func: f_run_dsi_workload + vars: + test_control: "linkbench2" + additional_tfvars: "tags: {expire-on-delta: 12}" + + - name: tsbs_load + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "tsbs_load" + + - name: tsbs_query + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "tsbs_query" + + - name: tsbs_query_manual_bucketing + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "tsbs_query_manual_bucketing" + + - name: tpcc + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "tpcc" + + - name: industry_benchmarks + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb" + + - name: ycsb_60GB + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb-60GB" + + - name: industry_benchmarks_secondary_reads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb-secondary-reads" + + - name: industry_benchmarks_wmajority + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb-wmajority" + + - name: industry_benchmarks_stepdowns + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb_stepdowns" + + - name: industry_benchmarks_rolling_restarts + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb_rolling_restarts" + + - name: industry_benchmarks_non_retryable_writes_stepdowns + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb_non_retryable_writes_stepdowns" + + - name: industry_benchmarks_non_retryable_writes_rolling_restarts + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "ycsb_non_retryable_writes_rolling_restarts" + + - name: crud_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "crud_workloads" + + - name: crud_workloads_majority + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "crud_workloads_majority" + + - name: cursor_manager + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "cursor_manager" + + - name: mixed_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "mixed_workloads" + + - name: mixed_workloads_genny_stepdowns + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "mixed_workloads_genny_stepdowns" + + - name: mixed_workloads_genny_rolling_restarts + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "mixed_workloads_genny_rolling_restarts" + + - name: big_update_10k + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "BigUpdate10k" + + - name: misc_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "misc_workloads" + + + - name: map_reduce_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "map_reduce_workloads" + + - name: genny_canaries + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "genny_canaries" + + - name: retryable_writes_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "retryable_writes" + + - name: snapshot_reads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "snapshot_reads" + + - name: secondary_reads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "secondary_reads" + + - name: bestbuy_agg + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "bestbuy_agg" + + - name: bestbuy_agg_merge_same_db + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "bestbuy_agg_merge_same_db" + + - name: bestbuy_agg_merge_different_db + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "bestbuy_agg_merge_different_db" + + - name: bestbuy_agg_merge_target_hashed + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "bestbuy_agg_merge_target_hashed" + + - name: bestbuy_agg_merge_wordcount + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "bestbuy_agg_merge_wordcount" + + - name: bestbuy_query + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "bestbuy_query" + + - name: tpch_1_normalized + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "tpch" + test_control_params: | + {scale: 1, + schema: normalized} + + - name: tpch_1_denormalized + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "tpch" + test_control_params: | + {scale: 1, + schema: denormalized} + + - name: non_sharded_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "non_sharded" + + - name: mongos_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "mongos" + + - name: mongos_large_catalog_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "mongos_large_catalog" + + - name: move_chunk_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "move_chunk" + + - name: move_chunk_waiting_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "move_chunk_waiting" + + - name: move_chunk_large_chunk_map_workloads + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "move_chunk_large_chunk_map" + + - name: refine_shard_key_transaction_stress + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "refine_shard_key_transaction_stress" + + - name: secondary_performance + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + # Unfortunately the dash/underscore style is different for mongodb_setup and test_control + test_control: "secondary_performance" + mongodb_setup: "secondary-performance" + + - name: initialsync + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync" + + - name: initialsync-fcbis + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync" + mongodb_setup: "replica-2node-fcbis" + + - name: initialsync-logkeeper-short + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync-logkeeper" + mongodb_setup: "initialsync-logkeeper-short" + # Logkeeper dataset with FCV set to 6.0 + mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.0.tgz" + + - name: initialsync-logkeeper-short-fcbis + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync-logkeeper" + mongodb_setup: "initialsync-logkeeper-short-fcbis" + # Logkeeper dataset with FCV set to 6.0 + mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.0.tgz" + + - name: initialsync-logkeeper + priority: 5 + exec_timeout_secs: 43200 # 12 hours + commands: + - func: f_run_dsi_workload + timeout_secs: 43200 # 12 hours + vars: + test_control: "initialsync-logkeeper" + + - name: initialsync-logkeeper-fcbis + priority: 5 + exec_timeout_secs: 43200 # 12 hours + commands: + - func: f_run_dsi_workload + timeout_secs: 43200 # 12 hours + vars: + test_control: "initialsync-logkeeper" + mongodb_setup: "initialsync-logkeeper-fcbis" + + # The following two initial sync logkeeper automation tasks are only used in the commented-out + # "Linux ReplSet Initial Sync LogKeeper Snapshot Update" variant below and are only intended to be + # run in patch builds to update FCV for logkeeper datasets. + - name: initialsync-logkeeper-short-s3-update + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync-logkeeper-short-s3-update" + mongodb_setup: "initialsync-logkeeper-short-s3-update" + # Update this to Logkeeper dataset with FCV set to latest after each LTS release. + mongodb_dataset: "https://s3-us-west-2.amazonaws.com/dsi-donot-remove/InitialSyncLogKeeper/logkeeper-slice-data-mongodb-6.0.tgz" + + - name: initialsync-logkeeper-snapshot-update + priority: 5 + exec_timeout_secs: 216000 # 2.5 days + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync-logkeeper-snapshot-update" + + - name: initialsync-large + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync-large" + + - name: initialsync-large-fcbis + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "initialsync-large" + mongodb_setup: "replica-2node-fcbis" + + - name: change_streams_throughput + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "change_streams_throughput" + + - name: change_streams_latency + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "change_streams_latency" + + - name: change_streams_preimage_throughput + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "change_streams_preimage_throughput" + + - name: change_streams_preimage_latency + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "change_streams_preimage_latency" + + - name: change_streams_listen_throughput + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "change_streams_listen_throughput" + + - name: change_streams_multi_mongos + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "change_streams_multi_mongos" + + - name: sb_large_scale + priority: 5 + exec_timeout_secs: 43200 # 12 hours + commands: + - func: f_run_dsi_workload + vars: + test_control: "sb_large_scale" + additional_tfvars: "tags: {expire-on-delta: 12}" + + - name: sb_timeseries + priority: 5 + commands: + - func: f_run_dsi_workload + vars: + test_control: "sb_timeseries" + + buildvariants: - name: task_generation display_name: Task Generation + cron: "0 0 1 1 *" # Every year starting 1/1 at 00:00 modules: *modules + expansions: + platform: linux + project_dir: dsi run_on: - amazon2-build tasks: @@ -419,8 +980,8 @@ buildvariants: - &compile-amazon2 name: compile-amazon2 display_name: Compile - cron: "0 4 21 * *" # 21st day of every month at 04:00 modules: *modules + cron: "0 0 * * *" # Everyday at 00:00 expansions: &compile-expansions platform: linux project_dir: &project_dir dsi @@ -439,15 +1000,782 @@ buildvariants: tasks: - name: compile - - &compile-amazon-linux2-arm64 - name: compile-amazon-linux2-arm64 - display_name: Compile for Amazon Linux 2 arm64 - cron: "0 4 21 * *" # 21st day of every month at 04:00 + - name: linux-standalone + display_name: Linux Standalone + cron: "0 0 * * *" # Everyday at 00:00 + modules: *modules + expansions: + mongodb_setup: standalone + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: &standalonetasks + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: dsi_integ_test_run_command_simple + - name: smoke_test + - name: industry_benchmarks + - name: ycsb_60GB + - name: crud_workloads + - name: genny_canaries + - name: cursor_manager + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: non_sharded_workloads + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: tpch_1_normalized + - name: tpch_1_denormalized + + - name: compile-rhel70 + display_name: Compile for Atlas-like modules: *modules + cron: "0 0 * * *" # Everyday at 00:00 expansions: <<: *compile-expansions - compile_variant: -arm64 + compile_flags: >- + --ssl + --separate-debug + MONGO_DISTMOD=rhel70 + -j$(grep -c ^processor /proc/cpuinfo) + --release + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + compile-variant: -rhel70 run_on: - - "amazon2-arm64-large" + - rhel70-large tasks: - name: compile + + - name: linux-1-node-replSet + display_name: Linux 1-Node ReplSet + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: single-replica + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: &1nodetasks + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: ycsb_60GB + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: smoke_test + - name: retryable_writes_workloads + - name: non_sharded_workloads + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: snapshot_reads + - name: linkbench + - name: linkbench2 + - name: tsbs_load + - name: tsbs_query + - name: tsbs_query_manual_bucketing + - name: tpcc + - name: tpch_1_normalized + - name: tpch_1_denormalized + - name: industry_benchmarks_wmajority + - name: sb_large_scale + - name: sb_timeseries + + - name: linux-standalone-audit + display_name: Linux Standalone Audit + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: standalone-audit + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: crud_workloads + - name: smoke_test + + - name: linux-1-node-replSet-fle + display_name: Linux 1-Node ReplSet FLE + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: single-replica-fle + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + fle: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: linkbench + + - name: linux-1-node-replSet-cwrwc + display_name: Linux 1-Node ReplSet CWRWC + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: single-replica-cwrwc + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: linkbench + + - name: linux-1-node-replSet-ese-cbc + display_name: Linux 1-Node ReplSet ESE CBC + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: single-replica-ese-cbc + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: smoke_test + - name: ycsb_60GB + + - name: linux-1-node-replSet-ese-gcm + display_name: Linux 1-Node ReplSet ESE GCM + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: single-replica-ese-gcm + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: smoke_test + - name: ycsb_60GB + + - name: linux-1-node-15gbwtcache + display_name: Linux 1-Node ReplSet 15 GB WiredTiger Cache + cron: "0 0 * * 0,2,4" # 00:00 on Sunday,Tuesday,Thursday + modules: *modules + expansions: + mongodb_setup: single-replica-15gbwtcache + infrastructure_provisioning: single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-single" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: smoke_test + + - name: linux-3-node-1dayhistory-15gbwtcache + display_name: Linux 3-Node ReplSet 1 Day History 15 GB WiredTiger Cache + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: replica-1dayhistory-15gbwtcache + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: crud_workloads + - name: crud_workloads_majority + - name: smoke_test + + - name: linux-3-shard + display_name: Linux 3-Shard Cluster + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: shard + infrastructure_provisioning: shard + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-shard" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: smoke_test + - name: industry_benchmarks_wmajority + - name: mongos_workloads + - name: mongos_large_catalog_workloads + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: change_streams_multi_mongos + + - name: linux-shard-lite-audit + display_name: Linux Shard Lite Cluster Audit + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: shard-lite-audit + infrastructure_provisioning: shard-lite + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-shard-lite" + depends_on: *_compile_amazon2 + tasks: + - name: industry_benchmarks + + - name: linux-shard-lite + display_name: Linux Shard Lite Cluster + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: shard-lite + infrastructure_provisioning: shard-lite + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-shard-lite" + depends_on: *_compile_amazon2 + tasks: &shardlitetasks + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_target_hashed + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: change_streams_latency + - name: change_streams_throughput + - name: change_streams_listen_throughput + - name: industry_benchmarks + - name: industry_benchmarks_wmajority + - name: linkbench + - name: mixed_workloads + - name: mongos_workloads + - name: mongos_large_catalog_workloads + - name: move_chunk_large_chunk_map_workloads + - name: move_chunk_workloads + - name: move_chunk_waiting_workloads + - name: retryable_writes_workloads + - name: smoke_test + + - name: linux-shard-lite-cwrwc + display_name: Linux Shard Lite Cluster CWRWC + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: shard-lite-cwrwc + infrastructure_provisioning: shard-lite + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-shard-lite" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: linkbench + + - name: linux-shard-lite-read-concern-available + display_name: Linux Shard Lite ReadConcern Available + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: shard-lite-read-concern-available + infrastructure_provisioning: shard-lite + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-shard-lite" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: linkbench + + - name: linux-shard-single + display_name: Linux Shard Single + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: shard-single + infrastructure_provisioning: shard-single + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-shard-lite" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + + - name: linux-3-node-replSet + display_name: Linux 3-Node ReplSet + cron: "0 0 * * *" # Everyday at 00:00 + modules: *modules + expansions: + mongodb_setup: replica + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: &3nodetasks + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: ycsb_60GB + - name: industry_benchmarks_secondary_reads + - name: crud_workloads + - name: crud_workloads_majority + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: refine_shard_key_transaction_stress + - name: smoke_test + - name: retryable_writes_workloads + - name: industry_benchmarks_wmajority + - name: secondary_performance # Uses a special 2 node mongodb setup + - name: non_sharded_workloads + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: snapshot_reads + - name: secondary_reads + - name: tpcc + - name: tpch_1_normalized + - name: tpch_1_denormalized + - name: linkbench + - name: linkbench2 + - name: tsbs_load + - name: tsbs_query + - name: tsbs_query_manual_bucketing + - name: sb_large_scale + - name: sb_timeseries + - name: big_update_10k + + - name: linux-3-node-replSet-last-continuous-fcv + display_name: Linux 3-Node ReplSet (Last Continuous FCV) + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: replica-last-continuous-fcv + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: ycsb_60GB + - name: industry_benchmarks_secondary_reads + - name: crud_workloads + - name: crud_workloads_majority + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: refine_shard_key_transaction_stress + - name: smoke_test + - name: retryable_writes_workloads + - name: industry_benchmarks_wmajority + - name: secondary_performance # Uses a special 2 node mongodb setup + - name: non_sharded_workloads + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: snapshot_reads + - name: secondary_reads + - name: tpcc + - name: tpch_1_normalized + - name: tpch_1_denormalized + - name: linkbench + - name: linkbench2 + # Time-series collections are available since v5.0. + # - name: tsbs_load + # - name: tsbs_query + # - name: tsbs_query_manual_bucketing + - name: sb_large_scale + - name: sb_timeseries + + - name: linux-3-node-replSet-last-lts-fcv + display_name: Linux 3-Node ReplSet (Last LTS FCV) + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: replica-last-lts-fcv + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: ycsb_60GB + - name: industry_benchmarks_secondary_reads + - name: crud_workloads + - name: crud_workloads_majority + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: refine_shard_key_transaction_stress + - name: smoke_test + - name: retryable_writes_workloads + - name: industry_benchmarks_wmajority + - name: secondary_performance # Uses a special 2 node mongodb setup + - name: non_sharded_workloads + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: snapshot_reads + - name: secondary_reads + - name: tpcc + - name: tpch_1_normalized + - name: tpch_1_denormalized + - name: linkbench + - name: linkbench2 + # Time-series collections are available since v5.0. + # - name: tsbs_load + # - name: tsbs_query + # - name: tsbs_query_manual_bucketing + - name: sb_large_scale + - name: sb_timeseries + + - name: linux-3-node-replSet-noflowcontrol + display_name: Linux 3-Node ReplSet (Flow Control off) + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: replica-noflowcontrol + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: industry_benchmarks_secondary_reads + - name: crud_workloads + - name: crud_workloads_majority + - name: mixed_workloads + - name: smoke_test + - name: industry_benchmarks_wmajority + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: tpcc + - name: linkbench + - name: linkbench2 + + - name: linux-3-node-replSet-ssl + display_name: Linux 3-Node ReplSet (SSL) + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: replica-ssl + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: mixed_workloads + + - name: linux-3-node-replSet-maintenance-events + display_name: Linux 3-Node ReplSet (Maintenance Events) + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: replica-maintenance-events + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks_stepdowns + - name: industry_benchmarks_rolling_restarts + - name: industry_benchmarks_non_retryable_writes_stepdowns + - name: industry_benchmarks_non_retryable_writes_rolling_restarts + - name: linkbench_stepdowns + - name: linkbench_rolling_restarts + - name: linkbench_non_retryable_writes_stepdowns + - name: linkbench_non_retryable_writes_rolling_restarts + - name: mixed_workloads_genny_stepdowns + - name: mixed_workloads_genny_rolling_restarts + + - name: linux-3-node-replSet-initialsync + display_name: Linux 3-Node ReplSet Initial Sync + cron: "0 0 * * 0,2,4" # 00:00 on Sunday,Tuesday,Thursday + modules: *modules + expansions: + mongodb_setup: replica-2node + infrastructure_provisioning: replica + platform: linux + authentication: disabled + storageEngine: wiredTiger + project_dir: *project_dir + depends_on: *_compile_amazon2 + run_on: + - "rhel70-perf-replset" + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: initialsync + - name: initialsync-fcbis + - name: initialsync-logkeeper-short + - name: initialsync-logkeeper-short-fcbis + - name: initialsync-large + - name: initialsync-large-fcbis + + - name: linux-replSet-initialsync-logkeeper + display_name: Linux ReplSet Initial Sync LogKeeper + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: initialsync-logkeeper + infrastructure_provisioning: initialsync-logkeeper + # EBS logkeeper snapshot with FCV set to 6.0 + snapshotId: snap-0306dce35f030ebec + platform: linux + authentication: disabled + storageEngine: wiredTiger + project_dir: *project_dir + run_on: + - "rhel70-perf-initialsync-logkeeper" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: initialsync-logkeeper + - name: initialsync-logkeeper-fcbis + + # Uncomment this to run logkeeper FCV updates automatically. This is only intended to be run in + # patch builds. + #- name: linux-replSet-initialsync-logkeeper-snapshot-update + # display_name: Linux ReplSet Initial Sync LogKeeper Snapshot Update + # cron: "0 0 * * 4" # 00:00 on Thursday + # modules: *modules + # expansions: + # mongodb_setup: initialsync-logkeeper-snapshot-update + # infrastructure_provisioning: initialsync-logkeeper-snapshot-update + # # Update this to latest snapshot after each LTS release. + # snapshotId: snap-0306dce35f030ebec + # platform: linux + # authentication: disabled + # storageEngine: wiredTiger + # project_dir: *project_dir + # run_on: + # - "rhel70-perf-initialsync-logkeeper" + # depends_on: *_compile_amazon2 + # tasks: + # - name: schedule_patch_auto_tasks + # - name: schedule_variant_auto_tasks + # - name: initialsync-logkeeper-snapshot-update + # - name: initialsync-logkeeper-short-s3-update + + - name: linux-replSet-audit + display_name: Linux 3-Node ReplSet Audit + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: replica-audit + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + + - name: linux-replSet-auth-delay + display_name: Linux 3-Node ReplSet (Auth Delay) + cron: "0 0 * * 4" # 00:00 on Thursday + modules: *modules + expansions: + mongodb_setup: replica-auth-cluster-delay + infrastructure_provisioning: replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + run_on: + - "rhel70-perf-replset" + depends_on: *_compile_amazon2 + tasks: + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + + - name: atlas-like-M60 + display_name: M60-Like 3-Node ReplSet + cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday + modules: *modules + expansions: + mongodb_setup: atlas-like-replica + infrastructure_provisioning: M60-like-replica + platform: linux + project_dir: *project_dir + authentication: enabled + storageEngine: wiredTiger + compile-variant: -rhel70 + run_on: + - "rhel70-perf-M60-like" + depends_on: *_compile_rhel70 + tasks: # Cannot use *3nodetasks because secondary_performance uses a special mongodb setup. + - name: schedule_patch_auto_tasks + - name: schedule_variant_auto_tasks + - name: industry_benchmarks + - name: ycsb_60GB + - name: industry_benchmarks_secondary_reads + - name: crud_workloads + - name: crud_workloads_majority + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: smoke_test + - name: retryable_writes_workloads + - name: industry_benchmarks_wmajority + - name: non_sharded_workloads + - name: bestbuy_agg + - name: bestbuy_agg_merge_different_db + - name: bestbuy_agg_merge_same_db + - name: bestbuy_agg_merge_wordcount + - name: bestbuy_query + - name: change_streams_throughput + - name: change_streams_latency + - name: change_streams_listen_throughput + - name: snapshot_reads + - name: secondary_reads + - name: tpcc + - name: linkbench + - name: linkbench2 diff --git a/etc/system_perf_yml_components/tasks.yml b/etc/system_perf_yml_components/tasks.yml deleted file mode 100644 index be0aa910373..00000000000 --- a/etc/system_perf_yml_components/tasks.yml +++ /dev/null @@ -1,927 +0,0 @@ -functions: - f_dsi_run_workload: &dsi_run_func # this function replaces f_run_dsi_workload - - command: timeout.update - params: - exec_timeout_secs: ${exec_timeout_secs_override} - timeout_secs: ${timeout_secs_override} - - command: git.get_project - params: - directory: src/mongo - clone_depth: 1000 - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - YCSB: ${YCSB_rev} - flamegraph: ${flamegraph_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - redacted: true - - command: shell.exec - params: - script: ./src/dsi/run-dsi run_workload - - command: shell.exec - type: system - params: - script: ./src/dsi/run-dsi determine_failure -m SYSTEM - - command: shell.exec - type: setup - params: - script: ./src/dsi/run-dsi determine_failure -m SETUP - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi determine_failure -m TEST - - f_run_dsi_workload: *dsi_run_func # Do not use this function. It is deprecated. - - ## Schedule Tasks ## - f_schedule_tasks: - - command: git.get_project - params: - directory: src/mongo - clone_depth: 1000 - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - mongo-perf: ${mongo-perf_rev} - YCSB: ${YCSB_rev} - py-tpcc: ${py-tpcc_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - - command: shell.exec - params: - script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks} - - command: generate.tasks - params: - files: - - build/TaskJSON/Tasks.json - -tasks: - ### - # Same in every DSI project - - name: schedule_global_auto_tasks - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: all_tasks - - name: schedule_variant_auto_tasks - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: variant_tasks - - name: schedule_patch_auto_tasks - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: patch_tasks - - name: smoke_test - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - - name: canaries_only - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: canaries - - name: smoke_test_ssl - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: replica-ssl - infrastructure_provisioning: replica - - name: smoke_test_standalone_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: standalone-auth - infrastructure_provisioning: single - - name: smoke_test_replset_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: replica-auth - infrastructure_provisioning: replica - - name: smoke_test_shard_lite_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: shard-lite-auth - infrastructure_provisioning: shard-lite - ### - - - name: linkbench - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench" - - - name: linkbench_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_stepdowns" - - - name: linkbench_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_rolling_restarts" - - - name: linkbench_non_retryable_writes_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_non_retryable_writes_stepdowns" - - - name: linkbench_non_retryable_writes_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_non_retryable_writes_rolling_restarts" - - - name: linkbench2 - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench2" - additional_tfvars: "tags: {expire-on-delta: 12}" - - - name: tsbs_load - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_load" - - - name: tsbs_query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query" - - - name: tsbs_query_manual_bucketing - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_manual_bucketing" - - - name: tpcc - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpcc" - - - name: tpcc_majority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpcc_majority" - - - name: industry_benchmarks - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb" - - - name: ycsb_60GB - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB" - - - name: industry_benchmarks_secondary_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-secondary-reads" - - - name: industry_benchmarks_wmajority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-wmajority" - - - name: industry_benchmarks_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_stepdowns" - - - name: industry_benchmarks_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_rolling_restarts" - - - name: industry_benchmarks_non_retryable_writes_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_stepdowns" - - - name: industry_benchmarks_non_retryable_writes_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_rolling_restarts" - - - name: ycsb.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb.2023-09" - - - name: ycsb_w1.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-w1.2023-09" - - - name: ycsb_60GB.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.2023-09" - - - name: ycsb_60GB.long.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.long.2023-09" - - - name: ycsb_secondary_reads.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-secondary-reads.2023-09" - - - name: ycsb.load - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb.load" - - - name: crud_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads" - - - name: crud_workloads_majority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads_majority" - - - name: crud_workloads_w1 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads_w1.2023-02" - - - name: cursor_manager - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "cursor_manager" - - - name: mixed_workloads_genny_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mixed_workloads_genny_stepdowns" - - - name: mixed_workloads_genny_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mixed_workloads_genny_rolling_restarts" - - - name: big_update_10k - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "BigUpdate10k" - - - name: misc_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "misc_workloads" - - - - name: map_reduce_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "map_reduce_workloads" - - - name: genny_canaries - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "genny_canaries" - - - name: retryable_writes_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "retryable_writes" - - - name: snapshot_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "snapshot_reads" - - - name: secondary_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "secondary_reads" - - - name: bestbuy_agg - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg" - - - name: bestbuy_agg_merge_same_db - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_same_db" - - - name: bestbuy_agg_merge_different_db - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_different_db" - - - name: bestbuy_agg_merge_target_hashed - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_target_hashed" - - - name: bestbuy_agg_merge_wordcount - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_wordcount" - - - name: bestbuy_query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_query" - - - name: tpch_1_normalized - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 1, - schema: normalized} - - - name: tpch_1_denormalized - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 1, - schema: denormalized} - - - name: tpch_10_normalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 10, - schema: normalized} - - - name: tpch_10_denormalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 10, - schema: denormalized} - - - name: non_sharded_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "non_sharded" - - - name: mongos_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mongos" - - - name: mongos_large_catalog_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mongos_large_catalog" - - - name: move_chunk_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk" - - - name: move_chunk_waiting_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk_waiting" - - - name: move_chunk_large_chunk_map_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk_large_chunk_map" - - - name: refine_shard_key_transaction_stress - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "refine_shard_key_transaction_stress" - - - name: secondary_performance - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - # Unfortunately the dash/underscore style is different for mongodb_setup and test_control - test_control: "secondary_performance" - mongodb_setup: "secondary-performance" - - - name: initialsync - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync" - - - name: initialsync-fcbis - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync" - mongodb_setup: "replica-2node-fcbis" - - - name: initialsync-logkeeper - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "initialsync-logkeeper" - - - name: initialsync-logkeeper-fcbis - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "initialsync-logkeeper" - mongodb_setup: "initialsync-logkeeper-fcbis" - - # The following two initial sync logkeeper automation tasks are only used in the commented-out - # "Linux ReplSet Initial Sync LogKeeper Snapshot Update" variant below and are only intended to be - # run in patch builds to update FCV for logkeeper datasets. - - - name: initialsync-logkeeper-snapshot-update - priority: 5 - exec_timeout_secs: 216000 # 2.5 days - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-logkeeper-snapshot-update" - - - name: initialsync-large - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-large" - - - name: initialsync-large-fcbis - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-large" - mongodb_setup: "replica-2node-fcbis" - - - name: change_streams_latency - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_latency" - - - name: change_streams_preimage_throughput - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_preimage_throughput" - - - name: change_streams_preimage_latency - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_preimage_latency" - - - name: change_streams_listen_throughput - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_listen_throughput" - - - name: change_streams_multi_mongos - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_multi_mongos" - - - name: genny_execution_UserAcquisition - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/execution/UserAcquisition.yml - - name: genny_scale_InsertRemove - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/InsertRemove.yml - - name: query_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: query_read_commands_large_dataset - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query_large_dataset, - include_filter_2: regression, - exclude_filter: none, - threads: "1 4", - read_cmd: 'true', - share_dataset: 'true'} - - name: big_collection - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query, - include_filter_2: getmore, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: views-query - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query_identityview, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: views-aggregation - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation_identityview, - include_filter_2: regression, - exclude_filter: none, - threads: "1", - read_cmd: 'true'} - - name: where_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: where, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: update_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: update, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: insert_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: insert, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - # - name: compound-wildcard-index-insert - # commands: - # - func: f_dsi_run_workload - # vars: - # test_control: mongo-perf.2023-02 - # test_control_params: | - # {include_filter_1: compound-wildcard-insert, - # include_filter_2: core regression, - # exclude_filter: single_threaded, - # threads: "1 2 4 8", - # read_cmd: 'true'} - - name: wildcard-index-read_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: wildcard_read, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: wildcard-index-write_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: wildcard_write, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: geo_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: geo, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_default_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-default.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_slow_or_sample_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-slow-or-sample.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_complex_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-complex.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_whole_doc_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-whole-doc.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_slowms_everything_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-slowms-everything.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: singleThreaded_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: single_threaded, - include_filter_2: core regression, - exclude_filter: none, - threads: "1", - read_cmd: 'true'} - - name: aggregation_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation, - include_filter_2: regression, - exclude_filter: js, - threads: "1", - read_cmd: 'true'} - - name: aggregation_read_commands_large_dataset - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation_large_dataset, - include_filter_2: regression, - exclude_filter: js, - threads: "1 4", - read_cmd: 'true', - share_dataset: 'true'} - - name: agg-query-comparison_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: agg_query_comparison, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: pipeline-updates - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: pipeline-updates, - include_filter_2: regression, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: javascript - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: js, - include_filter_2: aggregation, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} diff --git a/etc/system_perf_yml_components/variants.yml b/etc/system_perf_yml_components/variants.yml deleted file mode 100644 index c2a5a78b149..00000000000 --- a/etc/system_perf_yml_components/variants.yml +++ /dev/null @@ -1,272 +0,0 @@ -variables: - _modules: &modules - - mongo-tools - - dsi - - genny - - workloads - - linkbench - - linkbench2 - - tsbs - - mongo-perf - - YCSB - - py-tpcc - - PrivateWorkloads - - flamegraph - _project_dir: &project_dir dsi - _compile_amazon2: &_compile_amazon2 - - name: compile - variant: compile-amazon2 - - name: schedule_global_auto_tasks - variant: task_generation - _compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64 - - name: compile - variant: compile-amazon-linux2-arm64 - - name: schedule_global_auto_tasks - variant: task_generation - -buildvariants: - - name: perf-atlas-M60-real.arm.aws.2023-11 - display_name: PERF M60-Atlas ReplSet ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: atlas - canaries: none - atlas_setup: M60-repl - use_custom_build: true - infrastructure_provisioning: workload_client_arm.2023-04 - infrastructure_provisioning_release: 2023-09 - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - storageEngine: wiredTiger - compile_variant: "-arm64" - run_on: - - "rhel70-perf-atlas-large" - depends_on: - - name: compile - variant: compile-amazon2 - - name: schedule_global_auto_tasks - variant: task_generation - - name: compile - variant: compile-amazon-linux2-arm64 - - name: schedule_global_auto_tasks - variant: task_generation - tasks: - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: tpcc - - name: tpcc_majority - - name: linkbench - - name: linkbench2 - - - name: perf-atlas-M60-real.intel.azure.2023-11 - display_name: PERF M60-Atlas ReplSet Intel Azure 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: atlas - canaries: none - atlas_setup: M60-repl-azure - use_custom_build_azure: true - infrastructure_provisioning: workload_client_intel.2023-11 - infrastructure_provisioning_release: 2023-09 - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - storageEngine: wiredTiger - run_on: - - "rhel70-perf-atlas-large" - depends_on: *_compile_amazon2 - tasks: # Cannot use *3nodetasks because secondary_performance uses a special mongodb setup - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: tpcc - - name: tpcc_majority - - name: linkbench - - name: linkbench2 - - - name: perf-3-shard.arm.aws.2023-11 - display_name: PERF 3-Shard Cluster ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: shard - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: shard - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: "-arm64" - run_on: - - "rhel70-perf-shard" - depends_on: *_compile_amazon_linux2_arm64 - tasks: - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_w1.2023-09 - - name: crud_workloads_majority - - name: crud_workloads_w1 - - name: misc_workloads - - name: map_reduce_workloads - - name: smoke_test - - name: mongos_workloads - - name: mongos_large_catalog_workloads - - name: change_streams_latency - - name: change_streams_multi_mongos - - - name: perf-3-node-replSet.arm.aws.2023-11 - display_name: PERF 3-Node ReplSet ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: "-arm64" - run_on: - - "rhel70-perf-replset" - depends_on: *_compile_amazon_linux2_arm64 - tasks: &3nodetasks - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_w1.2023-09 - - name: ycsb_60GB.2023-09 - - name: ycsb.load - - name: ycsb_60GB.long.2023-09 - - name: ycsb_secondary_reads.2023-09 - - name: crud_workloads_majority - - name: crud_workloads_w1 - - name: misc_workloads - - name: map_reduce_workloads - - name: refine_shard_key_transaction_stress - - name: smoke_test - - name: secondary_performance # Uses a special 2 node mongodb setup - - name: non_sharded_workloads - - name: bestbuy_agg - - name: bestbuy_agg_merge_different_db - - name: bestbuy_agg_merge_same_db - - name: bestbuy_agg_merge_wordcount - - name: bestbuy_query - - name: change_streams_latency - - name: snapshot_reads - - name: secondary_reads - - name: tpcc - - name: linkbench - - name: linkbench2 - - - name: perf-3-node-replSet-intel.intel.aws.2023-11 - display_name: PERF 3-Node ReplSet Intel AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica-intel.2023-11 - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - authentication: enabled - storageEngine: wiredTiger - run_on: - - "rhel70-perf-replset" - depends_on: *_compile_amazon2 - tasks: &3nodetasks - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: industry_benchmarks - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: ycsb_60GB.long.2023-09 - - name: crud_workloads_majority - - name: smoke_test - - name: linkbench - - name: linkbench2 - - - - name: perf-2-node-replSet-initialsync.arm.aws.2023-11 - display_name: PERF 2-Node ReplSet Initial Sync ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica-2node - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica-2node - workload_setup: 2022-11 - platform: linux - authentication: disabled - storageEngine: wiredTiger - compile_variant: "-arm64" - project_dir: *project_dir - depends_on: *_compile_amazon_linux2_arm64 - run_on: - - "rhel70-perf-replset" - tasks: - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: initialsync-large - - - &linux-microbenchmarks-standalone-arm - name: perf-mongo-perf-standalone.arm.aws.2023-11 - display_name: PERF Monogo-Perf Standalone inMemory ARM AWS 2023-11 - cron: "0 0 * * 0" # Weekly, Sundays at 12 AM - modules: *modules - expansions: &standalone-arm-expansions - mongodb_setup_release: 2022-11 - mongodb_setup: mongo-perf-standalone.2023-02 - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: workload_client_mongod_combined.2023-01 - workload_setup: 2022-11 - use_scons_cache: true - platform: linux - canaries: none - storageEngine: inMemory - project_dir: *project_dir - compile_variant: "-arm64" - run_on: - - "rhel70-perf-microbenchmarks" - depends_on: *_compile_amazon_linux2_arm64 - tasks: - - name: genny_scale_InsertRemove - - name: genny_execution_UserAcquisition - - name: aggregation_read_commands - - name: agg-query-comparison_read_commands - - name: query_read_commands - - name: views-aggregation - - name: views-query - - name: where_read_commands - - name: update_read_commands - - name: insert_read_commands - - name: wildcard-index-read_read_commands - - name: wildcard-index-write_read_commands - - name: geo_read_commands - - name: misc_read_commands - - name: singleThreaded_read_commands - - name: pipeline-updates - - name: javascript - - - &linux-microbenchmarks-standalone-intel - <<: *linux-microbenchmarks-standalone-arm - name: perf-mongo-perf-standalone.intel.aws.2023-11 - display_name: PERF Mongo-Perf Standalone inMemory Intel AWS 2023-11 - cron: "0 0 * * 0" # Weekly, Sundays at 12 AM - expansions: &standalone-intel-expansions - <<: *standalone-arm-expansions - infrastructure_provisioning: workload_client_mongod_combined_intel.2023-01 - compile_variant: "" - run_on: - - "rhel70-perf-microbenchmarks" - depends_on: *_compile_amazon2 diff --git a/etc/third_party_components.yml b/etc/third_party_components.yml index 8f1e80afadb..bfbb43c8398 100644 --- a/etc/third_party_components.yml +++ b/etc/third_party_components.yml @@ -145,13 +145,6 @@ components: vulnerability_suppression: TODO SERVER-49211 upgrade_suppression: TODO SERVER-49211 - immer: - homepage_url: https://sinusoid.es/immer - open_hub_url: N/A - release_monitoring_id: -1 - local_directory_path: src/third_party/immer - team_owner: "Storage Execution" - "Intel Decimal Floating-Point Math Library": homepage_url: https://software.intel.com/content/www/us/en/develop/articles/intel-decimal-floating-point-math-library.html open_hub_url: N/A @@ -307,7 +300,7 @@ components: homepage_url: https://github.com/derickr/timelib open_hub_url: N/A release_monitoring_id: -1 - local_directory_path: src/third_party/timelib + local_directory_path: src/third_party/timelib-2021.06 team_owner: "Query" # Note: Not in Black Duck @@ -346,7 +339,7 @@ components: homepage_url: https://www.zlib.net/ open_hub_url: https://www.openhub.net/p/zlib release_monitoring_id: 5303 - local_directory_path: src/third_party/zlib-1.2.13 + local_directory_path: src/third_party/zlib-1.2.11 team_owner: "Execution" zstd: |