diff options
Diffstat (limited to 'etc/system_perf.yml')
| -rw-r--r-- | etc/system_perf.yml | 403 |
1 files changed, 110 insertions, 293 deletions
diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 79f2a3d1a89..137bc2b4a13 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -10,37 +10,41 @@ post: working_dir: work script: | source ./dsienv.sh - $DSI_PATH/bin/make_artifact.sh + make_artifact.sh - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: work/reports.tgz - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}.${ext|tgz} + local_file: work/dsi-artifacts.tgz + remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} - display_name: test-log + display_name: Dsi Artifacts - Execution ${execution} - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: work/reports/graphs/timeseries-p1.html - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-p1-${task_name}-${build_id}.html + local_file: src/workloads/workloads/jsdoc/jsdocs-redirect.html + remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html bucket: mciuploads permissions: public-read content_type: text/html - display_name: timeseries-p1.html + display_name: workloads documentation - command: attach.results params: file_location: work/report.json + - command: "json.send" + params: + name: "perf" + file: "work/perf.json" - command: shell.exec params: working_dir: work script: | source ./dsienv.sh if [ -e /data/infrastructure_provisioning/terraform/provisioned.${cluster} ]; then - $DSI_PATH/bin/mark_idle.sh + mark_idle.sh fi - command: shell.exec @@ -75,12 +79,15 @@ functions: working_dir: work script: | cat > bootstrap.yml <<EOF - cluster_type: ${cluster} + infrastructure_provisioning: ${cluster} platform: ${platform} - setup: ${setup} + mongodb_setup: ${setup} storageEngine: ${storageEngine} - test: ${test} + test_control: ${test} production: true + mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz" + workloads_dir: ../src/workloads/workloads + ycsb_dir: ../src/YCSB/YCSB EOF cat > runtime.yml <<EOF @@ -102,11 +109,7 @@ functions: ext: ${ext} script_flags : ${script_flags} dsi_rev: ${dsi_rev} - compare_task: ${compare_task} workloads_rev: ${workloads_rev} - - # compositions of expansions - mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz" EOF - command: shell.exec params: @@ -136,44 +139,32 @@ functions: virtualenv ./venv source ./venv/bin/activate pip install -r ../src/dsi/dsi/requirements.txt - python ../src/dsi/dsi/bin/bootstrap.py --production - ls - pwd + python ../src/dsi/dsi/bin/bootstrap.py - command: shell.exec params: script: | set -v set -e source work/dsienv.sh - $DSI_PATH/bin/setup-dsi-env.sh + setup-dsi-env.sh ls -a work - "infrastructure provisioning": + "deploy cluster": - command: shell.exec - # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances. params: working_dir: work script: | set -e set -v source ./dsienv.sh - export PRODUCTION=true - $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster} - - "configure mongodb cluster": - - command: shell.exec - # bring up the mongod - params: - working_dir: work - script: | - set -e - set -o verbose - source ./dsienv.sh source ./venv/bin/activate - $DSI_PATH/bin/mongodb_setup.py && echo "${setup} MongoDB Cluster STARTED." + infrastructure_provisioning.py + workload_setup.py + mongodb_setup.py "run test": - command: shell.exec + type: test params: working_dir: work script: | @@ -181,21 +172,12 @@ functions: set -v source ./dsienv.sh source ./venv/bin/activate - echo "Run test for ${test}-${storageEngine} with setup ${setup}" - $DSI_PATH/bin/run_test.py ${storageEngine} ${test} ${cluster} + test_control.py - command: "json.send" params: name: "perf" file: "work/perf.json" - "make test log artifact": - - command: shell.exec - params: - working_dir: work - script: | - source ./dsienv.sh - $DSI_PATH/bin/make_artifact.sh - "analyze": - command: json.get_history params: @@ -216,8 +198,8 @@ functions: silent: true script: | set -o errexit - TAGS="3.2.15-Baseline" - OVERRIDEFILE="../src/dsi/dsi/analysis/v3.4/system_perf_override.json" + TAGS="3.2.17-Baseline 3.4.9-Baseline" + OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" python -u ../src/dsi/dsi/analysis/dashboard_gen.py --rev ${revision} -f history.json -t tags.json --refTag $TAGS --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} --jira-user ${perf_jira_user} --jira-password ${perf_jira_pw} || true - command: "json.send" params: @@ -231,48 +213,9 @@ functions: script: | set -o errexit set -o verbose - TAG="3.2.15-Baseline" - OVERRIDEFILE="../src/dsi/dsi/analysis/v3.4/system_perf_override.json" - python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file reports/perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} - - "compare": - - command: shell.exec - params: - script: | - set -o verbose - rm -rf ./src ./work - mkdir src - mkdir work - - command: manifest.load - - command: git.get_project - params: - directory: src - revisions: # for each module include revision as <module_name> : ${<module_name>_rev} - dsi: ${dsi_rev} - - command: json.get - params: - task: ${compare_task} - variant : ${variant1} - file: "work/standalone.json" - name: "perf" - - command: json.get - params: - task: ${compare_task} - variant : ${variant2} - file: "work/oplog.json" - name: "perf" - - command: shell.exec - type : test - params: - working_dir: work - script: | - set -o errexit - set -o verbose - python -u ../src/dsi/dsi/analysis/compare.py -b standalone.json -c oplog.json - - command: "json.send" - params: - name: "perf" - file: "work/perf.json" + TAG="3.2.17-Baseline" + OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" + python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} ####################################### # Tasks # @@ -323,8 +266,17 @@ tasks: then echo "Fetching JS test DB correctness checks from directory jstests/hooks" cp -a jstests/hooks/* mongodb/jstests/hooks + + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF fi - tar cvf mongodb.tar mongodb + tar cf mongodb.tar mongodb gzip mongodb.tar - command: s3.put params: @@ -341,25 +293,20 @@ tasks: depends_on: - name: compile variant: linux-standalone + priority: 5 commands: - func: "prepare environment" vars: storageEngine: "wiredTiger" test: "ycsb" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "ycsb" - - func: "make test log artifact" - func: "analyze" vars: script_flags: --ycsb-throughput-analysis reports - name: industry_benchmarks_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -368,20 +315,46 @@ tasks: vars: storageEngine: "mmapv1" test: "ycsb" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" + - func: "deploy cluster" + - func: "run test" + - func: "analyze" vars: - storageEngine: "mmapv1" + script_flags: --ycsb-throughput-analysis reports + +- name: industry_benchmarks_wmajority_WT + priority: 5 + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" + vars: + storageEngine: "wiredTiger" + test: "ycsb-wmajority" + - func: "deploy cluster" - func: "run test" + - func: "analyze" + vars: + script_flags: --ycsb-throughput-analysis reports + +- name: industry_benchmarks_wmajority_MMAPv1 + priority: 5 + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" vars: storageEngine: "mmapv1" - test: "ycsb" - - func: "make test log artifact" + test: "ycsb-wmajority" + - func: "deploy cluster" + - func: "run test" - func: "analyze" vars: script_flags: --ycsb-throughput-analysis reports - name: core_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -390,18 +363,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "core" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "core" - - func: "make test log artifact" - func: "analyze" - name: core_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -410,18 +377,12 @@ tasks: vars: storageEngine: "mmapv1" test: "core" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "core" - - func: "make test log artifact" - func: "analyze" - name: non_sharded_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -430,18 +391,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "non_sharded" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "non_sharded" - - func: "make test log artifact" - func: "analyze" - name: non_sharded_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -450,18 +405,12 @@ tasks: vars: storageEngine: "mmapv1" test: "non_sharded" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "non_sharded" - - func: "make test log artifact" - func: "analyze" - name: mongos_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -470,18 +419,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "mongos" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "mongos" - - func: "make test log artifact" - func: "analyze" - name: mongos_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -491,18 +434,12 @@ tasks: vars: storageEngine: "mmapv1" test: "mongos" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "mongos" - - func: "make test log artifact" - func: "analyze" - name: move_chunk_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -511,18 +448,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "move_chunk" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "move_chunk" - - func: "make test log artifact" - func: "analyze" - name: move_chunk_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -531,18 +462,12 @@ tasks: vars: storageEngine: "mmapv1" test: "move_chunk" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "move_chunk" - - func: "make test log artifact" - func: "analyze" - name: secondary_performance_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -550,21 +475,15 @@ tasks: - func: "prepare environment" vars: storageEngine: "wiredTiger" + # Unfortunately the dash/underscore style is different for mongodb_setup and test_control test: "secondary_performance" - setup: "replica-2node" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" - setup: "replica-2node" + setup: "secondary-performance" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "secondary_performance" - - func: "make test log artifact" - func: "analyze" - name: secondary_performance_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -572,85 +491,15 @@ tasks: - func: "prepare environment" vars: storageEngine: "mmapv1" + # Unfortunately the dash/underscore style is different for mongodb_setup and test_control test: "secondary_performance" - setup: "replica-2node" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" - setup: "replica-2node" + setup: "secondary-performance" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "secondary_performance" - - func: "make test log artifact" - - func: "analyze" - -- name: industry_benchmarks_WT_oplog_comp - depends_on: - - name: industry_benchmarks_WT - variant: linux-standalone - status : "*" - - name: industry_benchmarks_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: industry_benchmarks_MMAPv1_oplog_comp - depends_on: - - name: industry_benchmarks_MMAPv1 - variant: linux-standalone - status: "*" - - name: industry_benchmarks_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_WT_oplog_comp - depends_on: - - name: core_workloads_WT - variant: linux-standalone - status: "*" - - name: core_workloads_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_MMAPv1_oplog_comp - depends_on: - - name: core_workloads_MMAPv1 - variant: linux-standalone - status: "*" - - name: core_workloads_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - func: "analyze" - name: initialsync_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -658,19 +507,13 @@ tasks: - func: "prepare environment" vars: storageEngine: "wiredTiger" - test: "initialSync" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + test: "initialsync" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "initialSync" - - func: "make test log artifact" - func: "analyze" - name: initialsync_MMAPv1 + priority: 5 depends_on: - name: compile variant : linux-standalone @@ -678,19 +521,13 @@ tasks: - func: "prepare environment" vars: storageEngine: "mmapv1" - test: "initialSync" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + test: "initialsync" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "initialSync" - - func: "make test log artifact" - func: "analyze" - name: initialsync-logkeeper_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -700,18 +537,10 @@ tasks: vars: storageEngine: "wiredTiger" test: "initialsync-logkeeper" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "initialsync-logkeeper" - - func: "make test log artifact" - func: "analyze" - ####################################### # Modules # ####################################### @@ -798,6 +627,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: mongos_workloads_WT - name: mongos_workloads_MMAPv1 @@ -820,6 +651,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: secondary_performance_WT - name: secondary_performance_MMAPv1 @@ -858,19 +691,3 @@ buildvariants: - "rhel70-perf-initialsync-logkeeper" tasks: - name: initialsync-logkeeper_WT - -- name: linux-oplog-compare - display_name: Linux Oplog Compare - batchtime: 10080 # 7 days - modules: *modules - expansions: - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy - use_scons_cache: true - project: *project - run_on: - - "rhel70-perf-single" - tasks: - - name: industry_benchmarks_WT_oplog_comp - - name: core_workloads_WT_oplog_comp - - name: industry_benchmarks_MMAPv1_oplog_comp - - name: core_workloads_MMAPv1_oplog_comp |