From 434e53cba2b14c1a4f04a0b3fb557507d0d9a77f Mon Sep 17 00:00:00 2001 From: Apollon Oikonomopoulos Date: Sat, 12 Aug 2017 12:12:15 -0400 Subject: New upstream version 3.2.16 --- SConstruct | 74 +- buildscripts/buildlogger.py | 12 + buildscripts/clang_format.py | 101 +- buildscripts/generate_compile_expansions.py | 4 +- buildscripts/hang_analyzer.py | 73 +- buildscripts/package_test/.kitchen.yml | 81 ++ buildscripts/package_test/metadata.rb | 2 + .../package_test/recipes/install_mongodb.rb | 120 ++ .../test/recipes/service/install_mongodb_spec.rb | 180 +++ buildscripts/packager-enterprise.py | 16 +- buildscripts/packager.py | 105 +- buildscripts/resmoke.py | 56 +- .../resmokeconfig/suites/multiversion_auth.yml | 37 + .../resmokeconfig/suites/no_passthrough.yml | 3 + .../resmokeconfig/suites/replica_sets_legacy.yml | 8 + .../sharded_collections_jscore_passthrough.yml | 6 +- .../resmokeconfig/suites/sharding_auth.yml | 1 + .../resmokeconfig/suites/sharding_auth_audit.yml | 1 + .../suites/sharding_continuous_config_stepdown.yml | 8 + .../suites/sharding_jscore_passthrough.yml | 5 +- buildscripts/resmokelib/__init__.py | 2 + buildscripts/resmokelib/logging/buildlogger.py | 6 +- buildscripts/resmokelib/logging/handlers.py | 63 +- buildscripts/resmokelib/reportfile.py | 29 + buildscripts/resmokelib/sighandler.py | 110 ++ buildscripts/resmokelib/testing/executor.py | 5 +- .../resmokelib/testing/fixtures/replicaset.py | 8 +- .../resmokelib/testing/fixtures/shardedcluster.py | 5 +- .../resmokelib/testing/fixtures/standalone.py | 5 +- buildscripts/resmokelib/testing/job.py | 16 +- buildscripts/resmokelib/testing/report.py | 205 ++-- buildscripts/resmokelib/testing/testgroup.py | 62 +- buildscripts/resmokelib/utils/timer.py | 66 +- buildscripts/smoke.py | 124 ++- etc/evergreen.yml | 234 +++- etc/longevity.yml | 132 ++- etc/perf.yml | 266 +---- etc/scons/gold_incremental_link.vars | 1 + etc/scons/mongodbtoolchain_clang.vars | 22 + etc/scons/mongodbtoolchain_gcc.vars | 22 + etc/system_perf.yml | 362 ++++-- jstests/auth/commands_builtin_roles.js | 10 +- jstests/auth/commands_user_defined_roles.js | 93 +- jstests/auth/server-4892.js | 6 +- jstests/concurrency/fsm_all.js | 7 +- jstests/concurrency/fsm_all_composed.js | 3 + jstests/concurrency/fsm_all_replication.js | 3 + jstests/concurrency/fsm_all_sharded_replication.js | 1 + ...ll_sharded_replication_legacy_config_servers.js | 1 + ...lication_legacy_config_servers_with_balancer.js | 6 + .../fsm_all_sharded_replication_with_balancer.js | 6 + .../concurrency/fsm_workloads/create_database.js | 150 +++ .../sharded_moveChunk_drop_shard_key_index.js | 84 ++ jstests/core/currentop.js | 114 +- jstests/core/dbadmin.js | 8 +- jstests/core/diagdata.js | 33 + jstests/core/drop2.js | 57 - jstests/core/evalh.js | 18 + jstests/core/evalj.js | 13 + jstests/core/hostinfo.js | 10 + jstests/core/index_partial_write_ops.js | 18 +- jstests/core/killop_drop_collection.js | 54 + jstests/core/list_collections1.js | 3 +- jstests/core/list_indexes.js | 3 +- jstests/core/list_indexes_invalidation.js | 37 + jstests/core/list_namespaces_invalidation.js | 73 ++ jstests/core/max_doc_size.js | 6 +- jstests/core/ref.js | 5 + jstests/core/server22053.js | 19 + jstests/core/server25192.js | 12 + jstests/core/tailable_skip_limit.js | 7 + jstests/libs/check_log.js | 84 ++ jstests/libs/chunk_manipulation_util.js | 6 +- jstests/libs/election_timing_test.js | 227 ---- jstests/libs/parallelTester.js | 6 +- jstests/libs/test_background_ops.js | 4 + jstests/libs/write_concern_util.js | 73 ++ jstests/multiVersion/libs/multi_rs.js | 3 +- .../multiVersion/minor_version_tags_new_old_new.js | 274 +---- .../multiVersion/minor_version_tags_old_new_old.js | 274 +---- jstests/noPassthrough/command_line_parsing.js | 4 +- jstests/noPassthrough/indexbg1.js | 18 +- jstests/noPassthrough/indexbg2.js | 20 +- jstests/noPassthrough/read_majority_reads.js | 270 +++++ jstests/noPassthrough/shell_history.js | 126 +++ jstests/noPassthrough/wt_index_option_defaults.js | 6 +- .../noPassthroughWithMongod/connections_opened.js | 6 +- jstests/noPassthroughWithMongod/ftdc_params.js | 4 +- jstests/noPassthroughWithMongod/indexbg_drop.js | 2 +- jstests/noPassthroughWithMongod/ttl_sharded.js | 30 +- jstests/replsets/chaining_removal.js | 42 +- .../replsets/disallow_adding_initialized_node1.js | 15 +- .../replsets/disallow_adding_initialized_node2.js | 20 +- .../replsets/do_not_sync_from_stale_sync_source.js | 44 + jstests/replsets/double_rollback.js | 32 +- jstests/replsets/double_rollback_early.js | 139 +++ jstests/replsets/drain.js | 19 +- jstests/replsets/election_timing.js | 143 --- jstests/replsets/last_vote.js | 229 ++++ jstests/replsets/libs/tags.js | 323 ++++++ jstests/replsets/maintenance.js | 4 + jstests/replsets/maxSyncSourceLagSecs.js | 41 +- jstests/replsets/no_chaining.js | 10 +- .../no_flapping_during_network_partition.js | 60 + .../oplog_replay_on_startup_update_and_delete.js | 48 + .../priority_takeover_cascading_priorities.js | 20 +- .../priority_takeover_two_nodes_equal_priority.js | 62 +- .../replsets/protocol_version_upgrade_downgrade.js | 9 +- jstests/replsets/read_after_optime.js | 29 +- jstests/replsets/read_committed.js | 165 ++- jstests/replsets/read_committed_after_rollback.js | 6 +- jstests/replsets/read_committed_stale_history.js | 149 +++ .../read_committed_with_catalog_changes.js | 343 ++++++ .../replsets/reconfig_without_increased_queues.js | 10 + jstests/replsets/replsetfreeze.js | 4 + jstests/replsets/request_primary_stepdown.js | 46 + jstests/replsets/resync_with_write_load.js | 2 +- .../rollback_after_sync_source_selection.js | 113 ++ jstests/replsets/rollback_empty_ns.js | 93 -- jstests/replsets/rollback_empty_o.js | 93 -- jstests/replsets/rollback_empty_o2.js | 94 -- jstests/replsets/rollback_too_new.js | 21 +- ...rollback_with_socket_error_then_steady_state.js | 119 ++ jstests/replsets/rslib.js | 176 +++ jstests/replsets/server8070.js | 13 +- jstests/replsets/shutdown_primary.js | 68 ++ jstests/replsets/slavedelay3.js | 7 +- jstests/replsets/stepdown.js | 88 +- .../replsets/stepdown_needs_electable_secondary.js | 143 +++ jstests/replsets/stepdown_needs_majority.js | 110 ++ jstests/replsets/stepdown_wrt_electable.js | 40 - jstests/replsets/sync2.js | 109 +- jstests/replsets/tags.js | 258 +---- .../temp_namespace_restart_as_standalone.js | 87 ++ jstests/replsets/write_concern_after_stepdown.js | 91 ++ .../write_concern_after_stepdown_and_stepup.js | 116 ++ jstests/sharding/authCommands.js | 7 +- jstests/sharding/cleanup_orphaned_basic.js | 4 +- jstests/sharding/conn_pool_stats.js | 43 +- jstests/sharding/cursor1.js | 5 - .../migration_critical_section_concurrency.js | 63 ++ jstests/sharding/move_stale_mongos.js | 14 +- jstests/sharding/shard6.js | 34 +- jstests/sharding/shard_kill_and_pooling.js | 4 +- jstests/sharding/sharding_rs2.js | 7 +- jstests/sharding/split_stale_mongos.js | 8 +- .../sharding/sync_cluster_config/sccc_read_pref.js | 18 + jstests/slow1/replsets_priority1.js | 2 +- jstests/ssl/ssl_fips.js | 5 +- rpm/mongodb-enterprise-unstable.spec | 6 +- rpm/mongodb-enterprise.spec | 6 +- rpm/mongodb-org-unstable.spec | 6 +- rpm/mongodb-org.spec | 6 +- site_scons/site_tools/compilation_db.py | 34 +- site_scons/site_tools/incremental_link.py | 45 + site_scons/site_tools/split_dwarf.py | 65 ++ src/mongo/SConscript | 1 + src/mongo/base/error_codes.err | 5 + src/mongo/base/string_data.h | 24 +- src/mongo/bson/timestamp.cpp | 8 + src/mongo/bson/timestamp.h | 3 + src/mongo/client/SConscript | 1 + src/mongo/client/authenticate.cpp | 6 +- src/mongo/client/authenticate.h | 2 +- src/mongo/client/authenticate_test.cpp | 8 +- src/mongo/client/connpool.cpp | 66 +- src/mongo/client/connpool.h | 28 +- src/mongo/client/dbclient.cpp | 16 +- src/mongo/client/dbclientcursor.cpp | 3 - src/mongo/client/dbclientcursor.h | 7 - src/mongo/client/mongo_uri.cpp | 19 +- src/mongo/client/mongo_uri_connect.cpp | 6 +- src/mongo/client/mongo_uri_test.cpp | 13 + src/mongo/client/native_sasl_client_session.cpp | 6 +- src/mongo/client/sasl_client_authenticate.cpp | 2 +- src/mongo/client/sasl_client_authenticate.h | 2 +- src/mongo/client/sasl_client_authenticate_impl.cpp | 7 +- src/mongo/client/sasl_client_session.h | 1 + .../client/sasl_scramsha1_client_conversation.cpp | 46 +- .../client/sasl_scramsha1_client_conversation.h | 12 +- src/mongo/client/scram_sha1_client_cache.cpp | 70 ++ src/mongo/client/scram_sha1_client_cache.h | 88 ++ src/mongo/client/syncclusterconnection.cpp | 19 +- src/mongo/client/syncclusterconnection.h | 2 +- src/mongo/crypto/SConscript | 41 +- src/mongo/crypto/crypto.h | 51 - src/mongo/crypto/crypto_openssl.cpp | 76 -- src/mongo/crypto/crypto_test.cpp | 521 --------- src/mongo/crypto/crypto_tom.cpp | 88 -- src/mongo/crypto/mechanism_scram.cpp | 292 +++-- src/mongo/crypto/mechanism_scram.h | 104 +- src/mongo/crypto/sha1_block.cpp | 88 ++ src/mongo/crypto/sha1_block.h | 113 ++ src/mongo/crypto/sha1_block_openssl.cpp | 96 ++ src/mongo/crypto/sha1_block_test.cpp | 565 ++++++++++ src/mongo/crypto/sha1_block_tom.cpp | 80 ++ src/mongo/db/SConscript | 2 +- src/mongo/db/auth/SConscript | 17 + src/mongo/db/auth/action_set.cpp | 6 + src/mongo/db/auth/action_set.h | 1 + src/mongo/db/auth/authorization_session.cpp | 7 +- src/mongo/db/auth/authorization_session.h | 26 +- .../db/auth/authorization_session_for_test.cpp | 83 ++ src/mongo/db/auth/authorization_session_for_test.h | 85 ++ src/mongo/db/auth/privilege_parser.cpp | 9 +- src/mongo/db/auth/role_graph_builtin_roles.cpp | 44 +- .../db/auth/sasl_plain_server_conversation.cpp | 18 +- .../db/auth/sasl_scramsha1_server_conversation.cpp | 55 +- src/mongo/db/auth/sasl_scramsha1_test.cpp | 501 +++++++++ src/mongo/db/catalog/apply_ops.cpp | 2 +- src/mongo/db/catalog/coll_mod.cpp | 243 ++-- src/mongo/db/catalog/coll_mod.h | 9 + src/mongo/db/catalog/collection.cpp | 16 +- src/mongo/db/catalog/collection.h | 28 +- src/mongo/db/catalog/database.cpp | 74 +- src/mongo/db/catalog/database.h | 14 +- src/mongo/db/catalog/database_holder.cpp | 98 +- src/mongo/db/catalog/database_holder.h | 9 +- src/mongo/db/catalog/index_catalog.cpp | 27 +- src/mongo/db/catalog/index_create.cpp | 38 +- src/mongo/db/catalog/rename_collection.cpp | 5 +- src/mongo/db/commands/apply_ops.cpp | 41 +- src/mongo/db/commands/find_cmd.cpp | 35 +- src/mongo/db/commands/generic.cpp | 56 + src/mongo/db/commands/getmore_cmd.cpp | 2 +- src/mongo/db/commands/list_databases.cpp | 14 +- src/mongo/db/concurrency/lock_manager.cpp | 13 +- src/mongo/db/concurrency/lock_manager_test.cpp | 61 + src/mongo/db/concurrency/lock_state_test.cpp | 82 ++ src/mongo/db/db.cpp | 60 +- src/mongo/db/db_raii.cpp | 6 +- src/mongo/db/dbcommands.cpp | 19 +- src/mongo/db/dbhelpers.cpp | 6 + src/mongo/db/exec/geo_near.cpp | 90 +- src/mongo/db/ftdc/SConscript | 23 +- src/mongo/db/ftdc/config.h | 2 +- src/mongo/db/ftdc/controller.cpp | 13 + src/mongo/db/ftdc/controller.h | 18 +- src/mongo/db/ftdc/file_writer.cpp | 18 +- src/mongo/db/ftdc/file_writer_test.cpp | 42 +- src/mongo/db/ftdc/ftdc_commands.cpp | 113 ++ src/mongo/db/ftdc/ftdc_mongod.cpp | 14 +- src/mongo/db/ftdc/ftdc_system_stats.cpp | 61 + src/mongo/db/ftdc/ftdc_system_stats.h | 64 ++ src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_linux.cpp | 127 +++ src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_osx.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_windows.cpp | 154 +++ src/mongo/db/ftdc/ftdc_test.cpp | 2 + src/mongo/db/index/index_access_method.cpp | 10 +- src/mongo/db/index/index_access_method.h | 9 +- src/mongo/db/instance.cpp | 4 +- src/mongo/db/mongod_options.cpp | 10 +- src/mongo/db/pipeline/SConscript | 1 + src/mongo/db/pipeline/document_source.h | 6 +- src/mongo/db/pipeline/pipeline.cpp | 84 +- src/mongo/db/pipeline/pipeline_test.cpp | 221 +++- src/mongo/db/query/find.cpp | 38 +- src/mongo/db/query/lite_parsed_query.cpp | 15 +- src/mongo/db/query/lite_parsed_query.h | 5 - src/mongo/db/repl/SConscript | 1 + src/mongo/db/repl/applier.h | 2 +- src/mongo/db/repl/bgsync.cpp | 334 ++++-- src/mongo/db/repl/bgsync.h | 14 +- src/mongo/db/repl/data_replicator.cpp | 6 +- src/mongo/db/repl/freshness_checker.cpp | 8 +- src/mongo/db/repl/freshness_checker_test.cpp | 6 +- src/mongo/db/repl/last_vote.cpp | 20 +- src/mongo/db/repl/last_vote.h | 11 +- src/mongo/db/repl/master_slave.cpp | 13 +- src/mongo/db/repl/oplog.cpp | 23 +- src/mongo/db/repl/oplog.h | 8 +- src/mongo/db/repl/oplogreader.cpp | 15 +- src/mongo/db/repl/oplogreader.h | 7 +- src/mongo/db/repl/repl_set_request_votes_args.cpp | 14 +- src/mongo/db/repl/repl_set_request_votes_args.h | 8 +- src/mongo/db/repl/replication_coordinator.h | 27 +- .../repl/replication_coordinator_external_state.h | 5 + ...replication_coordinator_external_state_impl.cpp | 34 +- .../replication_coordinator_external_state_impl.h | 1 + ...replication_coordinator_external_state_mock.cpp | 3 + .../replication_coordinator_external_state_mock.h | 1 + src/mongo/db/repl/replication_coordinator_impl.cpp | 233 ++-- src/mongo/db/repl/replication_coordinator_impl.h | 54 +- .../db/repl/replication_coordinator_impl_elect.cpp | 6 +- .../replication_coordinator_impl_elect_test.cpp | 1 + .../repl/replication_coordinator_impl_elect_v1.cpp | 4 +- .../replication_coordinator_impl_heartbeat.cpp | 77 +- ...lication_coordinator_impl_heartbeat_v1_test.cpp | 2 +- .../db/repl/replication_coordinator_impl_test.cpp | 722 ++++++++---- src/mongo/db/repl/replication_coordinator_mock.cpp | 14 +- src/mongo/db/repl/replication_coordinator_mock.h | 14 +- src/mongo/db/repl/replication_executor.cpp | 13 + src/mongo/db/repl/replication_executor.h | 1 + src/mongo/db/repl/replication_executor_test.cpp | 160 +++ src/mongo/db/repl/replset_commands.cpp | 21 +- src/mongo/db/repl/rs_rollback.cpp | 436 +++----- src/mongo/db/repl/rs_rollback.h | 85 +- src/mongo/db/repl/rs_rollback_test.cpp | 243 ++-- src/mongo/db/repl/sync_source_feedback.cpp | 19 +- src/mongo/db/repl/sync_tail.cpp | 73 +- src/mongo/db/repl/sync_tail.h | 17 +- src/mongo/db/repl/sync_tail_test.cpp | 284 ++++- src/mongo/db/repl/topology_coordinator.h | 26 +- src/mongo/db/repl/topology_coordinator_impl.cpp | 118 +- src/mongo/db/repl/topology_coordinator_impl.h | 17 +- .../db/repl/topology_coordinator_impl_test.cpp | 37 + src/mongo/db/repl/vote_requester.cpp | 13 +- src/mongo/db/repl/vote_requester.h | 6 +- src/mongo/db/s/migration_source_manager.cpp | 100 +- src/mongo/db/stats/SConscript | 21 + src/mongo/db/stats/fill_locker_info_test.cpp | 136 +++ src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp | 4 - .../storage/mmap_v1/catalog/namespace_details.cpp | 1 - .../catalog/namespace_details_collection_entry.cpp | 24 +- .../mmap_v1/mmap_v1_database_catalog_entry.cpp | 27 +- .../mmap_v1/mmap_v1_database_catalog_entry.h | 3 + .../db/storage/mmap_v1/record_store_v1_simple.cpp | 3 +- src/mongo/db/storage/snapshot.h | 4 + .../db/storage/wiredtiger/wiredtiger_index.cpp | 72 +- src/mongo/db/storage/wiredtiger/wiredtiger_index.h | 3 + .../db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 46 +- .../db/storage/wiredtiger/wiredtiger_kv_engine.h | 6 +- .../storage/wiredtiger/wiredtiger_record_store.cpp | 40 +- .../storage/wiredtiger/wiredtiger_record_store.h | 12 +- .../wiredtiger/wiredtiger_recovery_unit.cpp | 6 +- .../wiredtiger/wiredtiger_session_cache.cpp | 43 +- .../storage/wiredtiger/wiredtiger_session_cache.h | 11 +- .../db/storage/wiredtiger/wiredtiger_util.cpp | 4 +- src/mongo/db/write_concern.cpp | 5 + src/mongo/dbtests/SConscript | 1 + src/mongo/dbtests/replica_set_tests.cpp | 127 +++ src/mongo/executor/SConscript | 1 + src/mongo/executor/async_mock_stream_factory.cpp | 4 +- src/mongo/executor/async_mock_stream_factory.h | 59 + src/mongo/executor/async_stream_test.cpp | 6 +- src/mongo/executor/connection_pool.cpp | 118 +- src/mongo/executor/connection_pool.h | 14 +- src/mongo/executor/connection_pool_stats.cpp | 61 +- src/mongo/executor/connection_pool_stats.h | 19 +- src/mongo/executor/connection_pool_test.cpp | 327 +++++- .../executor/connection_pool_test_fixture.cpp | 34 +- src/mongo/executor/connection_pool_test_fixture.h | 2 + src/mongo/executor/network_interface.h | 5 + src/mongo/executor/network_interface_asio.cpp | 5 + src/mongo/executor/network_interface_asio.h | 2 + src/mongo/executor/network_interface_asio_auth.cpp | 10 +- .../executor/network_interface_asio_command.cpp | 6 +- .../network_interface_asio_integration_test.cpp | 10 +- .../executor/network_interface_asio_operation.cpp | 3 + src/mongo/executor/network_interface_asio_test.cpp | 40 + src/mongo/executor/network_interface_impl.h | 1 + src/mongo/executor/network_interface_mock.h | 2 + src/mongo/executor/thread_pool_task_executor.cpp | 23 +- src/mongo/executor/thread_pool_task_executor.h | 5 + src/mongo/installer/msi/SConscript | 2 +- src/mongo/logger/logstream_builder.cpp | 16 +- src/mongo/logger/logstream_builder.h | 9 +- src/mongo/logger/message_event_utf8_encoder.cpp | 34 +- src/mongo/logger/ramlog.cpp | 13 +- src/mongo/logger/ramlog.h | 5 + src/mongo/logger/rotatable_file_writer.cpp | 31 +- src/mongo/s/commands/SConscript | 1 + src/mongo/s/commands/cluster_ftdc_commands.cpp | 94 ++ .../s/commands/cluster_list_databases_cmd.cpp | 13 +- src/mongo/s/commands/cluster_pipeline_cmd.cpp | 4 +- src/mongo/s/query/SConscript | 1 + src/mongo/s/query/async_results_merger.cpp | 11 +- src/mongo/s/query/async_results_merger.h | 9 +- src/mongo/s/query/async_results_merger_test.cpp | 44 +- src/mongo/s/query/cluster_query_knobs.cpp | 39 + src/mongo/s/query/cluster_query_knobs.h | 40 + src/mongo/s/query/router_stage_merge.cpp | 4 + src/mongo/s/sharding_initialization.cpp | 4 + src/mongo/scripting/SConscript | 2 + src/mongo/scripting/deadline_monitor.cpp | 43 + src/mongo/scripting/deadline_monitor.h | 36 +- src/mongo/scripting/deadline_monitor_test.cpp | 1 + src/mongo/scripting/engine.cpp | 20 +- src/mongo/scripting/engine.h | 6 +- src/mongo/scripting/mozjs/implscope.cpp | 72 +- src/mongo/scripting/mozjs/implscope.h | 9 +- src/mongo/scripting/mozjs/mongo.cpp | 22 +- src/mongo/scripting/mozjs/mongo.h | 3 +- src/mongo/scripting/mozjs/objectwrapper.cpp | 117 +- src/mongo/scripting/mozjs/objectwrapper.h | 2 +- src/mongo/scripting/mozjs/proxyscope.cpp | 9 +- src/mongo/scripting/mozjs/proxyscope.h | 5 +- src/mongo/scripting/mozjs/valuewriter.cpp | 4 + src/mongo/scripting/utils.cpp | 4 + src/mongo/shell/assert.js | 29 +- src/mongo/shell/db.js | 2 +- src/mongo/shell/linenoise.cpp | 14 +- src/mongo/shell/replsettest.js | 182 ++- src/mongo/shell/shell_utils_launcher.cpp | 7 +- src/mongo/shell/types.js | 2 +- src/mongo/shell/utils.js | 10 +- src/mongo/stdx/thread.h | 35 +- src/mongo/stdx/type_traits.h | 56 + src/mongo/util/SConscript | 41 + src/mongo/util/exit_code.h | 1 + src/mongo/util/log.h | 16 + src/mongo/util/net/SConscript | 1 + src/mongo/util/net/message_port.cpp | 15 +- src/mongo/util/net/sock.cpp | 189 ++-- src/mongo/util/net/sock.h | 2 +- src/mongo/util/net/ssl_manager.cpp | 38 +- src/mongo/util/net/ssl_options.cpp | 7 + src/mongo/util/net/ssl_options.h | 2 + src/mongo/util/perfctr_collect.cpp | 505 +++++++++ src/mongo/util/perfctr_collect.h | 296 +++++ src/mongo/util/perfctr_collect_test.cpp | 416 +++++++ src/mongo/util/processinfo_linux.cpp | 2 +- src/mongo/util/procparser.cpp | 612 +++++++++++ src/mongo/util/procparser.h | 120 ++ src/mongo/util/procparser_test.cpp | 526 +++++++++ .../util/signal_handlers_synchronous_test.cpp | 2 +- .../boost-1.56.0/libs/thread/src/win32/thread.cpp | 5 +- src/third_party/mozjs-38/extract.sh | 1 + .../mozjs-38/extract/js/public/HashTable.h | 25 +- .../mozjs-38/extract/js/src/jit/CodeGenerator.cpp | 6 +- .../mozjs-38/extract/js/src/jit/ValueNumbering.cpp | 1 + .../extract/js/src/jit/none/MacroAssembler-none.h | 6 - src/third_party/mozjs-38/extract/js/src/jsapi.h | 8 - src/third_party/mozjs-38/extract/js/src/jscntxt.h | 4 +- src/third_party/mozjs-38/extract/js/src/jsobj.cpp | 5 +- .../mozjs-38/extract/js/src/jsscript.cpp | 5 +- src/third_party/mozjs-38/extract/js/src/jsscript.h | 2 +- .../mozjs-38/extract/js/src/jswatchpoint.cpp | 4 +- .../mozjs-38/extract/js/src/selfhosted.out.h | 6 +- .../mozjs-38/extract/js/src/vm/Shape.cpp | 4 +- .../mozjs-38/extract/js/src/vm/StructuredClone.cpp | 15 +- .../mozjs-38/extract/js/src/vm/TypeInference.cpp | 6 + .../mozjs-38/extract/js/src/vm/TypeInference.h | 3 + src/third_party/mozjs-38/extract/mfbt/Opaque.h | 44 + src/third_party/mozjs-38/extract/mfbt/moz.build | 1 + src/third_party/mozjs-38/get_sources.sh | 4 +- src/third_party/mozjs-38/include/js/HashTable.h | 25 +- src/third_party/mozjs-38/include/jsapi.h | 8 - src/third_party/mozjs-38/include/mozilla/Opaque.h | 44 + src/third_party/mozjs-38/mongodb.patch | 24 + .../platform/arm64/linux/build/js-confdefs.h | 6 +- .../platform/arm64/linux/include/js-config.h | 2 +- .../platform/i386/linux/build/js-confdefs.h | 6 +- .../platform/i386/linux/include/js-config.h | 2 +- .../platform/i386/openbsd/build/js-confdefs.h | 6 +- .../platform/i386/openbsd/include/js-config.h | 2 +- .../platform/i386/windows/build/js-confdefs.h | 6 +- .../platform/i386/windows/include/js-config.h | 2 +- .../platform/x86_64/freebsd/build/js-confdefs.h | 6 +- .../platform/x86_64/freebsd/include/js-config.h | 2 +- .../platform/x86_64/linux/build/js-confdefs.h | 6 +- .../platform/x86_64/linux/include/js-config.h | 2 +- .../platform/x86_64/openbsd/build/js-confdefs.h | 6 +- .../platform/x86_64/openbsd/include/js-config.h | 2 +- .../platform/x86_64/osx/build/js-confdefs.h | 6 +- .../platform/x86_64/osx/include/js-config.h | 2 +- .../platform/x86_64/solaris/build/js-confdefs.h | 6 +- .../platform/x86_64/solaris/include/js-config.h | 2 +- .../platform/x86_64/windows/build/js-confdefs.h | 6 +- .../platform/x86_64/windows/include/js-config.h | 2 +- src/third_party/wiredtiger/.gitignore | 37 +- src/third_party/wiredtiger/NEWS | 415 +++++++ src/third_party/wiredtiger/README | 6 +- src/third_party/wiredtiger/RELEASE_INFO | 4 +- src/third_party/wiredtiger/SConstruct | 3 +- .../wiredtiger/api/leveldb/basho/perf_count.cc | 9 +- src/third_party/wiredtiger/api/leveldb/hyper_wt.cc | 18 +- .../wiredtiger/api/leveldb/leveldb_wt.h | 1 + src/third_party/wiredtiger/bench/wtperf/config.c | 140 ++- .../wiredtiger/bench/wtperf/idle_table_cycle.c | 6 +- src/third_party/wiredtiger/bench/wtperf/misc.c | 4 +- .../bench/wtperf/runners/500m-btree-50r50u.wtperf | 2 +- .../bench/wtperf/runners/500m-btree-80r20u.wtperf | 2 +- .../wtperf/runners/500m-btree-populate.wtperf | 2 +- .../bench/wtperf/runners/500m-btree-rdonly.wtperf | 2 +- .../bench/wtperf/runners/checkpoint-stress.wtperf | 2 +- .../bench/wtperf/runners/evict-btree-1.wtperf | 2 +- .../wtperf/runners/evict-btree-readonly.wtperf | 2 +- .../wtperf/runners/evict-btree-stress-multi.wtperf | 2 +- .../bench/wtperf/runners/evict-btree-stress.wtperf | 2 +- .../bench/wtperf/runners/evict-btree.wtperf | 2 +- .../bench/wtperf/runners/evict-lsm-1.wtperf | 2 +- .../bench/wtperf/runners/evict-lsm-readonly.wtperf | 2 +- .../bench/wtperf/runners/evict-lsm.wtperf | 2 +- .../wiredtiger/bench/wtperf/runners/log.wtperf | 2 +- .../bench/wtperf/runners/many-table-stress.wtperf | 19 + .../wtperf/runners/mongodb-secondary-apply.wtperf | 2 +- .../runners/multi-btree-read-heavy-stress.wtperf | 2 +- .../bench/wtperf/runners/multi-btree-stress.wtperf | 2 +- .../runners/multi-btree-zipfian-populate.wtperf | 2 +- .../runners/multi-btree-zipfian-workload.wtperf | 2 +- .../wiredtiger/bench/wtperf/runners/wtperf_run.sh | 5 + .../bench/wtperf/stress/btree-split-stress.wtperf | 5 +- src/third_party/wiredtiger/bench/wtperf/track.c | 4 +- src/third_party/wiredtiger/bench/wtperf/wtperf.c | 346 ++++-- src/third_party/wiredtiger/bench/wtperf/wtperf.h | 4 + .../wiredtiger/bench/wtperf/wtperf_opt.i | 12 +- src/third_party/wiredtiger/build_posix/Make.base | 2 + .../wiredtiger/build_posix/Make.subdirs | 2 + .../build_posix/aclocal/ax_check_compile_flag.m4 | 74 ++ .../wiredtiger/build_posix/aclocal/options.m4 | 2 +- .../wiredtiger/build_posix/aclocal/strict.m4 | 5 + .../wiredtiger/build_posix/aclocal/version-set.m4 | 8 +- .../wiredtiger/build_posix/aclocal/version.m4 | 2 +- .../wiredtiger/build_posix/configure.ac.in | 65 +- src/third_party/wiredtiger/dist/api_config.py | 3 +- src/third_party/wiredtiger/dist/api_data.py | 62 +- src/third_party/wiredtiger/dist/api_err.py | 2 +- src/third_party/wiredtiger/dist/filelist | 4 +- src/third_party/wiredtiger/dist/flags.py | 13 +- .../wiredtiger/dist/package/wiredtiger.spec | 2 +- src/third_party/wiredtiger/dist/s_all | 28 +- src/third_party/wiredtiger/dist/s_define.list | 12 +- src/third_party/wiredtiger/dist/s_docs | 3 +- src/third_party/wiredtiger/dist/s_funcs.list | 1 - src/third_party/wiredtiger/dist/s_prototypes | 2 +- src/third_party/wiredtiger/dist/s_python | 4 +- src/third_party/wiredtiger/dist/s_stat | 3 - src/third_party/wiredtiger/dist/s_string.ok | 44 +- src/third_party/wiredtiger/dist/s_style | 19 +- src/third_party/wiredtiger/dist/s_void | 60 +- src/third_party/wiredtiger/dist/stat_data.py | 15 +- src/third_party/wiredtiger/examples/c/Makefile.am | 1 - src/third_party/wiredtiger/examples/c/ex_all.c | 10 +- src/third_party/wiredtiger/examples/c/ex_async.c | 6 +- src/third_party/wiredtiger/examples/c/ex_backup.c | 28 +- .../wiredtiger/examples/c/ex_data_source.c | 16 + src/third_party/wiredtiger/examples/c/ex_encrypt.c | 4 +- .../wiredtiger/examples/c/ex_file_system.c | 159 +-- src/third_party/wiredtiger/examples/c/ex_log.c | 12 +- src/third_party/wiredtiger/examples/c/ex_scope.c | 217 ---- src/third_party/wiredtiger/examples/c/ex_sync.c | 20 +- src/third_party/wiredtiger/examples/c/ex_thread.c | 7 +- .../java/com/wiredtiger/examples/ex_log.java | 75 +- .../ext/compressors/zlib/zlib_compress.c | 57 +- .../ext/compressors/zstd/zstd_compress.c | 37 +- .../wiredtiger/ext/datasources/helium/helium.c | 11 +- .../wiredtiger/ext/encryptors/rotn/rotn_encrypt.c | 42 +- .../wiredtiger/ext/test/fail_fs/Makefile.am | 9 + .../wiredtiger/ext/test/fail_fs/fail_fs.c | 847 ++++++++++++++ .../wiredtiger/ext/test/kvs_bdb/kvs_bdb.c | 3 +- src/third_party/wiredtiger/import.data | 6 + src/third_party/wiredtiger/lang/java/Makefile.am | 1 + src/third_party/wiredtiger/lang/java/java_doc.i | 1 + .../src/com/wiredtiger/db/PackOutputStream.java | 23 +- src/third_party/wiredtiger/lang/java/wiredtiger.i | 78 +- src/third_party/wiredtiger/lang/python/Makefile.am | 3 +- src/third_party/wiredtiger/lang/python/setup.py | 5 +- .../wiredtiger/lang/python/setup_pip.py | 408 +++++++ .../wiredtiger/lang/python/wiredtiger/pip_init.py | 48 + src/third_party/wiredtiger/src/async/async_api.c | 17 +- src/third_party/wiredtiger/src/async/async_op.c | 3 +- .../wiredtiger/src/async/async_worker.c | 2 +- src/third_party/wiredtiger/src/block/block_addr.c | 3 +- src/third_party/wiredtiger/src/block/block_ckpt.c | 1 - .../wiredtiger/src/block/block_compact.c | 22 +- src/third_party/wiredtiger/src/block/block_ext.c | 15 +- src/third_party/wiredtiger/src/block/block_open.c | 6 + src/third_party/wiredtiger/src/block/block_read.c | 4 +- src/third_party/wiredtiger/src/block/block_slvg.c | 1 - src/third_party/wiredtiger/src/block/block_vrfy.c | 2 +- src/third_party/wiredtiger/src/block/block_write.c | 4 +- src/third_party/wiredtiger/src/bloom/bloom.c | 11 +- src/third_party/wiredtiger/src/btree/bt_compact.c | 17 +- src/third_party/wiredtiger/src/btree/bt_curnext.c | 15 +- src/third_party/wiredtiger/src/btree/bt_curprev.c | 19 +- src/third_party/wiredtiger/src/btree/bt_cursor.c | 707 +++++++----- src/third_party/wiredtiger/src/btree/bt_debug.c | 63 +- src/third_party/wiredtiger/src/btree/bt_delete.c | 7 +- src/third_party/wiredtiger/src/btree/bt_discard.c | 94 +- src/third_party/wiredtiger/src/btree/bt_handle.c | 181 ++- src/third_party/wiredtiger/src/btree/bt_io.c | 9 +- src/third_party/wiredtiger/src/btree/bt_misc.c | 1 + src/third_party/wiredtiger/src/btree/bt_ovfl.c | 8 +- src/third_party/wiredtiger/src/btree/bt_page.c | 28 +- src/third_party/wiredtiger/src/btree/bt_random.c | 432 ++++++++ src/third_party/wiredtiger/src/btree/bt_read.c | 17 +- .../wiredtiger/src/btree/bt_rebalance.c | 17 +- src/third_party/wiredtiger/src/btree/bt_ret.c | 173 +-- src/third_party/wiredtiger/src/btree/bt_slvg.c | 47 +- src/third_party/wiredtiger/src/btree/bt_split.c | 336 +++--- src/third_party/wiredtiger/src/btree/bt_stat.c | 5 +- src/third_party/wiredtiger/src/btree/bt_sync.c | 20 +- src/third_party/wiredtiger/src/btree/bt_vrfy.c | 46 +- src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c | 8 +- src/third_party/wiredtiger/src/btree/bt_walk.c | 113 +- src/third_party/wiredtiger/src/btree/col_modify.c | 14 +- src/third_party/wiredtiger/src/btree/col_srch.c | 7 +- src/third_party/wiredtiger/src/btree/row_key.c | 12 +- src/third_party/wiredtiger/src/btree/row_modify.c | 31 +- src/third_party/wiredtiger/src/btree/row_srch.c | 230 +--- src/third_party/wiredtiger/src/cache/cache_las.c | 5 +- .../wiredtiger/src/checksum/power8/crc32_wrapper.c | 4 +- .../wiredtiger/src/checksum/software/checksum.c | 1 + .../wiredtiger/src/checksum/zseries/crc32-s390x.c | 26 +- src/third_party/wiredtiger/src/config/config.c | 2 +- src/third_party/wiredtiger/src/config/config_api.c | 41 +- .../wiredtiger/src/config/config_collapse.c | 1 + src/third_party/wiredtiger/src/config/config_def.c | 323 +++--- src/third_party/wiredtiger/src/config/config_ext.c | 77 +- src/third_party/wiredtiger/src/conn/conn_api.c | 130 ++- src/third_party/wiredtiger/src/conn/conn_cache.c | 9 +- .../wiredtiger/src/conn/conn_cache_pool.c | 68 +- src/third_party/wiredtiger/src/conn/conn_ckpt.c | 28 +- src/third_party/wiredtiger/src/conn/conn_dhandle.c | 182 +-- src/third_party/wiredtiger/src/conn/conn_handle.c | 34 +- src/third_party/wiredtiger/src/conn/conn_log.c | 88 +- src/third_party/wiredtiger/src/conn/conn_open.c | 61 +- src/third_party/wiredtiger/src/conn/conn_stat.c | 35 +- src/third_party/wiredtiger/src/conn/conn_sweep.c | 44 +- src/third_party/wiredtiger/src/cursor/cur_backup.c | 28 +- src/third_party/wiredtiger/src/cursor/cur_config.c | 2 - src/third_party/wiredtiger/src/cursor/cur_ds.c | 20 +- src/third_party/wiredtiger/src/cursor/cur_file.c | 169 +-- src/third_party/wiredtiger/src/cursor/cur_index.c | 55 +- src/third_party/wiredtiger/src/cursor/cur_join.c | 39 +- src/third_party/wiredtiger/src/cursor/cur_json.c | 186 ++-- src/third_party/wiredtiger/src/cursor/cur_log.c | 66 +- .../wiredtiger/src/cursor/cur_metadata.c | 4 +- src/third_party/wiredtiger/src/cursor/cur_stat.c | 5 +- src/third_party/wiredtiger/src/cursor/cur_std.c | 8 +- src/third_party/wiredtiger/src/cursor/cur_table.c | 71 +- src/third_party/wiredtiger/src/docs/Doxyfile | 8 + .../wiredtiger/src/docs/command-line.dox | 46 + src/third_party/wiredtiger/src/docs/cursor-ops.dox | 4 - .../wiredtiger/src/docs/cursor-random.dox | 5 - .../wiredtiger/src/docs/file-formats.dox | 6 +- .../wiredtiger/src/docs/programming.dox | 3 +- src/third_party/wiredtiger/src/docs/spell.ok | 2 + src/third_party/wiredtiger/src/docs/top/main.dox | 12 +- .../wiredtiger/src/docs/transactions.dox | 2 +- .../wiredtiger/src/docs/tune-compression.dox | 62 -- .../src/docs/tune-page-size-and-comp.dox | 426 +++++++ .../wiredtiger/src/docs/tune-page-sizes.dox | 142 --- src/third_party/wiredtiger/src/docs/upgrading.dox | 93 +- src/third_party/wiredtiger/src/docs/wtperf.dox | 8 +- src/third_party/wiredtiger/src/evict/evict_file.c | 23 +- src/third_party/wiredtiger/src/evict/evict_lru.c | 910 ++++++++++----- src/third_party/wiredtiger/src/evict/evict_page.c | 76 +- src/third_party/wiredtiger/src/evict/evict_stat.c | 2 +- src/third_party/wiredtiger/src/include/api.h | 48 +- src/third_party/wiredtiger/src/include/bitstring.i | 2 +- src/third_party/wiredtiger/src/include/btmem.h | 162 ++- src/third_party/wiredtiger/src/include/btree.h | 53 +- src/third_party/wiredtiger/src/include/btree.i | 155 +-- src/third_party/wiredtiger/src/include/buf.i | 37 +- src/third_party/wiredtiger/src/include/cache.h | 17 +- src/third_party/wiredtiger/src/include/cache.i | 17 +- src/third_party/wiredtiger/src/include/cell.i | 16 +- src/third_party/wiredtiger/src/include/column.i | 27 +- src/third_party/wiredtiger/src/include/compact.h | 4 +- src/third_party/wiredtiger/src/include/config.h | 67 +- .../wiredtiger/src/include/connection.h | 50 +- src/third_party/wiredtiger/src/include/cursor.h | 12 +- src/third_party/wiredtiger/src/include/cursor.i | 62 +- src/third_party/wiredtiger/src/include/dhandle.h | 22 +- src/third_party/wiredtiger/src/include/error.h | 6 +- src/third_party/wiredtiger/src/include/extern.h | 153 +-- .../wiredtiger/src/include/extern_posix.h | 21 +- .../wiredtiger/src/include/extern_win.h | 9 +- src/third_party/wiredtiger/src/include/flags.h | 93 +- src/third_party/wiredtiger/src/include/gcc.h | 2 - src/third_party/wiredtiger/src/include/hardware.h | 20 +- src/third_party/wiredtiger/src/include/intpack.i | 30 +- src/third_party/wiredtiger/src/include/lint.h | 16 +- src/third_party/wiredtiger/src/include/log.h | 26 +- src/third_party/wiredtiger/src/include/lsm.h | 36 +- src/third_party/wiredtiger/src/include/meta.h | 7 +- src/third_party/wiredtiger/src/include/misc.h | 22 +- src/third_party/wiredtiger/src/include/misc.i | 106 +- src/third_party/wiredtiger/src/include/msvc.h | 2 - src/third_party/wiredtiger/src/include/mutex.h | 73 +- src/third_party/wiredtiger/src/include/mutex.i | 42 +- src/third_party/wiredtiger/src/include/os.h | 10 +- .../wiredtiger/src/include/os_windows.h | 22 - src/third_party/wiredtiger/src/include/packing.i | 57 +- src/third_party/wiredtiger/src/include/schema.h | 211 +++- src/third_party/wiredtiger/src/include/serial.i | 30 +- src/third_party/wiredtiger/src/include/session.h | 49 +- src/third_party/wiredtiger/src/include/stat.h | 17 +- .../wiredtiger/src/include/thread_group.h | 2 +- src/third_party/wiredtiger/src/include/txn.h | 25 +- src/third_party/wiredtiger/src/include/txn.i | 23 +- .../wiredtiger/src/include/verify_build.h | 22 +- .../wiredtiger/src/include/wiredtiger.in | 691 +++++++----- .../wiredtiger/src/include/wiredtiger_ext.h | 36 +- .../wiredtiger/src/include/wt_internal.h | 6 +- src/third_party/wiredtiger/src/log/log.c | 132 ++- src/third_party/wiredtiger/src/log/log_slot.c | 394 ++++--- src/third_party/wiredtiger/src/lsm/lsm_cursor.c | 147 ++- .../wiredtiger/src/lsm/lsm_cursor_bulk.c | 4 +- src/third_party/wiredtiger/src/lsm/lsm_manager.c | 68 +- src/third_party/wiredtiger/src/lsm/lsm_merge.c | 20 +- src/third_party/wiredtiger/src/lsm/lsm_meta.c | 13 +- src/third_party/wiredtiger/src/lsm/lsm_stat.c | 14 +- src/third_party/wiredtiger/src/lsm/lsm_tree.c | 136 ++- src/third_party/wiredtiger/src/lsm/lsm_work_unit.c | 79 +- src/third_party/wiredtiger/src/lsm/lsm_worker.c | 25 +- src/third_party/wiredtiger/src/meta/meta_apply.c | 6 +- src/third_party/wiredtiger/src/meta/meta_ckpt.c | 10 +- src/third_party/wiredtiger/src/meta/meta_ext.c | 4 +- src/third_party/wiredtiger/src/meta/meta_table.c | 18 +- src/third_party/wiredtiger/src/meta/meta_track.c | 2 +- src/third_party/wiredtiger/src/meta/meta_turtle.c | 41 +- .../wiredtiger/src/os_common/filename.c | 19 +- .../wiredtiger/src/os_common/os_alloc.c | 2 + .../wiredtiger/src/os_common/os_errno.c | 2 +- .../wiredtiger/src/os_common/os_fs_inmemory.c | 2 - .../wiredtiger/src/os_common/os_fstream.c | 2 +- .../wiredtiger/src/os_common/os_getopt.c | 1 + .../wiredtiger/src/os_common/os_strtouq.c | 1 + src/third_party/wiredtiger/src/os_posix/os_dir.c | 2 - src/third_party/wiredtiger/src/os_posix/os_fs.c | 19 +- .../wiredtiger/src/os_posix/os_getenv.c | 1 + .../wiredtiger/src/os_posix/os_mtx_cond.c | 53 +- .../wiredtiger/src/os_posix/os_setvbuf.c | 2 + src/third_party/wiredtiger/src/os_posix/os_sleep.c | 1 + .../wiredtiger/src/os_posix/os_snprintf.c | 27 + .../wiredtiger/src/os_posix/os_thread.c | 25 +- src/third_party/wiredtiger/src/os_posix/os_time.c | 1 + src/third_party/wiredtiger/src/os_posix/os_yield.c | 9 + src/third_party/wiredtiger/src/os_win/os_dir.c | 2 - src/third_party/wiredtiger/src/os_win/os_fs.c | 36 +- .../wiredtiger/src/os_win/os_mtx_cond.c | 49 +- .../wiredtiger/src/os_win/os_snprintf.c | 50 +- src/third_party/wiredtiger/src/os_win/os_thread.c | 20 +- src/third_party/wiredtiger/src/os_win/os_utf8.c | 2 - .../wiredtiger/src/os_win/os_vsnprintf.c | 41 - src/third_party/wiredtiger/src/os_win/os_yield.c | 8 + .../wiredtiger/src/reconcile/rec_track.c | 20 +- .../wiredtiger/src/reconcile/rec_write.c | 1161 +++++++++++--------- .../wiredtiger/src/schema/schema_alter.c | 180 +++ .../wiredtiger/src/schema/schema_create.c | 5 +- .../wiredtiger/src/schema/schema_drop.c | 8 +- .../wiredtiger/src/schema/schema_list.c | 2 +- .../wiredtiger/src/schema/schema_rename.c | 2 +- .../wiredtiger/src/schema/schema_util.c | 6 +- .../wiredtiger/src/schema/schema_worker.c | 20 +- .../wiredtiger/src/session/session_api.c | 250 +++-- .../wiredtiger/src/session/session_compact.c | 246 +++-- .../wiredtiger/src/session/session_dhandle.c | 115 +- .../wiredtiger/src/session/session_salvage.c | 2 +- src/third_party/wiredtiger/src/support/cond_auto.c | 80 +- src/third_party/wiredtiger/src/support/crypto.c | 1 - src/third_party/wiredtiger/src/support/err.c | 123 ++- src/third_party/wiredtiger/src/support/hazard.c | 314 ++++-- src/third_party/wiredtiger/src/support/hex.c | 10 +- src/third_party/wiredtiger/src/support/mtx_rw.c | 499 +++++---- src/third_party/wiredtiger/src/support/rand.c | 15 + src/third_party/wiredtiger/src/support/scratch.c | 9 +- src/third_party/wiredtiger/src/support/stat.c | 60 +- .../wiredtiger/src/support/thread_group.c | 89 +- src/third_party/wiredtiger/src/txn/txn.c | 188 +++- src/third_party/wiredtiger/src/txn/txn_ckpt.c | 547 +++++---- src/third_party/wiredtiger/src/txn/txn_log.c | 21 +- src/third_party/wiredtiger/src/txn/txn_nsnap.c | 49 +- src/third_party/wiredtiger/src/txn/txn_recover.c | 6 +- src/third_party/wiredtiger/src/utilities/util.h | 4 +- .../wiredtiger/src/utilities/util_alter.c | 53 + .../wiredtiger/src/utilities/util_backup.c | 15 +- .../wiredtiger/src/utilities/util_compact.c | 14 +- .../wiredtiger/src/utilities/util_create.c | 12 +- .../wiredtiger/src/utilities/util_drop.c | 10 +- .../wiredtiger/src/utilities/util_dump.c | 242 +++- .../wiredtiger/src/utilities/util_list.c | 21 +- .../wiredtiger/src/utilities/util_load.c | 17 +- .../wiredtiger/src/utilities/util_load_json.c | 27 +- .../wiredtiger/src/utilities/util_loadtext.c | 13 +- .../wiredtiger/src/utilities/util_main.c | 142 ++- .../wiredtiger/src/utilities/util_misc.c | 5 +- .../wiredtiger/src/utilities/util_printlog.c | 14 +- .../wiredtiger/src/utilities/util_read.c | 19 +- .../wiredtiger/src/utilities/util_rebalance.c | 30 +- .../wiredtiger/src/utilities/util_rename.c | 15 +- .../wiredtiger/src/utilities/util_salvage.c | 30 +- .../wiredtiger/src/utilities/util_stat.c | 11 +- .../wiredtiger/src/utilities/util_truncate.c | 52 + .../wiredtiger/src/utilities/util_upgrade.c | 30 +- .../wiredtiger/src/utilities/util_verify.c | 41 +- .../wiredtiger/src/utilities/util_write.c | 28 +- src/third_party/wiredtiger/test/bloom/test_bloom.c | 15 +- .../wiredtiger/test/checkpoint/checkpointer.c | 26 +- .../wiredtiger/test/checkpoint/smoke.sh | 4 +- .../wiredtiger/test/checkpoint/test_checkpoint.c | 21 +- .../wiredtiger/test/checkpoint/test_checkpoint.h | 1 - .../wiredtiger/test/checkpoint/workers.c | 22 +- src/third_party/wiredtiger/test/csuite/Makefile.am | 22 +- .../wiredtiger/test/csuite/rwlock/main.c | 184 ++++ .../wiredtiger/test/csuite/scope/main.c | 288 +++++ .../test/csuite/wt1965_col_efficiency/main.c | 3 +- .../test/csuite/wt2246_col_append/main.c | 17 +- .../test/csuite/wt2323_join_visibility/main.c | 39 +- .../test/csuite/wt2447_join_main_table/main.c | 12 +- .../test/csuite/wt2535_insert_race/main.c | 5 +- .../test/csuite/wt2592_join_schema/main.c | 9 +- .../test/csuite/wt2834_join_bloom_fix/main.c | 26 +- .../wiredtiger/test/csuite/wt2853_perf/main.c | 21 +- .../test/csuite/wt2909_checkpoint_integrity/main.c | 669 +++++++++++ .../wiredtiger/test/csuite/wt3120_filesys/main.c | 99 ++ .../test/csuite/wt3135_search_near_collator/main.c | 360 ++++++ .../test/csuite/wt3184_dup_index_collator/main.c | 168 +++ .../wiredtiger/test/cursor_order/cursor_order.c | 14 +- .../test/cursor_order/cursor_order_file.c | 30 +- .../test/cursor_order/cursor_order_ops.c | 20 +- src/third_party/wiredtiger/test/fops/file.c | 33 +- src/third_party/wiredtiger/test/fops/t.c | 21 +- src/third_party/wiredtiger/test/format/backup.c | 4 +- src/third_party/wiredtiger/test/format/config.c | 190 +++- src/third_party/wiredtiger/test/format/config.h | 24 +- src/third_party/wiredtiger/test/format/format.h | 7 +- src/third_party/wiredtiger/test/format/ops.c | 445 +++++--- src/third_party/wiredtiger/test/format/rebalance.c | 16 +- src/third_party/wiredtiger/test/format/salvage.c | 25 +- src/third_party/wiredtiger/test/format/t.c | 15 +- src/third_party/wiredtiger/test/format/util.c | 91 +- src/third_party/wiredtiger/test/format/wts.c | 145 ++- src/third_party/wiredtiger/test/huge/huge.c | 7 +- .../java/com/wiredtiger/test/CursorTest03.java | 175 +++ .../java/com/wiredtiger/test/WiredTigerSuite.java | 1 + src/third_party/wiredtiger/test/manydbs/manydbs.c | 10 +- src/third_party/wiredtiger/test/mciproject.yml | 2 +- .../wiredtiger/test/readonly/readonly.c | 52 +- .../wiredtiger/test/recovery/random-abort.c | 54 +- .../wiredtiger/test/recovery/truncated-log.c | 24 +- src/third_party/wiredtiger/test/salvage/salvage.c | 33 +- src/third_party/wiredtiger/test/suite/run.py | 3 +- .../wiredtiger/test/suite/test_alter01.py | 169 +++ .../wiredtiger/test/suite/test_async01.py | 2 +- .../wiredtiger/test/suite/test_async02.py | 2 +- .../wiredtiger/test/suite/test_backup03.py | 2 +- .../wiredtiger/test/suite/test_backup04.py | 2 +- .../wiredtiger/test/suite/test_bug011.py | 17 +- .../wiredtiger/test/suite/test_collator.py | 32 +- .../wiredtiger/test/suite/test_compact02.py | 24 +- .../wiredtiger/test/suite/test_compress01.py | 20 +- .../wiredtiger/test/suite/test_config03.py | 12 +- .../wiredtiger/test/suite/test_config04.py | 4 - .../wiredtiger/test/suite/test_cursor01.py | 1 + .../wiredtiger/test/suite/test_cursor07.py | 2 +- .../wiredtiger/test/suite/test_cursor08.py | 20 +- .../wiredtiger/test/suite/test_cursor10.py | 127 +++ .../wiredtiger/test/suite/test_cursor11.py | 159 +++ .../wiredtiger/test/suite/test_cursor_random.py | 49 + src/third_party/wiredtiger/test/suite/test_dump.py | 54 +- .../wiredtiger/test/suite/test_encrypt01.py | 39 +- .../wiredtiger/test/suite/test_encrypt02.py | 48 +- .../wiredtiger/test/suite/test_encrypt03.py | 37 +- .../wiredtiger/test/suite/test_encrypt04.py | 45 +- .../wiredtiger/test/suite/test_encrypt05.py | 39 +- .../wiredtiger/test/suite/test_encrypt06.py | 39 +- .../wiredtiger/test/suite/test_encrypt07.py | 35 +- .../wiredtiger/test/suite/test_hazard.py | 58 + .../wiredtiger/test/suite/test_index02.py | 71 +- .../wiredtiger/test/suite/test_inmem01.py | 51 +- .../wiredtiger/test/suite/test_intpack.py | 4 +- .../wiredtiger/test/suite/test_join01.py | 11 +- .../wiredtiger/test/suite/test_join03.py | 30 +- .../wiredtiger/test/suite/test_join04.py | 30 +- .../wiredtiger/test/suite/test_join07.py | 30 +- .../wiredtiger/test/suite/test_jsondump02.py | 18 + .../wiredtiger/test/suite/test_lsm01.py | 2 +- .../wiredtiger/test/suite/test_nsnap04.py | 37 +- .../wiredtiger/test/suite/test_overwrite.py | 67 +- .../wiredtiger/test/suite/test_perf001.py | 3 +- .../wiredtiger/test/suite/test_readonly01.py | 3 +- .../wiredtiger/test/suite/test_reconfig01.py | 12 + .../wiredtiger/test/suite/test_reconfig02.py | 3 +- .../wiredtiger/test/suite/test_reconfig04.py | 45 + .../wiredtiger/test/suite/test_schema05.py | 30 +- .../wiredtiger/test/suite/test_schema07.py | 3 +- .../wiredtiger/test/suite/test_shared_cache01.py | 2 +- .../wiredtiger/test/suite/test_shared_cache02.py | 2 +- .../wiredtiger/test/suite/test_stat02.py | 2 +- .../wiredtiger/test/suite/test_sweep01.py | 7 +- .../wiredtiger/test/suite/test_truncate01.py | 10 +- .../wiredtiger/test/suite/test_truncate02.py | 3 +- .../wiredtiger/test/suite/test_txn02.py | 26 +- .../wiredtiger/test/suite/test_txn04.py | 22 +- .../wiredtiger/test/suite/test_txn05.py | 28 +- .../wiredtiger/test/suite/test_txn06.py | 4 +- .../wiredtiger/test/suite/test_txn07.py | 56 +- .../wiredtiger/test/suite/test_txn08.py | 2 +- .../wiredtiger/test/suite/test_txn09.py | 17 +- .../wiredtiger/test/suite/test_txn11.py | 2 +- .../wiredtiger/test/suite/test_txn13.py | 2 +- .../wiredtiger/test/suite/test_txn15.py | 2 +- .../wiredtiger/test/suite/test_util14.py | 92 ++ .../wiredtiger/test/suite/test_util15.py | 71 ++ .../wiredtiger/test/suite/test_util16.py | 71 ++ .../wiredtiger/test/suite/test_util17.py | 57 + src/third_party/wiredtiger/test/suite/wtdataset.py | 96 +- src/third_party/wiredtiger/test/suite/wttest.py | 87 +- .../wiredtiger/test/syscall/Makefile.am | 7 + src/third_party/wiredtiger/test/syscall/syscall.py | 857 +++++++++++++++ .../wiredtiger/test/syscall/wt2336_base/base.run | 180 +++ .../wiredtiger/test/syscall/wt2336_base/main.c | 69 ++ src/third_party/wiredtiger/test/thread/file.c | 35 +- src/third_party/wiredtiger/test/thread/rw.c | 41 +- src/third_party/wiredtiger/test/thread/stats.c | 3 +- src/third_party/wiredtiger/test/thread/t.c | 14 +- src/third_party/wiredtiger/test/utility/misc.c | 42 +- .../wiredtiger/test/utility/parse_opts.c | 11 +- .../wiredtiger/test/utility/test_util.h | 9 +- src/third_party/wiredtiger/test/utility/thread.c | 7 +- .../wiredtiger/test/windows/windows_shim.h | 11 +- .../wiredtiger/test/wtperf/test_conf_dump.py | 296 +++++ .../wiredtiger/tools/wtstats/stat_data.py | 10 + .../wiredtiger/tools/wtstats/wtstats.html.template | 27 +- version.json | 4 +- 912 files changed, 36192 insertions(+), 13814 deletions(-) create mode 100644 buildscripts/package_test/.kitchen.yml create mode 100644 buildscripts/package_test/metadata.rb create mode 100644 buildscripts/package_test/recipes/install_mongodb.rb create mode 100644 buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb create mode 100644 buildscripts/resmokeconfig/suites/multiversion_auth.yml create mode 100644 buildscripts/resmokelib/reportfile.py create mode 100644 buildscripts/resmokelib/sighandler.py create mode 100644 etc/scons/gold_incremental_link.vars create mode 100644 etc/scons/mongodbtoolchain_clang.vars create mode 100644 etc/scons/mongodbtoolchain_gcc.vars create mode 100644 jstests/concurrency/fsm_workloads/create_database.js create mode 100644 jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js create mode 100644 jstests/core/diagdata.js delete mode 100644 jstests/core/drop2.js create mode 100644 jstests/core/evalh.js create mode 100644 jstests/core/evalj.js create mode 100644 jstests/core/killop_drop_collection.js create mode 100644 jstests/core/list_indexes_invalidation.js create mode 100644 jstests/core/list_namespaces_invalidation.js create mode 100644 jstests/core/server22053.js create mode 100644 jstests/core/server25192.js create mode 100644 jstests/libs/check_log.js delete mode 100644 jstests/libs/election_timing_test.js create mode 100644 jstests/libs/write_concern_util.js create mode 100644 jstests/noPassthrough/read_majority_reads.js create mode 100644 jstests/noPassthrough/shell_history.js create mode 100644 jstests/replsets/do_not_sync_from_stale_sync_source.js create mode 100644 jstests/replsets/double_rollback_early.js delete mode 100644 jstests/replsets/election_timing.js create mode 100644 jstests/replsets/last_vote.js create mode 100644 jstests/replsets/libs/tags.js create mode 100644 jstests/replsets/no_flapping_during_network_partition.js create mode 100644 jstests/replsets/oplog_replay_on_startup_update_and_delete.js create mode 100644 jstests/replsets/read_committed_stale_history.js create mode 100644 jstests/replsets/read_committed_with_catalog_changes.js create mode 100644 jstests/replsets/request_primary_stepdown.js create mode 100644 jstests/replsets/rollback_after_sync_source_selection.js delete mode 100644 jstests/replsets/rollback_empty_ns.js delete mode 100644 jstests/replsets/rollback_empty_o.js delete mode 100644 jstests/replsets/rollback_empty_o2.js create mode 100644 jstests/replsets/rollback_with_socket_error_then_steady_state.js create mode 100644 jstests/replsets/shutdown_primary.js create mode 100644 jstests/replsets/stepdown_needs_electable_secondary.js create mode 100644 jstests/replsets/stepdown_needs_majority.js delete mode 100644 jstests/replsets/stepdown_wrt_electable.js create mode 100644 jstests/replsets/temp_namespace_restart_as_standalone.js create mode 100644 jstests/replsets/write_concern_after_stepdown.js create mode 100644 jstests/replsets/write_concern_after_stepdown_and_stepup.js create mode 100644 jstests/sharding/migration_critical_section_concurrency.js create mode 100644 jstests/sharding/sync_cluster_config/sccc_read_pref.js create mode 100644 site_scons/site_tools/incremental_link.py create mode 100644 site_scons/site_tools/split_dwarf.py create mode 100644 src/mongo/client/scram_sha1_client_cache.cpp create mode 100644 src/mongo/client/scram_sha1_client_cache.h delete mode 100644 src/mongo/crypto/crypto.h delete mode 100644 src/mongo/crypto/crypto_openssl.cpp delete mode 100644 src/mongo/crypto/crypto_test.cpp delete mode 100644 src/mongo/crypto/crypto_tom.cpp create mode 100644 src/mongo/crypto/sha1_block.cpp create mode 100644 src/mongo/crypto/sha1_block.h create mode 100644 src/mongo/crypto/sha1_block_openssl.cpp create mode 100644 src/mongo/crypto/sha1_block_test.cpp create mode 100644 src/mongo/crypto/sha1_block_tom.cpp create mode 100644 src/mongo/db/auth/authorization_session_for_test.cpp create mode 100644 src/mongo/db/auth/authorization_session_for_test.h create mode 100644 src/mongo/db/auth/sasl_scramsha1_test.cpp create mode 100644 src/mongo/db/ftdc/ftdc_commands.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats.h create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_linux.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_osx.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_windows.cpp create mode 100644 src/mongo/db/stats/fill_locker_info_test.cpp create mode 100644 src/mongo/dbtests/replica_set_tests.cpp create mode 100644 src/mongo/s/commands/cluster_ftdc_commands.cpp create mode 100644 src/mongo/s/query/cluster_query_knobs.cpp create mode 100644 src/mongo/s/query/cluster_query_knobs.h create mode 100644 src/mongo/scripting/deadline_monitor.cpp create mode 100644 src/mongo/stdx/type_traits.h create mode 100644 src/mongo/util/perfctr_collect.cpp create mode 100644 src/mongo/util/perfctr_collect.h create mode 100644 src/mongo/util/perfctr_collect_test.cpp create mode 100644 src/mongo/util/procparser.cpp create mode 100644 src/mongo/util/procparser.h create mode 100644 src/mongo/util/procparser_test.cpp create mode 100644 src/third_party/mozjs-38/extract/mfbt/Opaque.h create mode 100644 src/third_party/mozjs-38/include/mozilla/Opaque.h create mode 100644 src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf create mode 100644 src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 mode change 100644 => 100755 src/third_party/wiredtiger/dist/s_void delete mode 100644 src/third_party/wiredtiger/examples/c/ex_scope.c create mode 100644 src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am create mode 100644 src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c create mode 100644 src/third_party/wiredtiger/import.data create mode 100644 src/third_party/wiredtiger/lang/python/setup_pip.py create mode 100644 src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py create mode 100644 src/third_party/wiredtiger/src/btree/bt_random.c delete mode 100644 src/third_party/wiredtiger/src/docs/tune-compression.dox create mode 100644 src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox delete mode 100644 src/third_party/wiredtiger/src/docs/tune-page-sizes.dox create mode 100644 src/third_party/wiredtiger/src/os_posix/os_snprintf.c delete mode 100644 src/third_party/wiredtiger/src/os_win/os_vsnprintf.c create mode 100644 src/third_party/wiredtiger/src/schema/schema_alter.c create mode 100644 src/third_party/wiredtiger/src/utilities/util_alter.c create mode 100644 src/third_party/wiredtiger/src/utilities/util_truncate.c create mode 100644 src/third_party/wiredtiger/test/csuite/rwlock/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/scope/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c create mode 100644 src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java create mode 100644 src/third_party/wiredtiger/test/suite/test_alter01.py create mode 100644 src/third_party/wiredtiger/test/suite/test_cursor10.py create mode 100644 src/third_party/wiredtiger/test/suite/test_cursor11.py create mode 100644 src/third_party/wiredtiger/test/suite/test_hazard.py create mode 100644 src/third_party/wiredtiger/test/suite/test_reconfig04.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util14.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util15.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util16.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util17.py create mode 100644 src/third_party/wiredtiger/test/syscall/Makefile.am create mode 100644 src/third_party/wiredtiger/test/syscall/syscall.py create mode 100644 src/third_party/wiredtiger/test/syscall/wt2336_base/base.run create mode 100644 src/third_party/wiredtiger/test/syscall/wt2336_base/main.c create mode 100644 src/third_party/wiredtiger/test/wtperf/test_conf_dump.py diff --git a/SConstruct b/SConstruct index 7eb1536dc6e..3f487e34caf 100644 --- a/SConstruct +++ b/SConstruct @@ -716,6 +716,10 @@ env_vars.Add('VERBOSE', default='auto', ) +env_vars.Add('WINDOWS_OPENSSL_BIN', + help='Sets the path to the openssl binaries for packaging', + default='c:/openssl/bin') + # don't run configure if user calls --help if GetOption('help'): Return() @@ -830,8 +834,8 @@ envDict = dict(BUILD_ROOT=buildDir, UNITTEST_LIST='$BUILD_ROOT/unittests.txt', INTEGRATION_TEST_ALIAS='integration_tests', INTEGRATION_TEST_LIST='$BUILD_ROOT/integration_tests.txt', - CONFIGUREDIR=sconsDataDir.Dir('sconf_temp'), - CONFIGURELOG=sconsDataDir.File('config.log'), + CONFIGUREDIR='$BUILD_ROOT/scons/$VARIANT_DIR/sconf_temp', + CONFIGURELOG='$BUILD_ROOT/scons/config.log', INSTALL_DIR=installDir, CONFIG_HEADER_DEFINES={}, LIBDEPS_TAG_EXPANSIONS=[], @@ -1429,6 +1433,7 @@ elif env.TargetOSIs('windows'): 'crypt32.lib', 'kernel32.lib', 'shell32.lib', + 'pdh.lib', 'version.lib', 'winmm.lib', 'ws2_32.lib', @@ -2265,6 +2270,41 @@ def doConfigure(myenv): myenv = conf.Finish() + def CheckCXX14EnableIfT(context): + test_body = """ + #include + #include + + template + struct scons { + bool hasSupport() { return false; } + }; + + template <> + struct scons> { + bool hasSupport() { return true; } + }; + + int main(int argc, char **argv) { + scons<> SCons; + return SCons.hasSupport() ? EXIT_SUCCESS : EXIT_FAILURE; + } + """ + context.Message('Checking for C++14 std::enable_if_t support...') + ret = context.TryCompile(textwrap.dedent(test_body), '.cpp') + context.Result(ret) + return ret + + # Check for std::enable_if_t support without using the __cplusplus macro + conf = Configure(myenv, help=False, custom_tests = { + 'CheckCXX14EnableIfT' : CheckCXX14EnableIfT, + }) + + if conf.CheckCXX14EnableIfT(): + conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_STD_ENABLE_IF_T') + + myenv = conf.Finish() + def CheckCXX14MakeUnique(context): test_body = """ #include @@ -2333,6 +2373,18 @@ def doConfigure(myenv): }) libdeps.setup_conftests(conf) + def addOpenSslLibraryToDistArchive(file_name): + openssl_bin_path = os.path.normpath(env['WINDOWS_OPENSSL_BIN'].lower()) + full_file_name = os.path.join(openssl_bin_path, file_name) + if os.path.exists(full_file_name): + env.Append(ARCHIVE_ADDITIONS=[full_file_name]) + env.Append(ARCHIVE_ADDITION_DIR_MAP={ + openssl_bin_path: "bin" + }) + return True + else: + return False + if has_option( "ssl" ): sslLibName = "ssl" cryptoLibName = "crypto" @@ -2340,6 +2392,12 @@ def doConfigure(myenv): sslLibName = "ssleay32" cryptoLibName = "libeay32" + # Add the SSL binaries to the zip file distribution + files = ['ssleay32.dll', 'libeay32.dll'] + for extra_file in files: + if not addOpenSslLibraryToDistArchive(extra_file): + print("WARNING: Cannot find SSL library '%s'" % extra_file) + # Used to import system certificate keychains if conf.env.TargetOSIs('osx'): conf.env.AppendUnique(FRAMEWORKS=[ @@ -2565,10 +2623,20 @@ def doConfigure(myenv): env = doConfigure( env ) +# If the flags in the environment are configured for -gsplit-dwarf, +# inject the necessary emitter. +split_dwarf = Tool('split_dwarf') +if split_dwarf.exists(env): + split_dwarf(env) + # Load the compilation_db tool. We want to do this after configure so we don't end up with # compilation database entries for the configure tests, which is weird. env.Tool("compilation_db") +incremental_link = Tool('incremental_link') +if incremental_link.exists(env): + incremental_link(env) + def checkErrorCodes(): import buildscripts.errorcodes as x if x.checkErrorCodes() == False: @@ -2584,7 +2652,7 @@ def doLint( env , target , source ): raise Exception("ESLint errors") import buildscripts.clang_format - if not buildscripts.clang_format.lint(None, []): + if not buildscripts.clang_format.lint_all(None): raise Exception("clang-format lint errors") import buildscripts.lint diff --git a/buildscripts/buildlogger.py b/buildscripts/buildlogger.py index 48eb468a6e2..531c9cc9e9a 100644 --- a/buildscripts/buildlogger.py +++ b/buildscripts/buildlogger.py @@ -238,6 +238,12 @@ def run_and_echo(command): """ proc = subprocess.Popen(command) + # We write the pid of the spawned process as the first line of buildlogger.py's stdout because + # smoke.py expects to use it to terminate processes individually if already running inside a job + # object. + sys.stdout.write("[buildlogger.py] pid: %d\n" % (proc.pid)) + sys.stdout.flush() + def handle_sigterm(signum, frame): try: proc.send_signal(signum) @@ -415,6 +421,12 @@ def loop_and_callback(command, callback): stderr=subprocess.STDOUT, ) + # We write the pid of the spawned process as the first line of buildlogger.py's stdout because + # smoke.py expects to use it to terminate processes individually if already running inside a job + # object. + sys.stdout.write("[buildlogger.py] pid: %d\n" % (proc.pid)) + sys.stdout.flush() + def handle_sigterm(signum, frame): try: proc.send_signal(signum) diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py index a756ff58ae5..99c3e9cf849 100755 --- a/buildscripts/clang_format.py +++ b/buildscripts/clang_format.py @@ -31,7 +31,6 @@ from multiprocessing import cpu_count if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) -from buildscripts.resmokelib.utils import globstar from buildscripts import moduleconfig @@ -183,7 +182,6 @@ def get_clang_format_from_linux_cache(dest_file): # Destination Path shutil.move("llvm/Release/bin/clang-format", dest_file) - class ClangFormat(object): """Class encapsulates finding a suitable copy of clang-format, and linting/formating an individual file @@ -318,8 +316,6 @@ def parallel_process(items, func): except NotImplementedError: cpus = 1 - # print("Running across %d cpus" % (cpus)) - task_queue = Queue.Queue() # Use a list so that worker function will capture this variable @@ -412,18 +408,16 @@ class Repo(object): def __init__(self, path): self.path = path - # Get candidate files - self.candidate_files = self._get_candidate_files() - self.root = self._get_root() def _callgito(self, args): """Call git for this repository """ # These two flags are the equivalent of -C in newer versions of Git - # but we use these to support versions back to ~1.8 + # but we use these to support versions pre 1.8.5 but it depends on the command + # and what the current directory is return callo(['git', '--git-dir', os.path.join(self.path, ".git"), - '--work-tree', self.path] + args) + '--work-tree', self.path] + args) def _get_local_dir(self, path): """Get a directory path relative to the git root directory @@ -433,13 +427,10 @@ class Repo(object): return path def get_candidates(self, candidates): - """Get the set of candidate files to check by doing an intersection - between the input list, and the list of candidates in the repository + """Get the set of candidate files to check by querying the repository Returns the full path to the file for clang-format to consume. """ - # NOTE: Files may have an absolute root (i.e. leading /) - if candidates is not None and len(candidates) > 0: candidates = [self._get_local_dir(f) for f in candidates] valid_files = list(set(candidates).intersection(self.get_candidate_files())) @@ -448,6 +439,7 @@ class Repo(object): # Get the full file name here valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files] + return valid_files def get_root(self): @@ -462,15 +454,10 @@ class Repo(object): return gito.rstrip() - def get_candidate_files(self): - """Get a list of candidate files - """ - return self._get_candidate_files() - - def _get_candidate_files(self): - """Query git to get a list of all files in the repo to consider for analysis + def _git_ls_files(self, cmd): + """Run git-ls-files and filter the list of files to a valid candidate list """ - gito = self._callgito(["ls-files"]) + gito = self._callgito(cmd) # This allows us to pick all the interesting files # in the mongo and mongo-enterprise repos @@ -485,24 +472,48 @@ class Repo(object): return file_list + def get_candidate_files(self): + """Query git to get a list of all files in the repo to consider for analysis + """ + return self._git_ls_files(["ls-files", "--cached"]) + + def get_working_tree_candidate_files(self): + """Query git to get a list of all files in the working tree to consider for analysis + """ + return self._git_ls_files(["ls-files", "--cached", "--others"]) + + def get_working_tree_candidates(self): + """Get the set of candidate files to check by querying the repository + + Returns the full path to the file for clang-format to consume. + """ + valid_files = list(self.get_working_tree_candidate_files()) + + # Get the full file name here + valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files] -def expand_file_string(glob_pattern): - """Expand a string that represents a set of files + # Filter out files that git thinks exist but were removed. + valid_files = [f for f in valid_files if os.path.exists(f)] + + return valid_files + +def get_files_to_check_working_tree(): + """Get a list of files to check form the working tree. + This will pick up files not managed by git. """ - return [os.path.abspath(f) for f in globstar.iglob(glob_pattern)] + repos = get_repos() -def get_files_to_check(files): - """Filter the specified list of files to check down to the actual - list of files that need to be checked.""" - candidates = [] + valid_files = list(itertools.chain.from_iterable([r.get_working_tree_candidates() for r in repos])) - # Get a list of candidate_files - candidates = [expand_file_string(f) for f in files] - candidates = list(itertools.chain.from_iterable(candidates)) + return valid_files +def get_files_to_check(): + """Get a list of files that need to be checked + based on which files are managed by git. + """ repos = get_repos() - valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos])) + valid_files = list(itertools.chain.from_iterable([r.get_candidates(None) for r in repos])) return valid_files @@ -552,10 +563,19 @@ def lint_patch(clang_format, infile): if files: _lint_files(clang_format, files) -def lint(clang_format, glob): +def lint(clang_format): """Lint files command entry point """ - files = get_files_to_check(glob) + files = get_files_to_check() + + _lint_files(clang_format, files) + + return True + +def lint_all(clang_format): + """Lint files command entry point based on working tree + """ + files = get_files_to_check_working_tree() _lint_files(clang_format, files) @@ -572,18 +592,17 @@ def _format_files(clang_format, files): print("ERROR: failed to format files") sys.exit(1) -def format_func(clang_format, glob): +def format_func(clang_format): """Format files command entry point """ - files = get_files_to_check(glob) + files = get_files_to_check() _format_files(clang_format, files) def usage(): """Print usage """ - print("clang-format.py supports 3 commands [ lint, lint-patch, format ]. Run " - " -? for more information") + print("clang-format.py supports 4 commands [ lint, lint-all, lint-patch, format ].") def main(): """Main entry point @@ -597,11 +616,13 @@ def main(): command = args[1] if command == "lint": - lint(options.clang_format, args[2:]) + lint(options.clang_format) + elif command == "lint-all": + lint_all(options.clang_format) elif command == "lint-patch": lint_patch(options.clang_format, args[2:]) elif command == "format": - format_func(options.clang_format, args[2:]) + format_func(options.clang_format) else: usage() else: diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py index ab1661bcfea..907a8c27c94 100755 --- a/buildscripts/generate_compile_expansions.py +++ b/buildscripts/generate_compile_expansions.py @@ -52,8 +52,8 @@ if not version_parts: exit(1) if version_parts[0]: - print "suffix: latest" - print "src_suffix: latest" + print "suffix: v3.2-latest" + print "src_suffix: v3.2-latest" else: print "suffix: {0}".format(version_line) print "src_suffix: r{0}".format(version_line) diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index ea708cc0ecd..ca1baa00f39 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -24,6 +24,12 @@ import threading import time from distutils import spawn from optparse import OptionParser +_is_windows = (sys.platform == "win32") + +if _is_windows: + import win32event + import win32api + if sys.platform == "win32": import win32process @@ -365,7 +371,7 @@ def get_hang_analyzers(): elif sys.platform.startswith("sunos"): dbg = GDBDumper() ps = SolarisProcessList() - elif os.name == 'nt' or (os.name == "posix" and sys.platform == "cygwin"): + elif _is_windows or sys.platform == "cygwin": dbg = WindowsDumper() ps = WindowsProcessList() elif sys.platform == "darwin": @@ -383,10 +389,37 @@ def is_interesting_process(p): return False -def signal_process(pid): - """Signal python process with SIGUSR1, N/A on Windows""" +def signal_event_object(pid): + """Signal the Windows event object""" + + # Use unique event_name created. + event_name = "Global\\Mongo_Python_" + str(pid) + + try: + desired_access = win32event.EVENT_MODIFY_STATE + inherit_handle = False + task_timeout_handle = win32event.OpenEvent(desired_access, + inherit_handle, + event_name) + except win32event.error as err: + print "Exception from win32event.OpenEvent with error: " + str(err) + return + + try: + win32event.SetEvent(task_timeout_handle) + except win32event.error as err: + print "Exception from win32event.SetEvent with error: " + str(err) + finally: + win32api.CloseHandle(task_timeout_handle) + + print "Waiting for process to report" + time.sleep(5) + + +def signal_process(pid, signalnum): + """Signal process with signal, N/A on Windows""" try: - os.kill(pid, signal.SIGUSR1) + os.kill(pid, signalnum) print "Waiting for python process to report" time.sleep(5) @@ -417,8 +450,13 @@ def main(): print "OS: " + platform.platform() try: - distro = platform.linux_distribution() - print "Linux Distribution: " + str(distro) + if _is_windows or sys.platform == "cygwin": + distro = platform.win32_ver() + print "Windows Distribution: " + str(distro) + else: + distro = platform.linux_distribution() + print "Linux Distribution: " + str(distro) + except AttributeError: print "Cannot determine Linux distro since Python is too old" @@ -457,20 +495,29 @@ def main(): for process in processes_orig: sys.stdout.write("Ignoring process %d of %s\n" % (process[0], process[1])) else: - # Dump all other processes first since signaling the python script interrupts it + # Dump python processes by signalling them. The resmoke.py process will generate + # the report.json, when signalled, so we do this before attaching to other processes. + for process in [a for a in processes if a[1].startswith("python")]: + # On Windows, we set up an event object to wait on a signal. For Cygwin, we register + # a signal handler to wait for the signal since it supports POSIX signals. + if _is_windows: + sys.stdout.write("Calling SetEvent to signal python process %s with PID %d\n" % + (process[1], process[0])) + signal_event_object(process[0]) + else: + sys.stdout.write("Sending signal SIGUSR1 to python process %s with PID %d\n" % + (process[1], process[0])) + signal_process(process[0], signal.SIGUSR1) + + # Dump all other processes for process in [a for a in processes if not a[1].startswith("python")]: sys.stdout.write("Dumping process %d of %s\n" % (process[0], process[1])) dbg.dump_info(process[0], process[1], sys.stdout) - for process in [a for a in processes if a[1].startswith("python")]: - signal_process(process[0]) - - dbg.dump_info(process[0], process[1], sys.stdout) - # Suspend the timer so we can exit cleanly timer.cancel() - sys.stdout.write("Done analyzing processes for hangs\n") + sys.stdout.write("Done analyzing all processes for hangs\n") if __name__ == "__main__": main() diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml new file mode 100644 index 00000000000..e9b6989d221 --- /dev/null +++ b/buildscripts/package_test/.kitchen.yml @@ -0,0 +1,81 @@ +--- +driver: + name: ec2 + region: us-east-1 + subnet_id: <%= ENV['KITCHEN_SUBNET'] %> + security_group_ids: + - <%= ENV['KITCHEN_SECURITY_GROUP'] %> + aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> + interface: dns + associate_public_ip: true + +verifier: + name: inspec + sudo: true + +provisioner: + name: chef_solo + require_chef_omnibus: 12.6.0 + +platforms: + - name: amazon + driver: + image_id: ami-c481fad3 + transport: + username: ec2-user + - name: debian71 + driver: + image_id: ami-4b124a22 + transport: + username: admin + - name: debian81 + driver: + image_id: ami-896d85e2 + transport: + username: admin + - name: rhel62 + driver: + # Use rhel 6.3 because chef-solo fails on rhel 6.2 + image_id: ami-a35a33ca + transport: + username: root + - name: rhel70 + driver: + image_id: ami-60a1e808 + transport: + username: root + - name: suse11 + driver: + image_id: ami-7f2e6015 + transport: + username: ec2-user + - name: suse12 + driver: + image_id: ami-aeb532c6 + transport: + username: ec2-user + - name: ubuntu1204 + driver: + image_id: ami-3fec7956 + transport: + username: ubuntu + - name: ubuntu1404 + driver: + image_id: ami-1d8c9574 + transport: + username: ubuntu + - name: ubuntu1604 + driver: + image_id: ami-64140d0e + transport: + username: ubuntu + +transport: + ssh_key: ~/.ssh/kitchen.pem + +suites: + - name: service + run_list: + - recipe[package_test::install_mongodb] + attributes: + artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/metadata.rb b/buildscripts/package_test/metadata.rb new file mode 100644 index 00000000000..3bbfb147acc --- /dev/null +++ b/buildscripts/package_test/metadata.rb @@ -0,0 +1,2 @@ +name 'package_test' +version '0.1.0' diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb new file mode 100644 index 00000000000..627aa09b8af --- /dev/null +++ b/buildscripts/package_test/recipes/install_mongodb.rb @@ -0,0 +1,120 @@ +artifacts_tarball = 'artifacts.tgz' +user = node['current_user'] +homedir = node['etc']['passwd'][user]['dir'] + +ruby_block 'allow sudo over tty' do + block do + file = Chef::Util::FileEdit.new('/etc/sudoers') + file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults requiretty') + file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults !visiblepw') + file.write_file + end +end + +# These files limits processes. They therefore interferere with `ulimit -u` when present. +if platform_family? 'rhel' + file '/etc/security/limits.d/90-nproc.conf' do + action :delete + end + file '/etc/security/limits.d/20-nproc.conf' do + action :delete + end +end + +remote_file "#{homedir}/#{artifacts_tarball}" do + source node['artifacts_url'] +end + +execute 'extract artifacts' do + command "tar xzvf #{artifacts_tarball}" + cwd homedir +end + +if platform_family? 'debian' + execute 'apt-get update' do + command 'apt-get update' + end + + package 'openssl' + + # dpkg returns 1 if dependencies are not satisfied, which they will not be + # for enterprise builds. We install dependencies in the next block. + execute 'install mongod' do + command 'dpkg -i `find . -name "*server*.deb"`' + cwd homedir + returns [0, 1] + end + + # yum and zypper fetch dependencies automatically, but dpkg does not. + # Installing the dependencies explicitly is fragile, so we reply on apt-get + # to install dependencies after the fact. + execute 'install dependencies' do + command 'apt-get update && apt-get -y -f install' + end + + execute 'install mongo shell' do + command 'dpkg -i `find . -name "*shell*.deb"`' + cwd homedir + end +end + +if platform_family? 'rhel' + execute 'install mongod' do + command 'yum install -y `find . -name "*server*.rpm"`' + cwd homedir + end + + execute 'install mongo shell' do + command 'yum install -y `find . -name "*shell*.rpm"`' + cwd homedir + end +end + +if platform_family? 'suse' + bash 'wait for zypper lock to be released' do + code <<-EOD + retry_counter=0 + # We also need to make sure another instance of zypper isn't running while + # we do our install, so just run zypper refresh until it doesn't fail. + # Waiting for 2 minutes is copied from an internal project where we do this. + until [ "$retry_counter" -ge "12" ]; do + zypper refresh && exit 0 + retry_counter=$(($retry_counter + 1)) + [ "$retry_counter" = "12" ] && break + sleep 10 + done + exit 1 + EOD + end + + execute 'install mongod' do + command 'zypper -n install `find . -name "*server*.rpm"`' + cwd homedir + end + + execute 'install mongo' do + command 'zypper -n install `find . -name "*shell*.rpm"`' + cwd homedir + end +end + +inspec_wait = < 'unlimited', + 'Max cpu time' => 'unlimited', + 'Max address space' => 'unlimited', + 'Max open files' => '64000', + 'Max resident set' => 'unlimited', + 'Max processes' => '64000' +} +ulimits_cmd = 'cat /proc/$(pgrep mongod)/limits' + +ulimits.each do |limit, value| + describe command("#{ulimits_cmd} | grep \"#{limit}\"") do + its('stdout') { should match(/#{limit}\s+#{value}/) } + end +end + +############################################################ +# This section verifies reads and writes. +# - insert a document into the database +# - verify that findOne() returns a matching document +############################################################ + +describe command('mongo --eval "db.smoke.insert({answer: 42})"') do + its('exit_status') { should eq 0 } + its('stdout') { should match(/.+WriteResult\({ "nInserted" : 1 }\).+/m) } +end + +# read a document from the db +describe command('mongo --eval "db.smoke.findOne()"') do + its('exit_status') { should eq 0 } + its('stdout') { should match(/.+"answer" : 42.+/m) } +end + +############################################################ +# This section verifies uninstall. +############################################################ + +if rpm + describe command('rpm -e $(rpm -qa | grep "mongodb.*server" | awk \'{print $1}\')') do + its('exit_status') { should eq 0 } + end +elsif deb + describe command('dpkg -r $(dpkg -l | grep "mongodb.*server" | awk \'{print $2}\')') do + its('exit_status') { should eq 0 } + end +end + +# make sure we cleaned up +%w( + /lib/systemd/system/mongod.service + /usr/bin/mongod +).each do |filename| + describe file(filename) do + it { should_not exist } + end +end diff --git a/buildscripts/packager-enterprise.py b/buildscripts/packager-enterprise.py index f9437fd935f..e7ae856a9ad 100755 --- a/buildscripts/packager-enterprise.py +++ b/buildscripts/packager-enterprise.py @@ -26,20 +26,15 @@ # apt-get install dpkg-dev rpm debhelper fakeroot ia32-libs createrepo git-core libsnmp15 # echo "Now put the dist gnupg signing keys in ~root/.gnupg" -import argparse import errno -import getopt from glob import glob import packager import os import re import shutil -import stat -import subprocess import sys import tempfile import time -import urlparse # The MongoDB names for the architectures we support. ARCH_CHOICES=["x86_64", "ppc64le"] @@ -143,8 +138,6 @@ def main(argv): os.chdir(prefix) try: - # Download the binaries. - urlfmt="http://downloads.mongodb.com/linux/mongodb-linux-%s-enterprise-%s-%s.tgz" made_pkg = False # Build a package for each distro/spec/arch tuple, and @@ -154,12 +147,9 @@ def main(argv): for build_os in distro.build_os(arch): if build_os in args.distros or not args.distros: - if args.tarball: - filename = tarfile(build_os, arch, spec) - packager.ensure_dir(filename) - shutil.copyfile(args.tarball,filename) - else: - packager.httpget(urlfmt % (arch, build_os, spec.version()), packager.ensure_dir(tarfile(build_os, arch, spec))) + filename = tarfile(build_os, arch, spec) + packager.ensure_dir(filename) + shutil.copyfile(args.tarball,filename) repo = make_package(distro, build_os, arch, spec, srcdir) make_repo(repo, distro, build_os, spec) diff --git a/buildscripts/packager.py b/buildscripts/packager.py index 29fb4fdfa43..adae9fe5f6d 100755 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -28,18 +28,14 @@ import argparse import errno -import getopt -import httplib2 from glob import glob import os import re import shutil -import stat import subprocess import sys import tempfile import time -import urlparse # The MongoDB names for the architectures we support. ARCH_CHOICES=["x86_64"] @@ -54,11 +50,15 @@ class Spec(object): self.gitspec = gitspec self.rel = rel - # Nightly version numbers can be in the form: 3.0.7-pre-, or 3.0.7-5-g3b67ac + # Commit-triggerd version numbers can be in the form: 3.0.7-pre-, or 3.0.7-5-g3b67ac + # Patch builds version numbers are in the form: 3.5.5-64-g03945fa-patch-58debcdb3ff1223c9d00005b # def is_nightly(self): return bool(re.search("-$", self.version())) or bool(re.search("\d-\d+-g[0-9a-f]+$", self.version())) + def is_patch(self): + return bool(re.search("\d-\d+-g[0-9a-f]+-patch-[0-9a-f]+$", self.version())) + def is_rc(self): return bool(re.search("-rc\d+$", self.version())) @@ -68,6 +68,12 @@ class Spec(object): def version(self): return self.ver + def patch_id(self): + if self.is_patch(): + return re.sub(r'.*-([0-9a-f]+$)', r'\1', self.version()) + else: + return "none" + def metadata_gitspec(self): """Git revision to use for spec+control+init+manpage files. The default is the release tag for the version being packaged.""" @@ -85,35 +91,57 @@ class Spec(object): return "-org" if int(self.ver.split(".")[1])%2==0 else "-org-unstable" def prelease(self): - # "N" is either passed in on the command line, or "1" + # NOTE: This is only called for RPM packages, and only after + # pversion() below has been called. If you want to change this format + # and want DEB packages to match, make sure to update pversion() + # below # - # 1) Standard release - "N" - # 2) Nightly (snapshot) - "0.N.YYYYMMDDlatest" - # 3) RC's - "0.N.rcX" + # "N" is either passed in on the command line, or "1" if self.rel: - corenum = self.rel + corenum = self.rel else: - corenum = 1 - # RC's + corenum = 1 + + # Version suffix for RPM packages: + # 1) RC's - "0.N.rcX" + # 2) Nightly (snapshot) - "0.N.latest" + # 3) Patch builds - "0.N.patch." + # 4) Standard release - "N" if self.is_rc(): - return "0.%s.%s" % (corenum, re.sub('.*-','',self.version())) - # Nightlies + return "0.%s.%s" % (corenum, re.sub('.*-','',self.version())) elif self.is_nightly(): - return "0.%s.%s" % (corenum, time.strftime("%Y%m%d")) + return "0.%s.latest" % (corenum) + elif self.is_patch(): + return "0.%s.patch.%s" % (corenum, self.patch_id()) else: - return str(corenum) + return str(corenum) def pversion(self, distro): # Note: Debian packages have funny rules about dashes in # version numbers, and RPM simply forbids dashes. pversion # will be the package's version number (but we need to know # our upstream version too). + + # For RPM packages this just returns X.Y.X because of the + # aforementioned rules, and prelease (above) adds a suffix later, + # so detect this case early + if re.search("(suse|redhat|fedora|centos|amazon)", distro.name()): + return re.sub("-.*", "", self.version()) + + # For DEB packages, this code sets the full version. If you change + # this format and want RPM packages to match make sure you change + # prelease above as well if re.search("^(debian|ubuntu)", distro.name()): - return re.sub("-", "~", self.ver) - elif re.search("(suse|redhat|fedora|centos|amazon)", distro.name()): - return re.sub("-.*", "", self.ver) - else: - raise Exception("BUG: unsupported platform?") + if self.is_nightly(): + ver = re.sub("-.*", "-latest", self.ver) + elif self.is_patch(): + ver = re.sub("-.*", "", self.ver) + "-patch-" + self.patch_id() + else: + ver = self.ver + + return re.sub("-", "~", ver) + + raise Exception("BUG: unsupported platform?") def branch(self): """Return the major and minor portions of the specified version. @@ -275,7 +303,7 @@ def get_args(distros, arch_choices): parser.add_argument("-d", "--distros", help="Distros to build for", choices=distro_choices, required=False, default=[], action='append') parser.add_argument("-p", "--prefix", help="Directory to build into", required=False) parser.add_argument("-a", "--arches", help="Architecture to build", choices=arch_choices, default=[], required=False, action='append') - parser.add_argument("-t", "--tarball", help="Local tarball to package instead of downloading (only valid with one distro/arch combination)", required=False, type=lambda x: is_valid_file(parser, x)) + parser.add_argument("-t", "--tarball", help="Local tarball to package", required=True, type=lambda x: is_valid_file(parser, x)) args = parser.parse_args() @@ -304,9 +332,6 @@ def main(argv): os.chdir(prefix) try: - # Download the binaries. - urlfmt="http://downloads.mongodb.org/linux/mongodb-linux-%s-%s-%s.tgz" - # Build a package for each distro/spec/arch tuple, and # accumulate the repository-layout directories. for (distro, arch) in crossproduct(distros, args.arches): @@ -314,12 +339,9 @@ def main(argv): for build_os in distro.build_os(arch): if build_os in args.distros or not args.distros: - if args.tarball: - filename = tarfile(build_os, arch, spec) - ensure_dir(filename) - shutil.copyfile(args.tarball,filename) - else: - httpget(urlfmt % (arch, build_os, spec.version()), ensure_dir(tarfile(build_os, arch, spec))) + filename = tarfile(build_os, arch, spec) + ensure_dir(filename) + shutil.copyfile(args.tarball,filename) repo = make_package(distro, build_os, arch, spec, srcdir) make_repo(repo, distro, build_os, spec) @@ -371,27 +393,6 @@ def setupdir(distro, build_os, arch, spec): # or dst/x86_64/redhat/rhel55/mongodb-org-unstable/ return "dst/%s/%s/%s/%s%s-%s/" % (arch, distro.name(), build_os, distro.pkgbase(), spec.suffix(), spec.pversion(distro)) -def httpget(url, filename): - """Download the contents of url to filename, return filename.""" - print "Fetching %s to %s." % (url, filename) - conn = None - u=urlparse.urlparse(url) - assert(u.scheme=='http') - try: - h = httplib2.Http(cache = os.environ["HOME"] + "/.cache") - resp, content = h.request(url, "GET") - t=filename+'.TMP' - if resp.status==200: - with open(t, 'w') as f: - f.write(content) - else: - raise Exception("HTTP error %d" % resp.status) - os.rename(t, filename) - finally: - if conn: - conn.close() - return filename - def unpack_binaries_into(build_os, arch, spec, where): """Unpack the tarfile for (build_os, arch, spec) into directory where.""" rootdir=os.getcwd() diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index a6cb03cb620..837b0ec3b26 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -6,13 +6,10 @@ Command line utility for executing MongoDB tests of all kinds. from __future__ import absolute_import -import json import os.path import random -import signal import sys import time -import traceback # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: @@ -115,21 +112,6 @@ def _dump_suite_config(suite, logging_config): return "\n".join(sb) -def _write_report_file(suites, pathname): - """ - Writes the report.json file if requested. - """ - - reports = [] - for suite in suites: - for group in suite.test_groups: - reports.extend(group.get_reports()) - - combined_report_dict = resmokelib.testing.report.TestReport.combine(*reports).as_dict() - with open(pathname, "w") as fp: - json.dump(combined_report_dict, fp) - - def main(): start_time = time.time() @@ -151,6 +133,11 @@ def main(): interrupted = False suites = resmokelib.parser.get_suites(values, args) + + # Register a signal handler or Windows event object so we can write the report file if the task + # times out. + resmokelib.sighandler.register(resmoke_logger, suites) + try: for suite in suites: resmoke_logger.info(_dump_suite_config(suite, logging_config)) @@ -178,39 +165,8 @@ def main(): if not interrupted: resmokelib.logging.flush.stop_thread() - if resmokelib.config.REPORT_FILE is not None: - _write_report_file(suites, resmokelib.config.REPORT_FILE) + resmokelib.reportfile.write(suites) if __name__ == "__main__": - - def _dump_stacks(signum, frame): - """ - Signal handler that will dump the stacks of all threads. - """ - - header_msg = "Dumping stacks due to SIGUSR1 signal" - - sb = [] - sb.append("=" * len(header_msg)) - sb.append(header_msg) - sb.append("=" * len(header_msg)) - - frames = sys._current_frames() - sb.append("Total threads: %d" % (len(frames))) - sb.append("") - - for thread_id in frames: - stack = frames[thread_id] - sb.append("Thread %d:" % (thread_id)) - sb.append("".join(traceback.format_stack(stack))) - - sb.append("=" * len(header_msg)) - print "\n".join(sb) - - try: - signal.signal(signal.SIGUSR1, _dump_stacks) - except AttributeError: - print "Cannot catch signals on Windows" - main() diff --git a/buildscripts/resmokeconfig/suites/multiversion_auth.yml b/buildscripts/resmokeconfig/suites/multiversion_auth.yml new file mode 100644 index 00000000000..4db3f87f7af --- /dev/null +++ b/buildscripts/resmokeconfig/suites/multiversion_auth.yml @@ -0,0 +1,37 @@ +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +selector: + js_test: + roots: + - jstests/multiVersion/*.js + exclude_files: + # Needs extra work to support MONGODB-CR + - jstests/multiVersion/upgrade_cluster_v5_to_v6.js + # Uses threads which don't propagate jsTest() information + - jstests/multiVersion/mixed_storage_version_replication.js + # Uses ToolTest, which doesn't start servers with keyFile + - jstests/multiVersion/transitioning_to_and_from_WT.js + # TODO: SERVER-21578 + - jstests/multiVersion/balancer_multiVersion_detect.js + +# Multiversion tests start their own mongod's. +executor: + js_test: + config: + shell_options: + global_vars: + TestData: + auth: true + authMechanism: SCRAM-SHA-1 + keyFile: *keyFile + keyFileData: *keyFileData + authenticationDatabase: local + authenticationMechanism: SCRAM-SHA-1 + password: *keyFileData + username: __system + nodb: '' + readMode: legacy + writeMode: legacy diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index b4406cea871..2afbe6ae044 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -2,6 +2,9 @@ selector: js_test: roots: - jstests/noPassthrough/*.js + exclude_files: + # Disable inmem_full as per SERVER-27014 + - jstests/noPassthrough/inmem_full.js # noPassthrough tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml b/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml index 44b1b7a572d..11b160e4925 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml @@ -15,6 +15,14 @@ selector: - jstests/replsets/read_majority_two_arbs.js # The combination of new bridges and PV0 can lead to an improper spanning tree in sync2.js. - jstests/replsets/sync2.js + # PV0 does not persist the last vote + - jstests/replsets/last_vote.js + # PV0's w:majority guarantees aren't strong enough for this test to pass. + - jstests/replsets/write_concern_after_stepdown_and_stepup.js + # This test expects the server to log a PV1-only vote-not-granted reason + - jstests/replsets/no_flapping_during_network_partition.js + # This test requires terms. + - jstests/replsets/rollback_too_new.js executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index 6b51a5f17f5..8096b406f4d 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -12,15 +12,16 @@ selector: - jstests/core/check_shard_index.js # checkShardingIndex. - jstests/core/collection_truncate.js # emptycapped. - jstests/core/compact_keeps_indexes.js # compact. - - jstests/core/capped_update.js # uses godinsert and can't run under replication. + - jstests/core/currentop.js # uses fsync. - jstests/core/auth_copydb.js # copyDatabase. - jstests/core/copydb.js # copyDatabase. - jstests/core/dbadmin.js # "local" database. - jstests/core/dbhash.js # dbhash. - jstests/core/dbhash2.js # dbhash. + - jstests/core/diagdata.js # Command not supported in mongos - jstests/core/dropdb_race.js # syncdelay. - jstests/core/evalb.js # profiling. - - jstests/core/fsync.js # fsync. + - jstests/core/fsync.js # uses fsync. - jstests/core/geo_haystack*.js # geoSearch. - jstests/core/geo_s2cursorlimitskip.js # profiling. - jstests/core/geo_update_btree2.js # notablescan. @@ -68,6 +69,7 @@ selector: - jstests/core/eval0.js - jstests/core/eval7.js - jstests/core/eval9.js + - jstests/core/killop_drop_collection.js # Uses fsyncLock. executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharding_auth.yml b/buildscripts/resmokeconfig/suites/sharding_auth.yml index d99760820e7..1a988d5ab3b 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth.yml @@ -24,6 +24,7 @@ selector: - jstests/sharding/migration_with_source_deletes.js # SERVER-21713 - jstests/sharding/migration_sets_fromMigrate_flag.js # SERVER-21713 - jstests/sharding/donor_shard_abort_and_start_new_migration.js # SERVER-21713 + - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml index 61e50340b93..daa287b3f64 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml @@ -24,6 +24,7 @@ selector: - jstests/sharding/migration_with_source_deletes.js # SERVER-21713 - jstests/sharding/migration_sets_fromMigrate_flag.js # SERVER-21713 - jstests/sharding/donor_shard_abort_and_start_new_migration.js # SERVER-21713 + - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 43fbcaa94f6..0295f0e4f9d 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -88,6 +88,14 @@ selector: # for this command - jstests/sharding/major_version_check.js - jstests/sharding/replset_config/ssv_config_check.js + # Runs replSetGetStatus -- via awaitLastOpCommitted -- directly against the config server: + # retries aren't supported. + - jstests/sharding/coll_epoch_test1.js + - jstests/sharding/move_stale_mongos.js + - jstests/sharding/shard4.js + - jstests/sharding/shard5.js + - jstests/sharding/split_stale_mongos.js + - jstests/sharding/zero_shard_version.js # Already stop or blackholes the primary of the CSRS config shard - jstests/sharding/all_config_hosts_down.js - jstests/sharding/all_config_servers_blackholed_from_mongos.js diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 56a5b581ce8..98553af4366 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -14,12 +14,14 @@ selector: - jstests/core/check_shard_index.js # checkShardingIndex. - jstests/core/collection_truncate.js # emptycapped. - jstests/core/compact_keeps_indexes.js # compact. + - jstests/core/currentop.js # uses fsync. - jstests/core/dbadmin.js # "local" database. - jstests/core/dbhash.js # dbhash. - jstests/core/dbhash2.js # dbhash. + - jstests/core/diagdata.js # Command not supported in mongos - jstests/core/dropdb_race.js # syncdelay. - jstests/core/evalb.js # profiling. - - jstests/core/fsync.js # fsync. + - jstests/core/fsync.js # uses fsync. - jstests/core/geo_haystack*.js # geoSearch. - jstests/core/geo_s2cursorlimitskip.js # profiling. - jstests/core/geo_update_btree2.js # notablescan. @@ -41,6 +43,7 @@ selector: # TODO: SERVER-18292 remove once OP_COMMAND is implemented in mongos. - jstests/core/invalid_db_name.js - jstests/core/validate_cmd_ns.js + - jstests/core/killop_drop_collection.js # Uses fsyncLock. executor: js_test: diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py index 06b0539e25b..d8019dcf404 100644 --- a/buildscripts/resmokelib/__init__.py +++ b/buildscripts/resmokelib/__init__.py @@ -3,5 +3,7 @@ from __future__ import absolute_import from . import errors from . import logging from . import parser +from . import reportfile +from . import sighandler from . import testing from . import utils diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index c5f5d40401b..f058372df8c 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -184,7 +184,7 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): raise NotImplementedError("_append_logs must be implemented by _BaseBuildloggerHandler" " subclasses") - def flush_with_lock(self, close_called): + def _flush_buffer_with_lock(self, buf, close_called): """ Ensures all logging output has been flushed to the buildlogger server. @@ -194,7 +194,7 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): called. """ - self.retry_buffer.extend(self.buffer) + self.retry_buffer.extend(buf) if self._append_logs(self.retry_buffer): self.retry_buffer = [] @@ -209,8 +209,6 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): loggers._BUILDLOGGER_FALLBACK.info(message) self.retry_buffer = [] - self.buffer = [] - class BuildloggerTestHandler(_BaseBuildloggerHandler): """ diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 6ede7d38b3c..3d71399bfa5 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -43,10 +43,12 @@ class BufferedHandler(logging.Handler): self.capacity = capacity self.interval_secs = interval_secs - self.buffer = [] - self._lock = threading.Lock() - self._timer = None # Defer creation until actually begin to log messages. + self.__emit_lock = threading.Lock() # Prohibits concurrent access to 'self.__emit_buffer'. + self.__emit_buffer = [] + + self.__flush_lock = threading.Lock() # Serializes callers of self.flush(). + self.__timer = None # Defer creation until we actually begin to log messages. def _new_timer(self): """ @@ -54,7 +56,7 @@ class BufferedHandler(logging.Handler): flush() method after 'interval_secs' seconds. """ - return timer.AlarmClock(self.interval_secs, self.flush, args=[self]) + return timer.AlarmClock(self.interval_secs, self.flush) def process_record(self, record): """ @@ -79,37 +81,45 @@ class BufferedHandler(logging.Handler): will expire after another 'interval_secs' seconds. """ - with self._lock: - self.buffer.append(self.process_record(record)) - if len(self.buffer) >= self.capacity: - if self._timer is not None: - self._timer.snooze() - self.flush_with_lock(False) - if self._timer is not None: - self._timer.reset() + if self.__timer is None: + self.__timer = self._new_timer() + self.__timer.start() - if self._timer is None: - self._timer = self._new_timer() - self._timer.start() + with self.__emit_lock: + self.__emit_buffer.append(self.process_record(record)) + if len(self.__emit_buffer) >= self.capacity: + # Trigger the timer thread to cause it to flush the buffer early. + self.__timer.trigger() - def flush(self, close_called=False): + def flush(self): """ Ensures all logging output has been flushed. """ - with self._lock: - if self.buffer: - self.flush_with_lock(close_called) + self.__flush(close_called=False) - def flush_with_lock(self, close_called): + def __flush(self, close_called): """ Ensures all logging output has been flushed. + """ + + with self.__emit_lock: + buf = self.__emit_buffer + self.__emit_buffer = [] + + # The buffer 'buf' is flushed without holding 'self.__emit_lock' to avoid causing callers of + # self.emit() to block behind the completion of a potentially long-running flush operation. + if buf: + with self.__flush_lock: + self._flush_buffer_with_lock(buf, close_called) - This version resets the buffers back to an empty list and is - intended to be overridden by subclasses. + def _flush_buffer_with_lock(self, buf, close_called): + """ + Ensures all logging output has been flushed. """ - self.buffer = [] + raise NotImplementedError("_flush_buffer_with_lock must be implemented by BufferedHandler" + " subclasses") def close(self): """ @@ -118,9 +128,10 @@ class BufferedHandler(logging.Handler): Stops the timer and flushes the buffer. """ - if self._timer is not None: - self._timer.dismiss() - self.flush(close_called=True) + if self.__timer is not None: + self.__timer.dismiss() + + self.__flush(close_called=True) logging.Handler.close(self) diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py new file mode 100644 index 00000000000..7bf288069e2 --- /dev/null +++ b/buildscripts/resmokelib/reportfile.py @@ -0,0 +1,29 @@ +""" +Manages interactions with the report.json file. +""" + +from __future__ import absolute_import + +import json + +from . import config +from .testing import report as _report + + +def write(suites): + """ + Writes the combined report of all executions if --reportFile was + specified on the command line. + """ + + if config.REPORT_FILE is None: + return + + reports = [] + for suite in suites: + for group in suite.test_groups: + reports.extend(group.get_reports()) + + combined_report_dict = _report.TestReport.combine(*reports).as_dict() + with open(config.REPORT_FILE, "w") as fp: + json.dump(combined_report_dict, fp) diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py new file mode 100644 index 00000000000..cbce75ca0f7 --- /dev/null +++ b/buildscripts/resmokelib/sighandler.py @@ -0,0 +1,110 @@ +""" +Utility to support asynchronously signaling the current process. +""" + +from __future__ import absolute_import + +import atexit +import os +import signal +import sys +import threading +import traceback + +_is_windows = (sys.platform == "win32") +if _is_windows: + import win32api + import win32event + +from . import reportfile + + +def register(logger, suites): + """ + On Windows, set up an event object to wait for signal, otherwise, register a signal handler + for the SIGUSR1 signal. + """ + + def _handle_sigusr1(signum, frame): + """ + Signal handler that will dump the stacks of all threads and + then write out the report file. + """ + + header_msg = "Dumping stacks due to SIGUSR1 signal" + + _dump_stacks(logger, header_msg) + reportfile.write(suites) + + def _handle_set_event(event_handle): + """ + Windows event object handler that will dump the stacks of all threads and then write out + the report file. + """ + + while True: + try: + # Wait for task time out to dump stacks. + ret = win32event.WaitForSingleObject(event_handle, win32event.INFINITE) + if ret != win32event.WAIT_OBJECT_0: + logger.error("_handle_set_event WaitForSingleObject failed: %d" % ret) + return + except win32event.error as err: + logger.error("Exception from win32event.WaitForSingleObject with error: %s" % err) + else: + header_msg = "Dumping stacks due to signal from win32event.SetEvent" + + _dump_stacks(logger, header_msg) + reportfile.write(suites) + + + # On Windows spawn a thread to wait on an event object for signal to dump stacks. For Cygwin + # platforms, we use a signal handler since it supports POSIX signals. + if _is_windows: + # Create unique event_name. + event_name = "Global\\Mongo_Python_" + str(os.getpid()) + + try: + security_attributes = None + manual_reset = False + initial_state = False + task_timeout_handle = win32event.CreateEvent(security_attributes, + manual_reset, + initial_state, + event_name) + except win32event.error as err: + logger.error("Exception from win32event.CreateEvent with error: %s" % err) + return + + # Register to close event object handle on exit. + atexit.register(win32api.CloseHandle, task_timeout_handle) + + # Create thread. + event_handler_thread = threading.Thread(target=_handle_set_event, + kwargs={"event_handle": task_timeout_handle}, + name="windows_event_handler_thread") + event_handler_thread.daemon = True + event_handler_thread.start() + else: + # Otherwise register a signal handler + signal.signal(signal.SIGUSR1, _handle_sigusr1) + + +def _dump_stacks(logger, header_msg): + """ + Signal handler that will dump the stacks of all threads. + """ + + sb = [] + sb.append(header_msg) + + frames = sys._current_frames() + sb.append("Total threads: %d" % (len(frames))) + sb.append("") + + for thread_id in frames: + stack = frames[thread_id] + sb.append("Thread %d:" % (thread_id)) + sb.append("".join(traceback.format_stack(stack))) + + logger.info("\n".join(sb)) diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 5d79abd6ac6..40759539305 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -78,7 +78,10 @@ class TestGroupExecutor(object): num_repeats = _config.REPEAT while num_repeats > 0: test_queue = self._make_test_queue() - self._test_group.record_start() + + partial_reports = [job.report for job in self._jobs] + self._test_group.record_start(partial_reports) + (report, interrupted) = self._run_tests(test_queue) self._test_group.record_end(report) diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index c1d6521d5e9..3ac4b870678 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -104,23 +104,25 @@ class ReplicaSetFixture(interface.ReplFixture): # Wait for the primary to be elected. client = utils.new_mongo_client(port=self.port) while True: + self.logger.info("Waiting for primary on port %d to be elected.", self.port) is_master = client.admin.command("isMaster")["ismaster"] if is_master: break - self.logger.info("Waiting for primary on port %d to be elected.", self.port) time.sleep(0.1) # Wait a little bit before trying again. + self.logger.info("Primary on port %d successfully elected.", self.port) # Wait for the secondaries to become available. for secondary in self.get_secondaries(): client = utils.new_mongo_client(port=secondary.port, read_preference=pymongo.ReadPreference.SECONDARY) while True: + self.logger.info("Waiting for secondary on port %d to become available.", + secondary.port) is_secondary = client.admin.command("isMaster")["secondary"] if is_secondary: break - self.logger.info("Waiting for secondary on port %d to become available.", - secondary.port) time.sleep(0.1) # Wait a little bit before trying again. + self.logger.info("Secondary on port %d is now available.", secondary.port) def teardown(self): running_at_start = self.is_running() diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index e6523449636..282319a235c 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -301,9 +301,10 @@ class _MongoSFixture(interface.Fixture): # be established. while True: # Check whether the mongos exited for some reason. - if self.mongos.poll() is not None: + exit_code = self.mongos.poll() + if exit_code is not None: raise errors.ServerFailure("Could not connect to mongos on port %d, process ended" - " unexpectedly." % (self.port)) + " unexpectedly with code %d." % (self.port, exit_code)) try: # Use a shorter connection timeout to more closely satisfy the requested deadline. diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 87e661e2fab..edb38177fa2 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -127,9 +127,10 @@ class MongoDFixture(interface.Fixture): # be established. while True: # Check whether the mongod exited for some reason. - if self.mongod.poll() is not None: + exit_code = self.mongod.poll() + if exit_code is not None: raise errors.ServerFailure("Could not connect to mongod on port %d, process ended" - " unexpectedly." % (self.port)) + " unexpectedly with code %d." % (self.port, exit_code)) try: # Use a shorter connection timeout to more closely satisfy the requested deadline. diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 4ad35542982..d1856551fd1 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -113,14 +113,14 @@ class Job(object): raise except errors.ServerFailure: - self.logger.error("%s marked as a failure by a hook's before_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's before_test.", + test.shortDescription()) self._fail_test(test, sys.exc_info(), return_code=2) raise errors.StopExecution("A hook's before_test failed") except errors.TestFailure: - self.logger.error("%s marked as a failure by a hook's before_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's before_test.", + test.shortDescription()) self._fail_test(test, sys.exc_info(), return_code=1) if config.FAIL_FAST: raise errors.StopExecution("A hook's before_test failed") @@ -147,14 +147,14 @@ class Job(object): raise except errors.ServerFailure: - self.logger.error("%s marked as a failure by a hook's after_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's after_test.", + test.shortDescription()) self.report.setFailure(test, return_code=2) raise errors.StopExecution("A hook's after_test failed") except errors.TestFailure: - self.logger.error("%s marked as a failure by a hook's after_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's after_test.", + test.shortDescription()) self.report.setFailure(test, return_code=1) if config.FAIL_FAST: raise errors.StopExecution("A hook's after_test failed") diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index a5dc8744ddb..703076bff56 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -6,6 +6,7 @@ and timing information for the report.json file. from __future__ import absolute_import import copy +import threading import time import unittest @@ -30,6 +31,8 @@ class TestReport(unittest.TestResult): self.build_id = build_id self.build_config = build_config + self._lock = threading.Lock() + self.reset() @classmethod @@ -50,33 +53,35 @@ class TestReport(unittest.TestResult): if not isinstance(report, TestReport): raise TypeError("reports must be a list of TestReport instances") - for test_info in report.test_infos: - # If the user triggers a KeyboardInterrupt exception while a test is running, then - # it is possible for 'test_info' to be modified by a job thread later on. We make a - # shallow copy in order to ensure 'num_failed' is consistent with the actual number - # of tests that have status equal to "failed". - test_info = copy.copy(test_info) + with report._lock: + for test_info in report.test_infos: + # If the user triggers a KeyboardInterrupt exception while a test is running, + # then it is possible for 'test_info' to be modified by a job thread later on. + # We make a shallow copy in order to ensure 'num_interrupted' is consistent with + # the actual number of tests that have status equal to "timeout". + test_info = copy.copy(test_info) - # TestReport.addXX() may not have been called. - if test_info.status is None or test_info.return_code is None: - # Mark the test as having failed if it was interrupted. It might have passed if - # the suite ran to completion, but we wouldn't know for sure. - test_info.status = "fail" - test_info.return_code = -2 + # TestReport.addXX() may not have been called. + if test_info.status is None or test_info.return_code is None: + # Mark the test as having timed out if it was interrupted. It might have + # passed if the suite ran to completion, but we wouldn't know for sure. + test_info.status = "timeout" + test_info.return_code = -2 - # TestReport.stopTest() may not have been called. - if test_info.end_time is None: - # Use the current time as the time that the test finished running. - test_info.end_time = combining_time + # TestReport.stopTest() may not have been called. + if test_info.end_time is None: + # Use the current time as the time that the test finished running. + test_info.end_time = combining_time - combined_report.test_infos.append(test_info) + combined_report.test_infos.append(test_info) - combined_report.num_dynamic += report.num_dynamic + combined_report.num_dynamic += report.num_dynamic # Recompute number of success, failures, and errors. combined_report.num_succeeded = len(combined_report.get_successful()) combined_report.num_failed = len(combined_report.get_failed()) combined_report.num_errored = len(combined_report.get_errored()) + combined_report.num_interrupted = len(combined_report.get_interrupted()) return combined_report @@ -89,12 +94,10 @@ class TestReport(unittest.TestResult): test_info = _TestInfo(test.id(), dynamic) test_info.start_time = time.time() - self.test_infos.append(test_info) basename = test.basename() if dynamic: command = "(dynamic test case)" - self.num_dynamic += 1 else: command = test.as_command() self.logger.info("Running %s...\n%s", basename, command) @@ -117,6 +120,11 @@ class TestReport(unittest.TestResult): test.shortDescription(), test_info.url_endpoint) + with self._lock: + self.test_infos.append(test_info) + if dynamic: + self.num_dynamic += 1 + # Set up the test-specific logger. logger_name = "%s:%s" % (test.logger.name, test.short_name()) logger = logging.loggers.new_logger(logger_name, parent=test.logger) @@ -126,6 +134,8 @@ class TestReport(unittest.TestResult): build_config=self.build_config, test_id=test_id) + # TestReport.combine() doesn't access the '__original_loggers' attribute, so we don't bother + # protecting it with the lock. self.__original_loggers[test_info.test_id] = test.logger test.logger = logger @@ -136,8 +146,9 @@ class TestReport(unittest.TestResult): unittest.TestResult.stopTest(self, test) - test_info = self._find_test_info(test) - test_info.end_time = time.time() + with self._lock: + test_info = self._find_test_info(test) + test_info.end_time = time.time() time_taken = test_info.end_time - test_info.start_time self.logger.info("%s ran in %0.2f seconds.", test.basename(), time_taken) @@ -147,6 +158,9 @@ class TestReport(unittest.TestResult): logging.flush.close_later(test.logger) # Restore the original logger for the test. + # + # TestReport.combine() doesn't access the '__original_loggers' attribute, so we don't bother + # protecting it with the lock. test.logger = self.__original_loggers.pop(test.id()) def addError(self, test, err): @@ -156,28 +170,32 @@ class TestReport(unittest.TestResult): """ unittest.TestResult.addError(self, test, err) - self.num_errored += 1 - test_info = self._find_test_info(test) - test_info.status = "error" - test_info.return_code = test.return_code + with self._lock: + self.num_errored += 1 + + test_info = self._find_test_info(test) + test_info.status = "error" + test_info.return_code = test.return_code def setError(self, test): """ Used to change the outcome of an existing test to an error. """ - test_info = self._find_test_info(test) - if test_info.end_time is None: - raise ValueError("stopTest was not called on %s" % (test.basename())) + with self._lock: + test_info = self._find_test_info(test) + if test_info.end_time is None: + raise ValueError("stopTest was not called on %s" % (test.basename())) - test_info.status = "error" - test_info.return_code = 2 + test_info.status = "error" + test_info.return_code = 2 # Recompute number of success, failures, and errors. self.num_succeeded = len(self.get_successful()) self.num_failed = len(self.get_failed()) self.num_errored = len(self.get_errored()) + self.num_interrupted = len(self.get_interrupted()) def addFailure(self, test, err): """ @@ -186,28 +204,32 @@ class TestReport(unittest.TestResult): """ unittest.TestResult.addFailure(self, test, err) - self.num_failed += 1 - test_info = self._find_test_info(test) - test_info.status = "fail" - test_info.return_code = test.return_code + with self._lock: + self.num_failed += 1 + + test_info = self._find_test_info(test) + test_info.status = "fail" + test_info.return_code = test.return_code def setFailure(self, test, return_code=1): """ Used to change the outcome of an existing test to a failure. """ - test_info = self._find_test_info(test) - if test_info.end_time is None: - raise ValueError("stopTest was not called on %s" % (test.basename())) + with self._lock: + test_info = self._find_test_info(test) + if test_info.end_time is None: + raise ValueError("stopTest was not called on %s" % (test.basename())) - test_info.status = "error" - test_info.return_code = return_code + test_info.status = "fail" + test_info.return_code = return_code # Recompute number of success, failures, and errors. self.num_succeeded = len(self.get_successful()) self.num_failed = len(self.get_failed()) self.num_errored = len(self.get_errored()) + self.num_interrupted = len(self.get_interrupted()) def addSuccess(self, test): """ @@ -215,38 +237,57 @@ class TestReport(unittest.TestResult): """ unittest.TestResult.addSuccess(self, test) - self.num_succeeded += 1 - test_info = self._find_test_info(test) - test_info.status = "pass" - test_info.return_code = test.return_code + with self._lock: + self.num_succeeded += 1 + + test_info = self._find_test_info(test) + test_info.status = "pass" + test_info.return_code = test.return_code def wasSuccessful(self): """ Returns true if all tests executed successfully. """ - return self.num_failed == self.num_errored == 0 + + with self._lock: + return self.num_failed == self.num_errored == self.num_interrupted == 0 def get_successful(self): """ Returns the status and timing information of the tests that executed successfully. """ - return [test_info for test_info in self.test_infos if test_info.status == "pass"] + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "pass"] def get_failed(self): """ Returns the status and timing information of the tests that raised a failureException during their execution. """ - return [test_info for test_info in self.test_infos if test_info.status == "fail"] + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "fail"] def get_errored(self): """ Returns the status and timing information of the tests that raised a non-failureException during their execution. """ - return [test_info for test_info in self.test_infos if test_info.status == "error"] + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "error"] + + def get_interrupted(self): + """ + Returns the status and timing information of the tests that had + their execution interrupted. + """ + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "timeout"] def as_dict(self): """ @@ -256,42 +297,56 @@ class TestReport(unittest.TestResult): """ results = [] - for test_info in self.test_infos: - # Don't distinguish between failures and errors. - status = "pass" if test_info.status == "pass" else "fail" - - result = { - "test_file": test_info.test_id, - "status": status, - "exit_code": test_info.return_code, - "start": test_info.start_time, - "end": test_info.end_time, - "elapsed": test_info.end_time - test_info.start_time, + with self._lock: + for test_info in self.test_infos: + status = test_info.status + if status == "error": + # Don't distinguish between failures and errors. + status = "fail" + elif status == "timeout": + # Until EVG-1536 is completed, we shouldn't distinguish between failures and + # interrupted tests in the report.json file. In Evergreen, the behavior to sort + # tests with the "timeout" test status after tests with the "pass" test status + # effectively hides interrupted tests from the test results sidebar unless + # sorting by the time taken. + status = "fail" + + result = { + "test_file": test_info.test_id, + "status": status, + "exit_code": test_info.return_code, + "start": test_info.start_time, + "end": test_info.end_time, + "elapsed": test_info.end_time - test_info.start_time, + } + + if test_info.url_endpoint is not None: + result["url"] = test_info.url_endpoint + result["url_raw"] = test_info.url_endpoint + "?raw=1" + + results.append(result) + + return { + "results": results, + "failures": self.num_failed + self.num_errored + self.num_interrupted, } - if test_info.url_endpoint is not None: - result["url"] = test_info.url_endpoint - result["url_raw"] = test_info.url_endpoint + "?raw=1" - - results.append(result) - - return { - "results": results, - "failures": self.num_failed + self.num_errored, - } - def reset(self): """ Resets the test report back to its initial state. """ - self.test_infos = [] + with self._lock: + self.test_infos = [] - self.num_dynamic = 0 - self.num_succeeded = 0 - self.num_failed = 0 - self.num_errored = 0 + self.num_dynamic = 0 + self.num_succeeded = 0 + self.num_failed = 0 + self.num_errored = 0 + self.num_interrupted = 0 + # TestReport.combine() doesn't access the '__original_loggers' attribute, so we don't bother + # protecting it with the lock. self.__original_loggers = {} def _find_test_info(self, test): diff --git a/buildscripts/resmokelib/testing/testgroup.py b/buildscripts/resmokelib/testing/testgroup.py index 688d56c296d..866cc85a4c0 100644 --- a/buildscripts/resmokelib/testing/testgroup.py +++ b/buildscripts/resmokelib/testing/testgroup.py @@ -5,8 +5,10 @@ about when and how they execute. from __future__ import absolute_import +import itertools import time +from . import report as _report from . import summary as _summary @@ -30,17 +32,30 @@ class TestGroup(object): self._end_times = [] self._reports = [] + # We keep a reference to the TestReports from the currently running jobs so that we can + # report intermediate results. + self._partial_reports = None + def get_reports(self): """ - Returns the list of reports. + Returns the list of reports. If there's an execution currently + in progress, then a report for the partial results is included + in the returned list. """ + + if self._partial_reports is not None: + active_report = _report.TestReport.combine(*self._partial_reports) + return self._reports + [active_report] + return self._reports - def record_start(self): + def record_start(self, partial_reports): """ - Records the start time of an execution. + Records the start time of an execution and stores the + TestReports for currently running jobs. """ self._start_times.append(time.time()) + self._partial_reports = partial_reports def record_end(self, report): """ @@ -48,13 +63,24 @@ class TestGroup(object): """ self._end_times.append(time.time()) self._reports.append(report) + self._partial_reports = None def summarize_latest(self, sb): """ Returns a summary of the latest execution of the group and appends a summary of that execution onto the string builder 'sb'. + + If there's an execution currently in progress, then the partial + summary of that execution is appended to 'sb'. """ - return self._summarize_execution(-1, sb) + + if self._partial_reports is None: + return self._summarize_execution(-1, sb) + + active_report = _report.TestReport.combine(*self._partial_reports) + # Use the current time as the time that the test group finished running. + end_time = time.time() + return self._summarize_report(active_report, self._start_times[-1], end_time, sb) def summarize(self, sb): """ @@ -103,10 +129,26 @@ class TestGroup(object): string builder 'sb'. """ - report = self._reports[iteration] - time_taken = self._end_times[iteration] - self._start_times[iteration] + return self._summarize_report(self._reports[iteration], + self._start_times[iteration], + self._end_times[iteration], + sb) + + def _summarize_report(self, report, start_time, end_time, sb): + """ + Returns the summary information of the execution given by + 'report' that started at 'start_time' and finished at + 'end_time', and appends a summary of that execution onto the + string builder 'sb'. + """ + + time_taken = end_time - start_time - num_run = report.num_succeeded + report.num_errored + report.num_failed + # Tests that were interrupted are treated as failures because (1) the test has already been + # started and therefore isn't skipped and (2) the test has yet to finish and therefore + # cannot be said to have succeeded. + num_failed = report.num_failed + report.num_interrupted + num_run = report.num_succeeded + report.num_errored + num_failed num_skipped = len(self.tests) + report.num_dynamic - num_run if report.num_succeeded == num_run and num_skipped == 0: @@ -114,14 +156,14 @@ class TestGroup(object): return _summary.Summary(num_run, time_taken, num_run, 0, 0, 0) summary = _summary.Summary(num_run, time_taken, report.num_succeeded, num_skipped, - report.num_failed, report.num_errored) + num_failed, report.num_errored) sb.append("%d test(s) ran in %0.2f seconds" " (%d succeeded, %d were skipped, %d failed, %d errored)" % summary) - if report.num_failed > 0: + if num_failed > 0: sb.append("The following tests failed (with exit code):") - for test_info in report.get_failed(): + for test_info in itertools.chain(report.get_failed(), report.get_interrupted()): sb.append(" %s (%d)" % (test_info.test_id, test_info.return_code)) if report.num_errored > 0: diff --git a/buildscripts/resmokelib/utils/timer.py b/buildscripts/resmokelib/utils/timer.py index 80531d5db5c..a32ed99b670 100644 --- a/buildscripts/resmokelib/utils/timer.py +++ b/buildscripts/resmokelib/utils/timer.py @@ -1,10 +1,10 @@ """ Alternative to the threading.Timer class. -Enables a timer to be restarted without needing to construct a new thread -each time. This is necessary to execute periodic actions, e.g. flushing -log messages to buildlogger, while avoiding errors related to "can't start -new thread" that would otherwise occur on Windows. +Enables a timer to be restarted without needing to construct a new +thread each time. This is necessary to execute periodic actions, e.g. +flushing log messages to buildlogger, while avoiding errors related to +"can't start new thread" that would otherwise occur on Windows. """ from __future__ import absolute_import @@ -32,8 +32,8 @@ class AlarmClock(threading.Thread): self.args = args if args is not None else [] self.kwargs = kwargs if kwargs is not None else {} - self.lock = threading.Lock() - self.cond = threading.Condition(self.lock) + self.__lock = threading.Lock() + self.__cond = threading.Condition(self.__lock) self.snoozed = False # canceled for one execution self.dismissed = False # canceled for all time @@ -44,9 +44,9 @@ class AlarmClock(threading.Thread): Disables the timer. """ - with self.lock: + with self.__lock: self.dismissed = True - self.cond.notify_all() + self.__cond.notify_all() self.join() # Tidy up the started thread. @@ -54,24 +54,25 @@ class AlarmClock(threading.Thread): def snooze(self): """ - Skips the next execution of 'func' if it has not already started. + Skips the next execution of 'func' if it has not already + started. """ - with self.lock: + with self.__lock: if self.dismissed: raise ValueError("Timer cannot be snoozed if it has been dismissed") self.snoozed = True self.restarted = False - self.cond.notify_all() + self.__cond.notify_all() def reset(self): """ - Restarts the timer, causing it to wait 'interval' seconds before calling - 'func' again. + Restarts the timer, causing it to wait 'interval' seconds before + calling 'func' again. """ - with self.lock: + with self.__lock: if self.dismissed: raise ValueError("Timer cannot be reset if it has been dismissed") @@ -79,26 +80,43 @@ class AlarmClock(threading.Thread): raise ValueError("Timer cannot be reset if it has not been snoozed") self.restarted = True - self.cond.notify_all() + self.__cond.notify_all() + + def trigger(self): + """ + Signals the timer, causing 'func' to execute sooner than waiting + for 'interval' seconds to pass. + """ + + with self.__lock: + if self.dismissed: + raise ValueError("Timer cannot be triggered if it has been dismissed") + + if self.snoozed: + raise ValueError("Timer cannot be triggered if it has been snoozed") + + self.__cond.notify_all() def run(self): """ - Repeatedly calls 'func' with a delay of 'interval' seconds between executions. + Repeatedly calls 'func' with a delay of 'interval' seconds + between executions. - If the timer is snoozed before 'func' is called, then it waits to be reset. - After it has been reset, the timer will again wait 'interval' seconds and - then try to call 'func'. + If the timer is snoozed before 'func' is called, then it waits + to be reset. After it has been reset, the timer will again wait + 'interval' seconds and then try to call 'func'. - If the timer is dismissed, then no subsequent executions of 'func' are made. + If the timer is dismissed, then no subsequent executions of + 'func' are made. """ while True: - with self.lock: + with self.__lock: if self.dismissed: return # Wait for the specified amount of time. - self.cond.wait(self.interval) + self.__cond.wait(self.interval) if self.dismissed: return @@ -106,7 +124,7 @@ class AlarmClock(threading.Thread): # If the timer was snoozed, then it should wait to be reset. if self.snoozed: while not self.restarted: - self.cond.wait() + self.__cond.wait() if self.dismissed: return @@ -120,6 +138,6 @@ class AlarmClock(threading.Thread): self.func(*self.args, **self.kwargs) # Reacquire the lock. - with self.lock: + with self.__lock: # Ignore snoozes that took place while the function was being executed. self.snoozed = False diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py index 93fa2b38ad3..38675139ab9 100755 --- a/buildscripts/smoke.py +++ b/buildscripts/smoke.py @@ -36,6 +36,7 @@ from datetime import datetime from itertools import izip import glob +import logging from optparse import OptionParser import os import pprint @@ -75,6 +76,12 @@ except: except: json = None +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) + +from buildscripts.resmokelib.core import pipe + # TODO clean this up so we don't need globals... mongo_repo = os.getcwd() #'./' @@ -167,6 +174,10 @@ class mongod(NullMongod): self.proc = None self.auth = False + self.job_object = None + self._inner_proc_pid = None + self._stdout_pipe = None + def ensure_test_dirs(self): utils.ensureDir(smoke_db_prefix + "/tmp/unittest/") utils.ensureDir(smoke_db_prefix + "/data/") @@ -262,6 +273,28 @@ class mongod(NullMongod): print "running " + " ".join(argv) self.proc = self._start(buildlogger(argv, is_global=True)) + # If the mongod process is spawned under buildlogger.py, then the first line of output + # should include the pid of the underlying mongod process. If smoke.py didn't create its own + # job object because it is already inside one, then the pid is used to attempt to terminate + # the underlying mongod process. + first_line = self.proc.stdout.readline() + match = re.search("^\[buildlogger.py\] pid: (?P[0-9]+)$", first_line.rstrip()) + if match is not None: + self._inner_proc_pid = int(match.group("pid")) + else: + # The first line of output didn't include the pid of the underlying mongod process. We + # write the first line of output to smoke.py's stdout to ensure the message doesn't get + # lost since it's possible that buildlogger.py isn't being used. + sys.stdout.write(first_line) + + logger = logging.Logger("", level=logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter(fmt="%(message)s")) + logger.addHandler(handler) + + self._stdout_pipe = pipe.LoggerPipe(logger, logging.INFO, self.proc.stdout) + self._stdout_pipe.wait_until_started() + if not self.did_mongod_start(self.port): raise Exception("Failed to start mongod") @@ -275,12 +308,15 @@ class mongod(NullMongod): synced = synced and "syncedTo" in source and source["syncedTo"] def _start(self, argv): - """In most cases, just call subprocess.Popen(). On windows, - add the started process to a new Job Object, so that any - child processes of this process can be killed with a single - call to TerminateJobObject (see self.stop()). + """In most cases, just call subprocess.Popen(). On Windows, this + method also assigns the started process to a job object if a new + one was created. This ensures that any child processes of this + process can be killed with a single call to TerminateJobObject + (see self.stop()). """ + creation_flags = 0 + if os.sys.platform == "win32": # Create a job object with the "kill on job close" # flag; this is inherited by child processes (ie @@ -288,28 +324,30 @@ class mongod(NullMongod): # and lets us terminate the whole tree of processes # rather than orphaning the mongod. import win32job + import win32process - # Magic number needed to allow job reassignment in Windows 7 - # see: MSDN - Process Creation Flags - ms684863 - CREATE_BREAKAWAY_FROM_JOB = 0x01000000 + # Don't create a job object if the current process is already inside one. + if not win32job.IsProcessInJob(win32process.GetCurrentProcess(), None): + self.job_object = win32job.CreateJobObject(None, '') - proc = Popen(argv, creationflags=CREATE_BREAKAWAY_FROM_JOB) + job_info = win32job.QueryInformationJobObject( + self.job_object, win32job.JobObjectExtendedLimitInformation) + job_info['BasicLimitInformation']['LimitFlags'] |= \ + win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + win32job.SetInformationJobObject( + self.job_object, + win32job.JobObjectExtendedLimitInformation, + job_info) - self.job_object = win32job.CreateJobObject(None, '') + # Magic number needed to allow job reassignment in Windows 7 + # see: MSDN - Process Creation Flags - ms684863 + creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB - job_info = win32job.QueryInformationJobObject( - self.job_object, win32job.JobObjectExtendedLimitInformation) - job_info['BasicLimitInformation']['LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - win32job.SetInformationJobObject( - self.job_object, - win32job.JobObjectExtendedLimitInformation, - job_info) + proc = Popen(argv, creationflags=creation_flags, stdout=PIPE, stderr=None, bufsize=0) + if self.job_object is not None: win32job.AssignProcessToJobObject(self.job_object, proc._handle) - else: - proc = Popen(argv) - return proc def stop(self): @@ -317,11 +355,53 @@ class mongod(NullMongod): print >> sys.stderr, "probable bug: self.proc unset in stop()" return try: - if os.sys.platform == "win32": + if os.sys.platform == "win32" and self.job_object is not None: + # If smoke.py created its own job object, then we clean up the spawned processes by + # terminating it. import win32job win32job.TerminateJobObject(self.job_object, -1) # Windows doesn't seem to kill the process immediately, so give it some time to die time.sleep(5) + elif os.sys.platform == "win32": + # If smoke.py didn't create its own job object, then we attempt to clean up the + # spawned processes by terminating them individually. + import win32api + import win32con + import win32event + import win32process + import winerror + + def win32_terminate(handle): + # Adapted from implementation of Popen.terminate() in subprocess.py of Python + # 2.7 because earlier versions do not catch exceptions. + try: + win32process.TerminateProcess(handle, -1) + except win32process.error as err: + # ERROR_ACCESS_DENIED (winerror=5) is received when the process has + # already died. + if err.winerror != winerror.ERROR_ACCESS_DENIED: + raise + return_code = win32process.GetExitCodeProcess(handle) + if return_code == win32con.STILL_ACTIVE: + raise + + # Terminate the mongod process underlying buildlogger.py if one exists. + if self._inner_proc_pid is not None: + # The PROCESS_TERMINATE privilege is necessary to call TerminateProcess() and + # the SYNCHRONIZE privilege is necessary to call WaitForSingleObject(). See + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684880(v=vs.85).aspx + # for more details. + required_access = win32con.PROCESS_TERMINATE | win32con.SYNCHRONIZE + inner_proc_handle = win32api.OpenProcess(required_access, + False, + self._inner_proc_pid) + try: + win32_terminate(inner_proc_handle) + win32event.WaitForSingleObject(inner_proc_handle, win32event.INFINITE) + finally: + win32api.CloseHandle(inner_proc_handle) + + win32_terminate(self.proc._handle) elif hasattr(self.proc, "terminate"): # This method added in Python 2.6 self.proc.terminate() @@ -331,6 +411,10 @@ class mongod(NullMongod): print >> sys.stderr, "error shutting down mongod" print >> sys.stderr, e self.proc.wait() + + if self._stdout_pipe is not None: + self._stdout_pipe.wait_until_finished() + sys.stderr.flush() sys.stdout.flush() diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 0dc36dd54cf..9bef88f9a0e 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -385,6 +385,57 @@ functions: rm -rf src /data/db/* mongo-coredumps.tgz ~/.aws + "set up kitchen credentials": + command: shell.exec + params: + silent: true + script: | + set -o errexit + + mkdir -p ~/.ssh ~/.aws + echo -n "${kitchen_private_key}" > ~/.ssh/kitchen.pem + chmod 0600 ~/.ssh/kitchen.pem + + cat < ~/.aws/config + [default] + region = us-east-1 + EOF + + cat < ~/.aws/credentials + [default] + aws_access_key_id = ${kitchen_aws_key} + aws_secret_access_key = ${kitchen_aws_secret} + EOF + + "run kitchen": + command: shell.exec + type: test + params: + shell: bash + working_dir: src/buildscripts/package_test + script: | + set -o errexit + + export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz" + export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" + export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" + export KITCHEN_SUBNET="${kitchen_subnet}" + + for i in {1..3} + do + if ! kitchen verify "${packager_distro}"; then + verified="false" + kitchen destroy "${packager_distro}" || true + sleep 30 + else + verified="true" + break + fi + done + + kitchen destroy "${packager_distro}" || true + test "$verified" = "true" + pre: - command: shell.track - command: shell.exec @@ -594,6 +645,15 @@ tasks: ${python|python} ${packager_script} --prefix `pwd`/.. --distros ${packager_distro} --tarball `pwd`/../mongodb-binaries.tgz -s ${version} -m HEAD -a ${packager_arch} cd .. fi + + # Create separate shell archive + mkdir -p shell-archive/build + cd shell-archive + ${platform_decompress|tar xzvf} ../mongodb-binaries.tgz + find . -mindepth 3 ! -name "mongo${exe}" -type f -exec rm {} \; # delete bin/* except bin/mongo + ${python|python} ../buildscripts/make_archive.py -o mongodb-shell.${ext|tgz} $(find mongodb-* -type f) + cd .. + - command: archive.targz_pack params: target: "artifacts.tgz" @@ -632,6 +692,16 @@ tasks: permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Binaries + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/shell-archive/mongodb-shell.${ext|tgz} + remote_file: mongodb-mongo-v3.2/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Shell - command: s3.put params: aws_key: ${aws_key} @@ -1292,6 +1362,17 @@ tasks: resmoke_args: --suites=multiversion run_multiple_jobs: true +- <<: *task_template + name: multiversion_auth + commands: + - func: "do setup" + - func: "do multiversion setup" + - func: "run tests" + vars: + path_prefix: PATH=$PATH:/data/multiversion + resmoke_args: --suites=multiversion_auth + run_multiple_jobs: true + - <<: *task_template name: noPassthrough commands: @@ -1656,6 +1737,7 @@ tasks: - <<: *task_template name: sharding + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1665,6 +1747,7 @@ tasks: - <<: *task_template name: sharding_WT + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1699,6 +1782,7 @@ tasks: - <<: *task_template name: sharding_WT_ese + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1717,6 +1801,7 @@ tasks: - <<: *task_template name: sharding_auth + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1728,6 +1813,7 @@ tasks: name: sharding_auth_audit_WT depends_on: - name: sharding_auth + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1842,6 +1928,14 @@ tasks: resmoke_args: --suites=tool --storageEngine=wiredTiger run_multiple_jobs: true +- name: package + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "set up kitchen credentials" + - func: "run kitchen" + - name: push patchable: false depends_on: @@ -1850,6 +1944,13 @@ tasks: commands: - func: "fetch artifacts" - func: "fetch binaries" + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: mongodb-mongo-v3.2/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} + bucket: mciuploads + local_file: src/mongo-shell.tgz - command: s3.get params: aws_key: ${aws_key} @@ -1896,23 +1997,16 @@ tasks: set -o errexit set -o verbose - mv mongo-binaries.tgz mongodb-binaries.${ext|tgz} - mv mongo-debugsymbols.tgz debugsymbols-*.${ext|tgz} || true - cp mongodb-*.${ext|tgz} mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} - cp debugsymbols-*.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} || true + mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} + mv mongo-shell.tgz mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} + mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} || true mv distsrc.${ext|tgz} mongodb-src-${src_suffix}.${ext|tar.gz} || true /usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-win32-${push_arch}-${suffix}.msi || true - # Extract shell and banner files out of scons-made archive for separate shell archive - ${platform_decompress|tar --wildcards -zxvf} mongodb-binaries.${ext|tgz} mongodb-*/bin/mongo${exe} mongodb-*/\[A-Z\]* - - # Create new shell archive (make_archive.py uses filename to determine what format archive to create) - ${python|python} buildscripts/make_archive.py -o mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} $(ls -d mongodb-*/bin/mongo${exe} mongodb-*/[A-Z]*) - notary-client.py --key-name "server-3.2" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz} if [ "${has_packages|}" = "true" ]; then - CURATOR_RELEASE="fab90fac3cb45c0ef411e647b3d621cbe053fe19" + CURATOR_RELEASE="88f34a9f1c79db7ea9597b6e85eb5995d03714ba" curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo @@ -2822,6 +2916,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: ubuntu1404 @@ -2930,6 +3027,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: ubuntu1604 @@ -3038,15 +3138,19 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-linux-64-amazon-ami - display_name: Enterprise Amazon Linux + display_name: "Enterprise Amazon Linux" modules: - enterprise - mongo-tools run_on: - linux-64-amzn-test + batchtime: 1440 # 1 day expansions: tooltags: "-tags 'ssl sasl'" push_path: linux @@ -3114,6 +3218,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: amazon @@ -3225,6 +3332,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push distros: - rhel70 @@ -3424,6 +3534,7 @@ buildvariants: - name: mmap - name: mongosTest - name: multiversion + - name: multiversion_auth - name: noPassthrough - name: noPassthrough_WT - name: noPassthroughWithMongod @@ -3468,7 +3579,7 @@ buildvariants: - rhel70 - name: windows-64-2k8-debug - display_name: Windows 2008R2 DEBUG + display_name: "! Windows 2008R2 DEBUG" modules: - mongo-tools run_on: @@ -3564,7 +3675,7 @@ buildvariants: - name: unittests - name: enterprise-windows-64 - display_name: Enterprise Windows + display_name: "* Enterprise Windows" modules: - enterprise - mongo-tools @@ -3811,13 +3922,14 @@ buildvariants: modules: - mongo-tools run_on: - - osx-1010 + - macos-1012 batchtime: 1440 # 1 day expansions: push_path: osx push_bucket: downloads.mongodb.org push_name: osx push_arch: x86_64 + gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 compile_flags: --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ num_jobs_available: 1 tasks: @@ -3899,7 +4011,7 @@ buildvariants: modules: - mongo-tools run_on: - - osx-1010 + - macos-1012 batchtime: 1440 # 1 day expansions: push_path: osx @@ -3907,7 +4019,8 @@ buildvariants: push_name: osx-ssl push_arch: x86_64 tooltags: "-tags ssl" - compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ + gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 + compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include num_jobs_available: 1 tasks: - name: compile @@ -3984,17 +4097,18 @@ buildvariants: - rhel70 - name: osx-107-debug - display_name: OS X 10.7 DEBUG + display_name: "* OS X 10.7 DEBUG" modules: - mongo-tools run_on: - - osx-1010 + - macos-1012 expansions: push_path: osx push_bucket: downloads.mongodb.org push_name: osx-debug push_arch: x86_64 num_jobs_available: 1 + gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 compile_flags: --dbg=on --opt=on --allocator=system -j$(sysctl -n hw.logicalcpu) --osx-version-min=10.7 --libc++ tasks: - name: compile @@ -4029,7 +4143,7 @@ buildvariants: - enterprise - mongo-tools run_on: - - osx-1010 + - macos-1012 batchtime: 1440 # 1 day expansions: push_path: osx @@ -4037,7 +4151,8 @@ buildvariants: push_name: osx push_arch: x86_64-enterprise tooltags: "-tags 'ssl sasl'" - compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ + gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 + compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include num_jobs_available: 1 tasks: - name: compile @@ -4054,8 +4169,6 @@ buildvariants: - name: replicasets_auth - name: sasl - name: sharding_csrs_upgrade_auth_WT - - name: snmp - - name: snmp_WT - name: ssl - name: sslSpecial - name: unittests @@ -4117,7 +4230,7 @@ buildvariants: - name: push - name: enterprise-rhel-62-64-bit - display_name: Enterprise RHEL 6.2 + display_name: "! Enterprise RHEL 6.2" modules: - enterprise - mongo-tools @@ -4205,6 +4318,7 @@ buildvariants: - name: mmap - name: mongosTest - name: multiversion + - name: multiversion_auth - name: noPassthrough - name: noPassthroughWithMongod - name: noPassthroughWithMongod_WT @@ -4258,6 +4372,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push distros: - rhel70 @@ -4308,6 +4425,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-rhel-71-ppc64le @@ -4662,6 +4782,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push distros: - rhel70 @@ -4771,6 +4894,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push ########################################### @@ -4824,6 +4950,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-ubuntu1404-64 @@ -4873,6 +5002,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-ubuntu1604-64 @@ -4922,6 +5054,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -4977,6 +5112,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: suse12 @@ -5086,6 +5224,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -5136,6 +5277,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: suse11 @@ -5245,6 +5389,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push ########################################### @@ -5252,7 +5399,7 @@ buildvariants: ########################################### - name: solaris-64-bit - display_name: Solaris + display_name: "* Solaris" modules: - mongo-tools run_on: @@ -5381,6 +5528,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-debian81-64 @@ -5430,6 +5580,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -5543,6 +5696,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: debian81 @@ -5655,6 +5811,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -5890,20 +6049,21 @@ buildvariants: # Experimental buildvariants # ########################################### -- name: ubuntu1410-debug-asan - display_name: ~ ASAN Enterprise SSL Ubuntu 14.10 DEBUG +- name: ubuntu1604-debug-asan + display_name: ~ ASAN Enterprise SSL Ubuntu 16.04 DEBUG modules: - enterprise - mongo-tools run_on: - - ubuntu1410-build + - ubuntu1604-build stepback: false batchtime: 1440 # 1 day expansions: tooltags: "-tags 'ssl'" - asan_symbolizer: ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5 + # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. + path_prefix: PATH=$PATH:/usr/lib/llvm-3.8/bin enable_lsan: LSAN_OPTIONS="suppressions=etc/lsan.suppressions" ASAN_OPTIONS=detect_leaks=1 - compile_flags: CC=/usr/bin/clang CXX=/usr/bin/clang++ --dbg=on --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build + compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" --dbg=on --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build num_jobs_available: $(grep -c ^processor /proc/cpuinfo) test_flags: -j1 # Avoid starting too many mongod's under ASAN build. tasks: @@ -5967,6 +6127,7 @@ buildvariants: - name: mmap - name: mongosTest - name: multiversion + - name: multiversion_auth - name: noPassthrough - name: noPassthroughWithMongod - name: noPassthroughWithMongod_WT @@ -6018,19 +6179,20 @@ buildvariants: - name: tool_WT - name: unittests -- name: ubuntu1410-asan - display_name: ~ ASAN SSL Ubuntu 14.10 +- name: ubuntu1604-asan + display_name: ~ ASAN SSL Ubuntu 16.04 modules: - mongo-tools run_on: - - ubuntu1410-build + - ubuntu1604-build stepback: false batchtime: 1440 # 1 day expansions: tooltags: "-tags 'ssl'" - asan_symbolizer: ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5 + # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. + path_prefix: PATH=$PATH:/usr/lib/llvm-3.8/bin enable_lsan: LSAN_OPTIONS="suppressions=etc/lsan.suppressions" ASAN_OPTIONS=detect_leaks=1 - compile_flags: CC=/usr/bin/clang CXX=/usr/bin/clang++ --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build + compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build num_jobs_available: $(grep -c ^processor /proc/cpuinfo) test_flags: -j1 # Avoid starting too many mongod's under ASAN build. tasks: @@ -6039,4 +6201,4 @@ buildvariants: - name: jstestfuzz_WT - name: jstestfuzz_replication_WT - name: jstestfuzz_sharded_WT - - name: unittests \ No newline at end of file + - name: unittests diff --git a/etc/longevity.yml b/etc/longevity.yml index 0b820841616..b5cb674a354 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -43,8 +43,8 @@ post: set -e set -o verbose # call terraform destroy twice to avoid AWS timeout - yes yes | ./terraform destroy - yes yes | ./terraform destroy + yes yes | ./terraform destroy --var-file=cluster.json + yes yes | ./terraform destroy --var-file=cluster.json # clean all file to be safe rm -rf * echo "Cluster DESTROYED." @@ -57,66 +57,99 @@ functions: rm -rf ./* mkdir src mkdir work - mkdir bin - mkdir keys - pwd - ls - command: manifest.load - command: git.get_project params: directory: src revisions: # for each module include revision as : ${_rev} dsi: ${dsi_rev} - YCSB: ${YCSB_rev} - - command: git.apply_patch + - command: shell.exec params: - directory: src + working_dir: work + script: | + cat > bootstrap.yml < runtime.yml < secret - chmod 400 secret - echo "${ec2_pem}" > keys/aws.pem - chmod 400 keys/aws.pem + # AWS ssh secret key + echo "${ec2_pem}" > aws_ssh_key.pem + chmod 400 aws_ssh_key.pem + + cat > runtime_secret.yml < : ${_rev} dsi: ${dsi_rev} workloads: ${workloads_rev} - YCSB: ${YCSB_rev} - - command: git.apply_patch + - command: shell.exec params: - directory: src + working_dir: work + script: | + cat > bootstrap.yml < runtime.yml < secret - chmod 400 secret - echo "${ec2_pem}" > keys/aws.pem - chmod 400 keys/aws.pem + # AWS ssh secret key + echo "${ec2_pem}" > aws_ssh_key.pem + chmod 400 aws_ssh_key.pem + + cat > runtime_secret.yml < 1) || privileges.length > 1) { - for (var j = 0; j < privileges.length; j++) { - var p = privileges[j]; + if ((testcase.privileges.length == 1 && testcase.privileges[0].actions.length > 1) || + testcase.privileges.length > 1) { + for (var j = 0; j < testcase.privileges.length; j++) { + var p = testcase.privileges[j]; var resource = p.resource; var actions = p.actions; + // A particular privilege can explicitly specify that it should not be removed when + // testing for authorization failure. This accommodates special-case behavior for + // views in conjunction with the create and collMod commands. + if (p.removeWhenTestingAuthzFailure === false) { + continue; + } + for (var k = 0; k < actions.length; k++) { var privDoc = { resource: resource, @@ -114,27 +120,58 @@ function runOneTest(conn, t) { } } - msg = testProperAuthorization(conn, t, testcase); + // Test for proper authorization with the privileges specified in the test case. + msg = testProperAuthorization(conn, t, testcase, testcase.privileges); if (msg) { failures.push(t.testname + ": " + msg); } - // test resource pattern where collection is "" - privileges.forEach(function(j) { - if (j.resource.collection && !j.resource.collection.startsWith('system.')) { - j.resource.collection = ""; - } - }); - msg = testProperAuthorization(conn, t, testcase); + + var specialResource = function(resource) { + if (!resource) + return true; + + // Tests which use {db: "local", collection: "oplog.rs"} will not work with + // {db: "", collection: "oplog.rs"}. oplog.rs is special, and does not match with + // forDatabaseName or anyNormalResource ResourcePatterns. The same is true of + // oplog.$main, but oplog.$main is also an illegal collection name on any database + // other than local. The other collections checked for here in the local database have + // the same property as oplog.rs. + return !resource.db || !resource.collection || + resource.collection.startsWith("system.") || resource.db == "local"; + }; + + // Test for proper authorization with the test case's privileges where non-system + // collections are modified to be the empty string. + msg = testProperAuthorization( + conn, + t, + testcase, + testcase.privileges.map(function(priv) { + // Make a copy of the privilege so as not to modify the original array. + var modifiedPrivilege = Object.extend({}, priv, true); + if (modifiedPrivilege.resource.collection && !specialResource(priv.resource)) { + modifiedPrivilege.resource.collection = ""; + } + return modifiedPrivilege; + })); if (msg) { failures.push(t.testname + ": " + msg); } - // test resource pattern where database is "" - privileges.forEach(function(j) { - if (j.resource.db) { - j.resource.db = ""; - } - }); - msg = testProperAuthorization(conn, t, testcase); + + // Test for proper authorization with the test case's privileges where the database is the + // empty string. + msg = testProperAuthorization(conn, + t, + testcase, + testcase.privileges.map(function(priv) { + // Make a copy of the privilege so as not to modify the + // original array. + var modifiedPrivilege = Object.extend({}, priv, true); + if (!specialResource(priv.resource)) { + modifiedPrivilege.resource.db = ""; + } + return modifiedPrivilege; + })); if (msg) { failures.push(t.testname + ": " + msg); } diff --git a/jstests/auth/server-4892.js b/jstests/auth/server-4892.js index ef0c95c868e..4ca6318230f 100644 --- a/jstests/auth/server-4892.js +++ b/jstests/auth/server-4892.js @@ -74,11 +74,7 @@ withMongod({auth: ""}, cursor.next(); expectNumLiveCursors(mongod, 1); - cursor = null; - // NOTE(schwerin): We assume that after setting cursor = null, there are no remaining - // references - // to the cursor, and that gc() will deterministically garbage collect it. - gc(); + cursor.close(); // NOTE(schwerin): dbKillCursors gets piggybacked on subsequent messages on the // connection, so we diff --git a/jstests/concurrency/fsm_all.js b/jstests/concurrency/fsm_all.js index 35031becb89..4c45ac7b99d 100644 --- a/jstests/concurrency/fsm_all.js +++ b/jstests/concurrency/fsm_all.js @@ -4,7 +4,12 @@ load('jstests/concurrency/fsm_libs/runner.js'); var dir = 'jstests/concurrency/fsm_workloads'; -var blacklist = [].map(function(file) { +var blacklist = [ + // Disabled due to MongoDB restrictions and/or workload restrictions + + // This workload assumes it is running against a sharded cluster. + 'sharded_moveChunk_drop_shard_key_index.js', +].map(function(file) { return dir + '/' + file; }); diff --git a/jstests/concurrency/fsm_all_composed.js b/jstests/concurrency/fsm_all_composed.js index 159ff0919c9..c32b039dd32 100644 --- a/jstests/concurrency/fsm_all_composed.js +++ b/jstests/concurrency/fsm_all_composed.js @@ -16,6 +16,9 @@ var blacklist = [ // is slow and the composer doesn't honor iteration counts: 'remove_single_document_eval_nolock.js', 'update_simple_eval_nolock.js', + + // This workload assumes it is running against a sharded cluster. + 'sharded_moveChunk_drop_shard_key_index.js', ].map(function(file) { return dir + '/' + file; }); diff --git a/jstests/concurrency/fsm_all_replication.js b/jstests/concurrency/fsm_all_replication.js index 31770af9f80..93be7b7dfe2 100644 --- a/jstests/concurrency/fsm_all_replication.js +++ b/jstests/concurrency/fsm_all_replication.js @@ -9,6 +9,9 @@ var blacklist = [ 'agg_group_external.js', // uses >100MB of data, which can overwhelm test hosts 'agg_sort_external.js', // uses >100MB of data, which can overwhelm test hosts 'findAndModify_update_grow.js', // can cause OOM kills on test hosts + + // This workload assumes it is running against a sharded cluster. + 'sharded_moveChunk_drop_shard_key_index.js', ].map(function(file) { return dir + '/' + file; }); diff --git a/jstests/concurrency/fsm_all_sharded_replication.js b/jstests/concurrency/fsm_all_sharded_replication.js index d4068148d47..435277a2a47 100644 --- a/jstests/concurrency/fsm_all_sharded_replication.js +++ b/jstests/concurrency/fsm_all_sharded_replication.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'. diff --git a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js index 28151a05643..e4355a62789 100644 --- a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js +++ b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'. diff --git a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js index c903331e971..167efc452ed 100644 --- a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js +++ b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'remove_where.js', // SERVER-14669 Multi-removes that use $where miscount removed documents @@ -73,6 +74,11 @@ var blacklist = [ 'rename_collection_dbname_droptarget.js', 'rename_collection_droptarget.js', + // This workload assumes that the distributed lock can always be acquired when running the split + // command in its setup() function; however, a LockBusy error may be returned if the balancer is + // running. + 'sharded_moveChunk_drop_shard_key_index.js', + 'update_simple_eval.js', // eval doesn't work with sharded collections 'update_simple_eval_nolock.js', // eval doesn't work with sharded collections 'update_upsert_multi.js', // our update queries lack shard keys diff --git a/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js b/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js index 444f7eab3cb..9e63ec701ee 100644 --- a/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js +++ b/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'remove_where.js', // SERVER-14669 Multi-removes that use $where miscount removed documents @@ -74,6 +75,11 @@ var blacklist = [ 'rename_collection_dbname_droptarget.js', 'rename_collection_droptarget.js', + // This workload assumes that the distributed lock can always be acquired when running the split + // command in its setup() function; however, a LockBusy error may be returned if the balancer is + // running. + 'sharded_moveChunk_drop_shard_key_index.js', + 'update_simple_eval.js', // eval doesn't work with sharded collections 'update_simple_eval_nolock.js', // eval doesn't work with sharded collections 'update_upsert_multi.js', // our update queries lack shard keys diff --git a/jstests/concurrency/fsm_workloads/create_database.js b/jstests/concurrency/fsm_workloads/create_database.js new file mode 100644 index 00000000000..6b996064d4d --- /dev/null +++ b/jstests/concurrency/fsm_workloads/create_database.js @@ -0,0 +1,150 @@ +'use strict'; + +/** + * create_database.js + * + * Repeatedly creates and drops a database, with the focus on creation using different name casing. + * Create using all different methods, implicitly by inserting, creating views/indexes etc. + * + * Each thread uses its own database, though sometimes threads may try to create databases with + * names that only differ in case, expecting the appriopriate error code. + */ +var $config = (function() { + + var data = { + checkCommandResult: function checkCommandResult(mayFailWithDatabaseDifferCase, res) { + if (mayFailWithDatabaseDifferCase && !res.ok) + assertAlways.commandFailedWithCode(res, ErrorCodes.DatabaseDifferCase); + else + assertAlways.commandWorked(res); + return res; + }, + + checkWriteResult: function checkWriteResult(mayFailWithDatabaseDifferCase, res) { + if (mayFailWithDatabaseDifferCase && res.hasWriteError()) { + assertAlways.eq(res.getWriteError().code, ErrorCodes.DatabaseDifferCase); + } else + assertAlways.writeOK(res); + return res; + } + }; + + var states = (function() { + function init(db, collName) { + var uniqueNr = this.tid; + var semiUniqueNr = Math.floor(uniqueNr / 2); + + // The semiUniqueDBName may clash and result in a DatabaseDifferCas error on + // creation, + // while the uniqueDBName does not clash. The unique and created variables track + // this. + this.semiUniqueDBName = + (this.tid % 2 ? 'create_database' : 'CREATE_DATABASE') + semiUniqueNr; + this.uniqueDBName = 'CreateDatabase' + uniqueNr; + this.myDB = db.getSiblingDB(this.uniqueDBName); + this.created = false; + this.unique = true; + } + + function useSemiUniqueDBName(db, collName) { + this.myDB = db.getSiblingDB(this.semiUniqueDBName); + this.unique = false; + } + + function createCollection(db, collName) { + this.created = + this.checkCommandResult(!this.unique, this.myDB.createCollection(collName)).ok; + } + + function createIndex(db, collName) { + var background = Math.random > 0.5; + var res = this.myDB.getCollection(collName).createIndex({x: 1}, {background}); + this.created |= + this.checkCommandResult(!this.unique, res).createdCollectionAutomatically; + } + + function insert(db, collName) { + this.created |= this.checkWriteResult(!this.created && !this.unique, + this.myDB.getCollection(collName).insert({x: 1})) + .nInserted == 1; + } + + function upsert(db, collName) { + this.created |= this.checkWriteResult(!this.created && !this.unique, + this.myDB.getCollection(collName).update( + {x: 1}, {x: 2}, {upsert: 1})).nUpserted == 1; + } + + function drop(db, collName) { + if (this.created) + assertAlways(this.myDB.getCollection(collName).drop()); + } + + function dropDatabase(db, collName) { + if (this.created) + assertAlways.commandWorked(this.myDB.dropDatabase()); + } + + function listDatabases(db, collName) { + for (var database of db.adminCommand({listDatabases: 1}).databases) { + var res = db.getSiblingDB(database.name).runCommand({listCollections: 1}); + assertAlways.commandWorked(res); + assertAlways.neq(database.name, this.myDB.toString(), "this DB shouldn't exist"); + } + } + + function listDatabasesNameOnly(db, collName) { + for (var database of db.adminCommand({listDatabases: 1, nameOnly: 1}).databases) { + var res = db.getSiblingDB(database.name).runCommand({listCollections: 1}); + assertAlways.commandWorked(res); + assertAlways.neq(database.name, this.myDB.toString(), "this DB shouldn't exist"); + } + } + + return { + init: init, + useSemiUniqueDBName: useSemiUniqueDBName, + createCollection: createCollection, + createIndex: createIndex, + insert: insert, + upsert: upsert, + drop: drop, + dropDatabase: dropDatabase, + listDatabases: listDatabases, + listDatabasesNameOnly: listDatabasesNameOnly, + }; + })(); + + var transitions = { + init: { + useSemiUniqueDBName: 0.25, + createCollection: 0.375, + createIndex: 0.125, + insert: 0.125, + upsert: 0.125 + }, + useSemiUniqueDBName: {createCollection: 1.00}, + createCollection: {dropDatabase: 0.25, createIndex: 0.25, insert: 0.25, upsert: 0.25}, + createIndex: {insert: 0.25, upsert: 0.25, dropDatabase: 0.5}, + insert: {dropDatabase: 0.2, drop: 0.05, insert: 0.5, upsert: 0.25}, + upsert: {dropDatabase: 0.2, drop: 0.05, insert: 0.25, upsert: 0.5}, + drop: {dropDatabase: 0.75, init: 0.25}, // OK to leave the empty database behind sometimes + dropDatabase: {init: 0.75, listDatabases: 0.15, listDatabasesNameOnly: 0.10}, + listDatabases: {init: 0.75, listDatabases: 0.15, listDatabasesNameOnly: 0.10}, + listDatabasesNameOnly: {init: 0.75, listDatabases: 0.10, listDatabasesNameOnly: 0.15}, + }; + + return { + data: data, + // We only run a few iterations to reduce the amount of data cumulatively + // written to disk by mmapv1. For example, setting 10 threads and 180 + // iterations (with an expected 6 transitions per create/drop roundtrip) + // causes this workload to write at least 32MB (.ns and .0 files) * 10 threads + // * 30 iterations worth of data to disk, or about 10GB, which can be slow on + // test hosts. + threadCount: 10, + iterations: 180, + states: states, + transitions: transitions, + }; +})(); diff --git a/jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js b/jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js new file mode 100644 index 00000000000..c476b7a0dab --- /dev/null +++ b/jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js @@ -0,0 +1,84 @@ +'use strict'; + +/** + * sharded_moveChunk_drop_shard_key_index.js + * + * Tests that dropping the shard key index while migrating a chunk doesn't cause the shard to abort. + * + * This workload was designed to reproduce SERVER-24994. + */ + +var $config = (function() { + + var data = { + numSplitPoints: 100, + shardKey: {key: 1} + }; + + var states = { + + init: function init(db, collName) { + // No-op + }, + + moveChunk: function moveChunk(db, collName) { + var configDB = db.getSiblingDB('config'); + var shards = configDB.shards.aggregate([{$sample: {size: 1}}]).toArray(); + assertAlways.eq(1, shards.length, tojson(shards)); + + var shardName = shards[0]._id; + var chunkBoundary = Random.randInt(this.numSplitPoints); + + // We don't assert that the command succeeded when migrating a chunk because it's + // possible another thread has already started migrating a chunk. + db.adminCommand({ + moveChunk: db[collName].getFullName(), + find: {key: chunkBoundary}, + to: shardName, + _waitForDelete: true, + }); + }, + + dropIndex: function dropIndex(db, collName) { + // We don't assert that the command succeeded when dropping an index because it's + // possible another thread has already dropped this index. + db[collName].dropIndex(this.shardKey); + + // Re-create the index that was dropped. + assertAlways.commandWorked(db[collName].createIndex(this.shardKey)); + } + + }; + + var transitions = { + init: {moveChunk: 0.5, dropIndex: 0.5}, + moveChunk: {moveChunk: 0.5, dropIndex: 0.5}, + dropIndex: {moveChunk: 0.5, dropIndex: 0.5} + }; + + function setup(db, collName, cluster) { + var bulk = db[collName].initializeUnorderedBulkOp(); + for (var i = 0; i < this.numSplitPoints; ++i) { + bulk.insert({key: i}); + } + + var res = bulk.execute(); + assertAlways.writeOK(res); + assertAlways.eq(this.numSplitPoints, res.nInserted, tojson(res)); + + for (i = 0; i < this.numSplitPoints; ++i) { + assertWhenOwnColl.commandWorked( + db.adminCommand({split: db[collName].getFullName(), middle: {key: i}})); + } + } + + return { + threadCount: 10, + iterations: 100, + data: data, + states: states, + transitions: transitions, + setup: setup + }; + +})(); diff --git a/jstests/core/currentop.js b/jstests/core/currentop.js index ef948d415be..296ad69355c 100644 --- a/jstests/core/currentop.js +++ b/jstests/core/currentop.js @@ -1,82 +1,32 @@ -print("BEGIN currentop.js"); - -// test basic currentop functionality + querying of nested documents -t = db.jstests_currentop; -t.drop(); - -for (i = 0; i < 100; i++) { - t.save({"num": i}); -} - -print("count:" + t.count()); - -function ops(q) { - printjson(db.currentOp().inprog); - return db.currentOp(q).inprog; -} - -print("start shell"); - -// sleep for a second for each (of 100) documents; can be killed in between documents & test should -// complete before 100 seconds -s1 = startParallelShell("db.jstests_currentop.count( { '$where': function() { sleep(1000); } } )"); - -print("sleep"); -sleep(1000); - -print("inprog:"); -printjson(db.currentOp().inprog); -print(); -sleep(1); -print("inprog:"); -printjson(db.currentOp().inprog); -print(); - -// need to wait for read to start -print("wait have some ops"); -assert.soon(function() { - return ops({"locks.Collection": "r", "ns": "test.jstests_currentop"}).length + - ops({"locks.Collection": "R", "ns": "test.jstests_currentop"}).length >= - 1; -}, "have_some_ops"); -print("ok"); - -s2 = startParallelShell("db.jstests_currentop.update({ '$where': function() { sleep(150); } }," + - " { '$inc': {num: 1} }, false, true );"); - -o = []; - -function f() { - o = ops({"ns": "test.jstests_currentop"}); - - printjson(o); - - var writes = ops({"locks.Collection": "w", "ns": "test.jstests_currentop"}).length; - - var readops = ops({"locks.Collection": "r", "ns": "test.jstests_currentop"}); - print("readops:"); - printjson(readops); - var reads = readops.length; - - print("total: " + o.length + " w: " + writes + " r:" + reads); - - return o.length > writes && o.length > reads; -} - -print("go"); - -assert.soon(f, "f"); - -// avoid waiting for the operations to complete (if soon succeeded) -for (var i in o) { - db.killOp(o[i].opid); -} - -start = new Date(); - -// The operations running in the parallel shells may or may not have been killed. -s1({checkExitSuccess: false}); -s2({checkExitSuccess: false}); - -// don't want to pass if timeout killed the js function -assert((new Date()) - start < 30000); +/** + * Tests that long-running operations show up in currentOp and report the locks they are holding. + */ +(function() { + "use strict"; + const coll = db.jstests_currentop; + coll.drop(); + + // We fsync+lock the server to cause all subsequent write operations to block. + assert.commandWorked(db.fsyncLock()); + + const awaitInsertShell = startParallelShell(function() { + assert.writeOK(db.jstests_currentop.insert({})); + }); + + // Wait until the write appears in the currentOp output reporting that it is waiting for a lock. + assert.soon( + function() { + return db.currentOp({ + ns: coll.getFullName(), + "locks.Global": "w", + "waitingForLock": true, + }).inprog.length === 1; + }, + function() { + return "Failed to find blocked insert in currentOp() output: " + tojson(db.currentOp()); + }); + + // Unlock the server and make sure the write finishes. + assert.commandWorked(db.fsyncUnlock()); + awaitInsertShell(); +}()); diff --git a/jstests/core/dbadmin.js b/jstests/core/dbadmin.js index 061b9ff979e..2afc958c526 100644 --- a/jstests/core/dbadmin.js +++ b/jstests/core/dbadmin.js @@ -7,8 +7,12 @@ load('jstests/aggregation/extras/utils.js'); t.save({x: 1}); t.save({x: 1}); - var res = db._adminCommand("listDatabases"); - assert(res.databases && res.databases.length > 0, "listDatabases 1 " + tojson(res)); + var res = db.adminCommand("listDatabases"); + assert(res.databases && res.databases.length > 0, "listDatabases: " + tojson(res)); + + var res = db.adminCommand({listDatabases: 1, nameOnly: true}); + assert(res.databases && res.databases.length > 0 && res.totalSize === undefined, + "listDatabases nameOnly: " + tojson(res)); var now = new Date(); var x = db._adminCommand("ismaster"); diff --git a/jstests/core/diagdata.js b/jstests/core/diagdata.js new file mode 100644 index 00000000000..490e4a3eb2b --- /dev/null +++ b/jstests/core/diagdata.js @@ -0,0 +1,33 @@ +// Test that verifies getDiagnosticData returns FTDC data + +(function() { + "use strict"; + + // Verify we require admin database + assert.commandFailed(db.diagdata.runCommand("getDiagnosticData")); + + // We need to retry a few times if run this test immediately after mongod is started as FTDC may + // not have run yet. + var foundGoodDocument = false; + + for (var i = 0; i < 60; ++i) { + var result = db.adminCommand("getDiagnosticData"); + assert.commandWorked(result); + + var data = result.data; + + if (!data.hasOwnProperty("start")) { + // Wait a little longer for FTDC to start + sleep(500); + } else { + // Check for a few common properties to ensure we got data + assert(data.hasOwnProperty("serverStatus"), + "does not have 'serverStatus' in '" + tojson(data) + "'"); + assert(data.hasOwnProperty("end"), "does not have 'end' in '" + tojson(data) + "'"); + foundGoodDocument = true; + } + } + assert(foundGoodDocument, + "getDiagnosticData failed to return a non-empty command, is FTDC running?"); + +})(); diff --git a/jstests/core/drop2.js b/jstests/core/drop2.js deleted file mode 100644 index 54f6b8b78ab..00000000000 --- a/jstests/core/drop2.js +++ /dev/null @@ -1,57 +0,0 @@ -var coll = db.jstests_drop2; -coll.drop(); - -function debug(x) { - printjson(x); -} - -coll.save({}); - -function getOpId(drop) { - var inProg = db.currentOp().inprog; - debug(inProg); - for (var id in inProg) { - var op = inProg[id]; - if (drop) { - if (op.query && op.query.drop && op.query.drop == coll.getName()) { - return op.opid; - } - } else { - if (op.query && op.query.query && op.query.query.$where && op.ns == (coll + "")) { - return op.opid; - } - } - } - return null; -} - -var awaitCount = startParallelShell( - "print(\"Count thread started\");" + "db.getMongo().getCollection(\"" + (coll + "") + "\")" + - ".count( { $where: function() {" + "while( 1 ) { sleep( 1 ); } } } );" + - "print(\"Count thread terminating\");"); -countOpId = null; -assert.soon(function() { - countOpId = getOpId(false); - return countOpId; -}); - -var awaitDrop = - startParallelShell("print(\"Drop thread started\");" + "print(\"drop result: \" + " + - "db.getMongo().getCollection(\"" + (coll + "") + "\")" + ".drop() );" + - "print(\"Drop thread terminating\")"); -dropOpId = null; -assert.soon(function() { - dropOpId = getOpId(true); - return dropOpId; -}); - -db.killOp(dropOpId); -db.killOp(countOpId); - -var exitCode = awaitCount({checkExitSuccess: false}); -assert.neq(0, exitCode, "expected shell to exit abnormally due to JS execution being terminated"); - -// The drop operation may or may not have been killed. -awaitDrop({checkExitSuccess: false}); - -coll.drop(); // in SERVER-1818, this fails diff --git a/jstests/core/evalh.js b/jstests/core/evalh.js new file mode 100644 index 00000000000..e1058fbdce4 --- /dev/null +++ b/jstests/core/evalh.js @@ -0,0 +1,18 @@ +/** + * Test that db.eval does not support auth. + */ +(function() { + 'use strict'; + + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); + assert(db.evalprep.drop()); + + // The db.auth method call getMongo().auth but catches the exception. + assert.eq(0, db.eval('db.auth("reader", "reader")')); + + // Call the native implementation auth function and verify it does not exist under the db.eval + // javascript context. + assert.throws(function() { + db.eval('db.getMongo().auth("reader", "reader")'); + }); +})(); diff --git a/jstests/core/evalj.js b/jstests/core/evalj.js new file mode 100644 index 00000000000..f2326fff365 --- /dev/null +++ b/jstests/core/evalj.js @@ -0,0 +1,13 @@ +(function() { + "use strict"; + + db.col.insert({data: 5}); + + db.eval("print(5)"); + + db.system.js.insert({_id: "foo", value: Code("db.col.drop()")}); + + db.eval("print(5)"); + + assert.eq(5, db.col.findOne()["data"]); +})(); diff --git a/jstests/core/hostinfo.js b/jstests/core/hostinfo.js index 6d27b195f39..110fa8f9d09 100644 --- a/jstests/core/hostinfo.js +++ b/jstests/core/hostinfo.js @@ -31,3 +31,13 @@ if (hostinfo.os.type != "") { assert.neq(hostinfo.system.cpuArch, "" || null, "Missing CPU Architecture"); assert.neq(hostinfo.system.numaEnabled, "" || null, "Missing NUMA flag"); } + +var buildInfo = assert.commandWorked(db.runCommand({buildInfo: 1})); +if (buildInfo.buildEnvironment && buildInfo.buildEnvironment.target_arch) { + var targetArch = buildInfo.buildEnvironment.target_arch; + if (targetArch == "i386") + assert.eq(hostinfo.system.cpuAddrSize, 32); + else + assert.eq(hostinfo.system.cpuAddrSize, 64); + assert.eq(hostinfo.system.cpuAddrSize, buildInfo.bits); +} diff --git a/jstests/core/index_partial_write_ops.js b/jstests/core/index_partial_write_ops.js index b962347a26d..22c4d0dd519 100644 --- a/jstests/core/index_partial_write_ops.js +++ b/jstests/core/index_partial_write_ops.js @@ -19,7 +19,7 @@ coll.drop(); // Create partial index. - assert.commandWorked(coll.ensureIndex({x: 1}, {partialFilterExpression: {a: 1}})); + assert.commandWorked(coll.ensureIndex({x: 1}, {unique: true, partialFilterExpression: {a: 1}})); assert.writeOK(coll.insert({_id: 1, x: 5, a: 2, b: 1})); // Not in index. assert.writeOK(coll.insert({_id: 2, x: 6, a: 1, b: 1})); // In index. @@ -59,4 +59,20 @@ // Delete that does affect partial index. assert.writeOK(coll.remove({x: 6})); assert.eq(0, getNumKeys("x_1")); + + // Documents with duplicate keys that straddle the index. + assert.writeOK(coll.insert({_id: 3, x: 1, a: 1})); // In index. + assert.writeOK(coll.insert({_id: 4, x: 1, a: 0})); // Not in index. + assert.writeErrorWithCode( + coll.insert({_id: 5, x: 1, a: 1}), + ErrorCodes.DuplicateKey); // Duplicate key constraint prevents insertion. + + // Only _id 3 is in the index. + assert.eq(1, getNumKeys("x_1")); + + // Remove _id 4, _id 3 should remain in index. + assert.writeOK(coll.remove({_id: 4})); + + // _id 3 is still in the index. + assert.eq(1, getNumKeys("x_1")); })(); diff --git a/jstests/core/killop_drop_collection.js b/jstests/core/killop_drop_collection.js new file mode 100644 index 00000000000..7138ee8eda6 --- /dev/null +++ b/jstests/core/killop_drop_collection.js @@ -0,0 +1,54 @@ +/** + * A killOp command issued against a collection drop should not interfere with the drop and allow it + * to complete. Interrupting a collection drop could leave the database in an inconsistent state. + * This test confirms that killOp won't interrupt a collection drop, and that the drop occurs + * successfully. + */ +(function() { + "use strict"; + + var collectionName = "killop_drop"; + var collection = db.getCollection(collectionName); + collection.drop(); + assert.writeOK(collection.insert({x: 1})); + + // Attempt to fsyncLock the database, aborting early if the storage engine doesn't support it. + var storageEngine = jsTest.options().storageEngine; + var fsyncRes = db.fsyncLock(); + if (!fsyncRes.ok) { + assert.commandFailedWithCode(fsyncRes, ErrorCodes.CommandNotSupported); + jsTest.log("Skipping test on storage engine " + storageEngine + + ", which does not support fsyncLock."); + return; + } + + // Kick off a drop on the collection. + var useDefaultPort = null; + var noConnect = false; + var awaitDropCommand = startParallelShell(function() { + assert.commandWorked(db.getSiblingDB("test").runCommand({drop: "killop_drop"})); + }, useDefaultPort, noConnect); + + // Wait for the drop operation to appear in the db.currentOp() output. + var dropCommandOpId = null; + assert.soon(function() { + var dropOpsInProgress = db.currentOp().inprog.filter(function(op) { + return op.query && op.query.drop === collection.getName(); + }); + if (dropOpsInProgress.length > 0) { + dropCommandOpId = dropOpsInProgress[0].opid; + } + return dropCommandOpId; + }); + + // Issue a killOp for the drop command, then unlock the server. We expect that the drop + // operation was *not* killed, and that the collection was dropped successfully. + assert.commandWorked(db.killOp(dropCommandOpId)); + assert.commandWorked(db.fsyncUnlock()); + awaitDropCommand(); + + // Ensure that the collection has been dropped. + assert.eq(-1, + db.getCollectionNames().indexOf(collectionName), + "Expected collection to not appear in listCollections output after being dropped"); +}()); diff --git a/jstests/core/list_collections1.js b/jstests/core/list_collections1.js index 04acb82290b..897ebfa1ca9 100644 --- a/jstests/core/list_collections1.js +++ b/jstests/core/list_collections1.js @@ -277,8 +277,7 @@ res = mydb.runCommand("listCollections", {cursor: {batchSize: 0}}); cursor = new DBCommandCursor(mydb.getMongo(), res, 2); - cursor = null; - gc(); // Shell will send a killCursors message when cleaning up underlying cursor. + cursor.close(); cursor = new DBCommandCursor(mydb.getMongo(), res, 2); assert.throws(function() { cursor.hasNext(); diff --git a/jstests/core/list_indexes.js b/jstests/core/list_indexes.js index 520406be59f..de0f4473980 100644 --- a/jstests/core/list_indexes.js +++ b/jstests/core/list_indexes.js @@ -164,8 +164,7 @@ res = coll.runCommand("listIndexes", {cursor: {batchSize: 0}}); cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); - cursor = null; - gc(); // Shell will send a killCursors message when cleaning up underlying cursor. + cursor.close(); cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); assert.throws(function() { cursor.hasNext(); diff --git a/jstests/core/list_indexes_invalidation.js b/jstests/core/list_indexes_invalidation.js new file mode 100644 index 00000000000..1a7770b75fb --- /dev/null +++ b/jstests/core/list_indexes_invalidation.js @@ -0,0 +1,37 @@ +// SERVER-24963/SERVER-27930 Missing invalidation for system.indexes writes +(function() { + 'use strict'; + var collName = 'system_indexes_invalidations'; + var collNameRenamed = 'renamed_collection'; + var coll = db[collName]; + var collRenamed = db[collNameRenamed]; + + function testIndexInvalidation(isRename) { + coll.drop(); + collRenamed.drop(); + assert.commandWorked(coll.createIndexes([{a: 1}, {b: 1}, {c: 1}])); + + // Get the first two indexes. Use find on 'system.indexes' on MMAPv1, listIndexes otherwise. + var cmd = db.system.indexes.count() ? {find: 'system.indexes'} : {listIndexes: collName}; + Object.extend(cmd, {batchSize: 2}); + var res = db.runCommand(cmd); + assert.commandWorked(res, 'could not run ' + tojson(cmd)); + printjson(res); + + // Ensure the cursor has data, rename or drop the collection, and exhaust the cursor. + var cursor = new DBCommandCursor(db.getMongo(), res); + var errMsg = + 'expected more data from command ' + tojson(cmd) + ', with result ' + tojson(res); + assert(cursor.hasNext(), errMsg); + if (isRename) { + assert.commandWorked(coll.renameCollection(collNameRenamed)); + } else { + assert(coll.drop()); + } + assert.gt(cursor.itcount(), 0, errMsg); + } + + // Test that we invalidate indexes for both collection drops and renames. + testIndexInvalidation(false); + testIndexInvalidation(true); +}()); diff --git a/jstests/core/list_namespaces_invalidation.js b/jstests/core/list_namespaces_invalidation.js new file mode 100644 index 00000000000..c257421b376 --- /dev/null +++ b/jstests/core/list_namespaces_invalidation.js @@ -0,0 +1,73 @@ +// SERVER-27996/SERVER-28022 Missing invalidation for system.namespaces writes +(function() { + 'use strict'; + var dbInvalidName = 'system_namespaces_invalidations'; + var dbInvalid = db.getSiblingDB(dbInvalidName); + var num_collections = 3; + var DROP = 1; + var RENAME = 2; + var MOVE = 3; + function testNamespaceInvalidation(namespaceAction, batchSize) { + dbInvalid.dropDatabase(); + + // Create enough collections to necessitate multiple cursor batches. + for (var i = 0; i < num_collections; i++) { + assert.commandWorked(dbInvalid.createCollection('coll' + i.toString())); + } + + // Get the first two namespaces. Use find on 'system.namespaces' on MMAPv1, listCollections + // otherwise. + var cmd = dbInvalid.system.indexes.count() ? {find: 'system.namespaces'} + : {listCollections: dbInvalidName}; + Object.extend(cmd, {batchSize: batchSize}); + var res = dbInvalid.runCommand(cmd); + assert.commandWorked(res, 'could not run ' + tojson(cmd)); + printjson(res); + + // Ensure the cursor has data, invalidate the namespace, and exhaust the cursor. + var cursor = new DBCommandCursor(dbInvalid.getMongo(), res); + var errMsg = + 'expected more data from command ' + tojson(cmd) + ', with result ' + tojson(res); + assert(cursor.hasNext(), errMsg); + if (namespaceAction == RENAME) { + // Rename the collection to something that does not fit in the previously allocated + // memory for the record. + assert.commandWorked( + dbInvalid['coll1'].renameCollection('coll1' + + 'lkdsahflaksjdhfsdkljhfskladhfkahfsakfla' + + 'skfjhaslfaslfkhasklfjhsakljhdsjksahkldjslh')); + } else if (namespaceAction == DROP) { + assert(dbInvalid['coll1'].drop()); + } else if (namespaceAction == MOVE) { + var modCmd = { + collMod: 'coll1', + validator: { + $or: [ + {phone: {$type: "string"}}, + {email: {$regex: /@mongodb\.com$/}}, + {status: {$in: ["Unknown", "Incomplete"]}}, + {address: {$type: "string"}}, + {ssn: {$type: "string"}}, + {favoriteBook: {$type: "string"}}, + {favoriteColor: {$type: "string"}}, + {favoriteBeverage: {$type: "string"}}, + {favoriteDay: {$type: "string"}}, + {favoriteFood: {$type: "string"}}, + {favoriteSport: {$type: "string"}}, + {favoriteMovie: {$type: "string"}}, + {favoriteShow: {$type: "string"}} + ] + } + }; + assert.commandWorked(dbInvalid.runCommand(modCmd)); + } + assert.gt(cursor.itcount(), 0, errMsg); + } + // Test that we invalidate the old namespace record ID when we remove, rename, or move a + // namespace record. + for (var j = 2; j < 7; j++) { + testNamespaceInvalidation(DROP, j); + testNamespaceInvalidation(RENAME, j); + testNamespaceInvalidation(MOVE, j); + } +}()); diff --git a/jstests/core/max_doc_size.js b/jstests/core/max_doc_size.js index 03deeafb307..11bdc66d398 100644 --- a/jstests/core/max_doc_size.js +++ b/jstests/core/max_doc_size.js @@ -1,11 +1,7 @@ var maxBsonObjectSize = db.isMaster().maxBsonObjectSize; var docOverhead = Object.bsonsize({_id: new ObjectId(), x: ''}); var maxStrSize = maxBsonObjectSize - docOverhead; - -var maxStr = 'a'; -while (maxStr.length < maxStrSize) - maxStr += 'a'; - +var maxStr = 'a'.repeat(maxStrSize); var coll = db.max_doc_size; coll.drop(); diff --git a/jstests/core/ref.js b/jstests/core/ref.js index 02c4cb92a07..63c6bf0b472 100644 --- a/jstests/core/ref.js +++ b/jstests/core/ref.js @@ -10,6 +10,11 @@ var other = { }; db.otherthings.save(other); +// Verify that the DBPointer prototype is not serializable +assert.throws(function() { + db.things.save({a: DBPointer.prototype}); +}); + db.things.save({name: "abc"}); x = db.things.findOne(); x.o = new DBPointer("otherthings", other._id); diff --git a/jstests/core/server22053.js b/jstests/core/server22053.js new file mode 100644 index 00000000000..5549dbc95ee --- /dev/null +++ b/jstests/core/server22053.js @@ -0,0 +1,19 @@ +(function() { + + "use strict"; + var t = db.jstests_server22053; + + /* eslint-disable no-sparse-arrays */ + var s0 = [, , 3, , , 6]; + t.coll.insert({mys: s0}); + + var cur = t.coll.find(); + var doc = cur.next(); + assert.eq(6, doc['mys'].length); + assert.eq(undefined, doc['mys'][0]); + assert.eq(undefined, doc['mys'][1]); + assert.eq(3, doc['mys'][2]); + assert.eq(undefined, doc['mys'][3]); + assert.eq(undefined, doc['mys'][4]); + assert.eq(6, doc['mys'][5]); +}()); diff --git a/jstests/core/server25192.js b/jstests/core/server25192.js new file mode 100644 index 00000000000..e07cfdcf50c --- /dev/null +++ b/jstests/core/server25192.js @@ -0,0 +1,12 @@ +(function() { + + "use strict"; + + var x = {}; + + assert.doesNotThrow(function() { + Object.extend(x, {a: null}, true); + }, [], "Extending an object with a null field does not throw"); + + assert.eq(x.a, null); +}()); diff --git a/jstests/core/tailable_skip_limit.js b/jstests/core/tailable_skip_limit.js index 2f3dbd19aea..7ed0b8ae7c2 100644 --- a/jstests/core/tailable_skip_limit.js +++ b/jstests/core/tailable_skip_limit.js @@ -79,4 +79,11 @@ assert.eq(t.find().addOption(2).itcount(), 0); assert.writeOK(t.insert({a: 1})); assert.eq(t.find().addOption(2).itcount(), 1); + + // Tests that find with 'awaitData' option but without 'tailable' option results in an + // assertion. + assert.throws(function() { + var cursor = t.find().addOption(DBQuery.Option.awaitData); + cursor.next(); + }); })(); diff --git a/jstests/libs/check_log.js b/jstests/libs/check_log.js new file mode 100644 index 00000000000..edfd8b1d5cf --- /dev/null +++ b/jstests/libs/check_log.js @@ -0,0 +1,84 @@ +/* + * Helper functions which connect to a server, and check its logs for particular strings. + */ +var checkLog; + +(function() { + "use strict"; + + if (checkLog) { + return; // Protect against this file being double-loaded. + } + + checkLog = (function() { + var getGlobalLog = function(conn) { + var cmdRes; + try { + cmdRes = conn.adminCommand({getLog: 'global'}); + } catch (e) { + // Retry with network errors. + print("checkLog ignoring failure: " + e); + return null; + } + + return assert.commandWorked(cmdRes).log; + }; + + /* + * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until + * the provided 'msg' is found in the logs, or 5 minutes have elapsed. Throws an exception + * on timeout. + */ + var contains = function(conn, msg) { + assert.soon( + function() { + var logMessages = getGlobalLog(conn); + if (logMessages === null) { + return false; + } + for (var i = 0; i < logMessages.length; i++) { + if (logMessages[i].indexOf(msg) != -1) { + return true; + } + } + return false; + }, + 'Could not find log entries containing the following message: ' + msg, + 5 * 60 * 1000, + 300); + }; + + /* + * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until + * the provided 'msg' is found in the logs exactly 'expectedCount' times, or 5 minutes have + * elapsed. + * Throws an exception on timeout. + */ + var containsWithCount = function(conn, msg, expectedCount) { + var count = 0; + assert.soon( + function() { + var logMessages = getGlobalLog(conn); + if (logMessages === null) { + return false; + } + for (var i = 0; i < logMessages.length; i++) { + if (logMessages[i].indexOf(msg) != -1) { + count++; + } + } + + return expectedCount === count; + }, + 'Expected ' + expectedCount + ', but instead saw ' + count + + ' log entries containing the following message: ' + msg, + 5 * 60 * 1000, + 300); + }; + + return { + contains: contains, + containsWithCount: containsWithCount + }; + })(); +})(); diff --git a/jstests/libs/chunk_manipulation_util.js b/jstests/libs/chunk_manipulation_util.js index ab56b061c9a..79359eb7e10 100644 --- a/jstests/libs/chunk_manipulation_util.js +++ b/jstests/libs/chunk_manipulation_util.js @@ -101,7 +101,8 @@ function configureMoveChunkFailPoint(shardConnection, stepNumber, mode) { assert(stepNumber >= 1); assert(stepNumber <= 6); var admin = shardConnection.getDB('admin'); - admin.runCommand({configureFailPoint: 'moveChunkHangAtStep' + stepNumber, mode: mode}); + assert.commandWorked( + admin.runCommand({configureFailPoint: 'moveChunkHangAtStep' + stepNumber, mode: mode})); } // @@ -166,7 +167,8 @@ function configureMigrateFailPoint(shardConnection, stepNumber, mode) { assert(stepNumber >= 1); assert(stepNumber <= 5); var admin = shardConnection.getDB('admin'); - admin.runCommand({configureFailPoint: 'migrateThreadHangAtStep' + stepNumber, mode: mode}); + assert.commandWorked(admin.runCommand( + {configureFailPoint: 'migrateThreadHangAtStep' + stepNumber, mode: mode})); } // diff --git a/jstests/libs/election_timing_test.js b/jstests/libs/election_timing_test.js deleted file mode 100644 index d9be493c381..00000000000 --- a/jstests/libs/election_timing_test.js +++ /dev/null @@ -1,227 +0,0 @@ -/** - * ElectionTimingTest - set up a ReplSetTest and use default or provided functions to - * trigger an election. The time it takes to discover a new primary is recorded. - */ -var ElectionTimingTest = function(opts) { - // How many times do we start a new ReplSetTest. - this.testRuns = opts.testRuns || 1; - - // How many times do we step down during a ReplSetTest"s lifetime. - this.testCycles = opts.testCycles || 1; - - // The config is set to two electable nodes since we use waitForMemberState - // to wait for the electable secondary to become primary. - this.nodes = opts.nodes || [{}, {}, {rsConfig: {arbiterOnly: true}}]; - - // The name of the replica set and of the collection. - this.name = opts.name || "election_timing"; - - // Pass additional replicaSet config options. - this.settings = opts.settings || {}; - - // pv1 is the default in master and here. - this.protocolVersion = opts.hasOwnProperty("protocolVersion") ? opts.protocolVersion : 1; - - // A function that runs after the ReplSetTest is initialized. - this.testSetup = opts.testSetup || Function.prototype; - - // A function that triggers election, default is to kill the mongod process. - this.electionTrigger = opts.electionTrigger || this.stopPrimary; - - // A function that waits for new primary to be elected. - this.waitForNewPrimary = opts.waitForNewPrimary || this.waitForNewPrimary; - - // A function that cleans up after the election trigger. - this.testReset = opts.testReset || this.stopPrimaryReset; - - // The interval passed to stepdown that primaries may not seek re-election. - // We also have to wait out this interval before allowing another stepdown. - this.stepDownGuardTime = opts.stepDownGuardTime || 60; - - // Test results will be stored in these arrays. - this.testResults = []; - this.testErrors = []; - - this._runTimingTest(); -}; - -ElectionTimingTest.prototype._runTimingTest = function() { - for (var run = 0; run < this.testRuns; run++) { - var collectionName = "test." + this.name; - var cycleData = { - testRun: run, - results: [] - }; - - jsTestLog("Starting ReplSetTest for test " + this.name + " run: " + run); - this.rst = - new ReplSetTest({name: this.name, nodes: this.nodes, nodeOptions: {verbose: ""}}); - this.rst.startSet(); - - // Get the replset config and apply the settings object. - var conf = this.rst.getReplSetConfig(); - conf.settings = conf.settings || {}; - conf.settings = Object.merge(conf.settings, this.settings); - - // Explicitly setting protocolVersion. - conf.protocolVersion = this.protocolVersion; - this.rst.initiate(conf); - - // Run the user supplied testSetup() method. Typical uses would be to set up - // bridging, or wait for a particular state after initiate(). - try { - this.testSetup(); - } catch (e) { - // If testSetup() fails, we are in an unknown state, log and return. - this.testErrors.push({testRun: run, status: "testSetup() failed", error: e}); - this.rst.stopSet(); - return; - } - - // Create and populate a collection. - var primary = this.rst.getPrimary(); - - this.electionTimeoutLimitMillis = - ElectionTimingTest.calculateElectionTimeoutLimitMillis(primary); - jsTestLog('Election timeout limit: ' + this.electionTimeoutLimitMillis + ' ms'); - - var coll = primary.getCollection(collectionName); - for (var i = 0; i < 100; i++) { - assert.writeOK(coll.insert({_id: i, x: i * 3, arbitraryStr: "this is a string"})); - } - - // Run the election tests on this ReplSetTest instance. - var secondary; - for (var cycle = 0; cycle < this.testCycles; cycle++) { - // Wait for replication. - this.rst.awaitSecondaryNodes(); - this.rst.awaitReplication(); - primary = this.rst.getPrimary(); - secondary = this.rst.getSecondary(); - - jsTestLog("Starting test: " + this.name + " run: " + run + " cycle: " + cycle); - var isMasterResult = primary.getDB("admin").isMaster(); - assert.commandWorked(isMasterResult, "isMaster() failed"); - var oldElectionId = isMasterResult.electionId; - assert.neq(undefined, oldElectionId, "isMaster() failed to return a valid electionId"); - - // Time the new election. - var stepDownTime = Date.now(); - - // Run the specified election trigger method. Default is to sigstop the primary. - try { - this.electionTrigger(); - } catch (e) { - // Left empty on purpose. - } - - // Wait for the electable secondary to become primary. - try { - this.waitForNewPrimary(this.rst, secondary); - } catch (e) { - // If we didn"t find a primary, save the error, break so this - // ReplSetTest is stopped. We can"t continue from a flaky state. - this.testErrors.push( - {testRun: run, cycle: cycle, status: "new primary not elected", error: e}); - break; - } - - var electionCompleteTime = Date.now(); - - // Verify we had an election and we have a new primary. - var newPrimary = this.rst.getPrimary(); - isMasterResult = newPrimary.getDB("admin").isMaster(); - assert.commandWorked(isMasterResult, "isMaster() failed"); - var newElectionId = isMasterResult.electionId; - assert.neq(undefined, newElectionId, "isMaster() failed to return a valid electionId"); - - if (bsonWoCompare(oldElectionId, newElectionId) !== 0) { - this.testErrors.push({ - testRun: run, - cycle: cycle, - status: "electionId not changed, no election was triggered" - }); - break; - } - - if (primary.host === newPrimary.host) { - this.testErrors.push( - {testRun: run, cycle: cycle, status: "Previous primary was re-elected"}); - break; - } - - cycleData.results.push((electionCompleteTime - stepDownTime) / 1000); - - // If we are running another test on this ReplSetTest, call the reset function. - if (cycle + 1 < this.testCycles) { - try { - this.testReset(); - } catch (e) { - this.testErrors.push( - {testRun: run, cycle: cycle, status: "testReset() failed", error: e}); - break; - } - } - } - this.testResults.push(cycleData); - this.rst.stopSet(); - } -}; - -ElectionTimingTest.prototype.stopPrimary = function() { - this.originalPrimary = this.rst.getNodeId(this.rst.getPrimary()); - this.rst.stop(this.originalPrimary); -}; - -ElectionTimingTest.prototype.stopPrimaryReset = function() { - this.rst.restart(this.originalPrimary); -}; - -ElectionTimingTest.prototype.stepDownPrimary = function() { - var adminDB = this.rst.getPrimary().getDB("admin"); - adminDB.runCommand({replSetStepDown: this.stepDownGuardTime, force: true}); -}; - -ElectionTimingTest.prototype.stepDownPrimaryReset = function() { - sleep(this.stepDownGuardTime * 1000); -}; - -ElectionTimingTest.prototype.waitForNewPrimary = function(rst, secondary) { - assert.commandWorked(secondary.adminCommand({ - replSetTest: 1, - waitForMemberState: ReplSetTest.State.PRIMARY, - timeoutMillis: 60 * 1000 - }), - "node " + secondary.host + " failed to become primary"); -}; - -/** - * Calculates upper limit for actual failover time in milliseconds. - */ -ElectionTimingTest.calculateElectionTimeoutLimitMillis = function(primary) { - var configResult = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})); - var config = configResult.config; - // Protocol version is 0 if missing from config. - var protocolVersion = config.hasOwnProperty("protocolVersion") ? config.protocolVersion : 0; - var electionTimeoutMillis = 0; - var electionTimeoutOffsetLimitFraction = 0; - if (protocolVersion === 0) { - electionTimeoutMillis = 30000; // from TopologyCoordinatorImpl::VoteLease::leaseTime - electionTimeoutOffsetLimitFraction = 0; - } else { - electionTimeoutMillis = config.settings.electionTimeoutMillis; - var getParameterResult = assert.commandWorked(primary.adminCommand({ - getParameter: 1, - replElectionTimeoutOffsetLimitFraction: 1, - })); - electionTimeoutOffsetLimitFraction = - getParameterResult.replElectionTimeoutOffsetLimitFraction; - } - var assertSoonIntervalMillis = 200; // from assert.js - var applierDrainWaitMillis = 1000; // from SyncTail::tryPopAndWaitForMore() - var padding = 5000; // extra padding for slow hosts - var electionTimeoutLimitMillis = - (1 + electionTimeoutOffsetLimitFraction) * electionTimeoutMillis + applierDrainWaitMillis + - assertSoonIntervalMillis + padding; - return electionTimeoutLimitMillis; -}; diff --git a/jstests/libs/parallelTester.js b/jstests/libs/parallelTester.js index b043e3de78d..ed55078934b 100644 --- a/jstests/libs/parallelTester.js +++ b/jstests/libs/parallelTester.js @@ -139,7 +139,6 @@ if (typeof _threadInject != "undefined") { "killop.js", "run_program1.js", "notablescan.js", - "drop2.js", "dropdb_race.js", "fsync2.js", // May be placed in serialTestsArr once SERVER-4243 is fixed. "bench_test1.js", @@ -149,7 +148,6 @@ if (typeof _threadInject != "undefined") { // this has a chance to see the message "connections_opened.js", // counts connections, globally "opcounters_write_cmd.js", - "currentop.js", // SERVER-8673, plus rwlock yielding issues "set_param1.js", // changes global state "geo_update_btree2.js", // SERVER-11132 test disables table scans "update_setOnInsert.js", // SERVER-9982 @@ -168,8 +166,10 @@ if (typeof _threadInject != "undefined") { // some tests can't be run in parallel with each other var serialTestsArr = [ + // These tests use fsyncLock. parallelFilesDir + "/fsync.js", - parallelFilesDir + "/auth1.js", + parallelFilesDir + "/currentop.js", + parallelFilesDir + "/killop_drop_collection.js", // These tests expect the profiler to be on or off at specific points // during the test run. diff --git a/jstests/libs/test_background_ops.js b/jstests/libs/test_background_ops.js index ce21a636ff6..f30314234da 100644 --- a/jstests/libs/test_background_ops.js +++ b/jstests/libs/test_background_ops.js @@ -84,6 +84,10 @@ var getResult = function(mongo, name) { * Overrides the parallel shell code in mongo */ function startParallelShell(jsCode, port) { + if (TestData) { + jsCode = "TestData = " + tojson(TestData) + ";" + jsCode; + } + var x; if (port) { x = startMongoProgramNoConnect("mongo", "--port", port, "--eval", jsCode); diff --git a/jstests/libs/write_concern_util.js b/jstests/libs/write_concern_util.js new file mode 100644 index 00000000000..526acb9b6b5 --- /dev/null +++ b/jstests/libs/write_concern_util.js @@ -0,0 +1,73 @@ +/** + * Utilities for testing writeConcern. + */ + +load("jstests/libs/check_log.js"); + +// Stops replication on the given server(s). +function stopServerReplication(conn) { + if (conn.length) { + conn.forEach(function(n) { + stopServerReplication(n); + }); + return; + } + + // Clear ramlog so checkLog can't find log messages from previous times this fail point was + // enabled. + assert.commandWorked(conn.adminCommand({clearLog: 'global'})); + var errMsg = 'Failed to enable stopReplProducer failpoint.'; + assert.commandWorked( + conn.adminCommand({configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'}), errMsg); + + // Wait until the fail point is actually hit. + checkLog.contains(conn, 'bgsync - stopReplProducer fail point enabled'); +} + +// Stops replication at all replicaset secondaries. +function stopReplicationOnSecondaries(rs) { + stopServerReplication(rs.getSecondaries()); +} + +// Stops replication at all shard secondaries. +function stopReplicationOnSecondariesOfAllShards(st) { + st._rsObjects.forEach(stopReplicationOnSecondaries); +} + +// Restarts replication on the given server(s). +function restartServerReplication(conn) { + if (conn.length) { + conn.forEach(function(n) { + restartServerReplication(n); + }); + return; + } + + var errMsg = 'Failed to disable stopReplProducer failpoint.'; + assert.commandWorked( + conn.getDB('admin').runCommand({configureFailPoint: 'stopReplProducer', mode: 'off'}), + errMsg); +} + +// Restarts replication at all nodes in a replicaset. +function restartReplSetReplication(rs) { + restartServerReplication(rs.nodes); +} + +// Restarts replication at all replicaset secondaries. +function restartReplicationOnSecondaries(rs) { + restartServerReplication(rs.getSecondaries()); +} + +// Restarts replication at all nodes in a sharded cluster. +function restartReplicationOnAllShards(st) { + st._rsObjects.forEach(restartReplSetReplication); + restartReplSetReplication(st.configRS); +} + +// Asserts that a writeConcernError was received. +function assertWriteConcernError(res) { + assert(res.writeConcernError, "No writeConcernError received, got: " + tojson(res)); + assert(res.writeConcernError.code); + assert(res.writeConcernError.errmsg); +} diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index 109db580453..673be42d3df 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -45,6 +45,7 @@ ReplSetTest.prototype.upgradeNode = function(node, opts, user, pwd) { if (user != undefined) { assert.eq(1, node.getDB("admin").auth(user, pwd)); } + jsTest.authenticate(node); var isMaster = node.getDB('admin').runCommand({isMaster: 1}); @@ -71,7 +72,7 @@ ReplSetTest.prototype.stepdown = function(nodeId) { var node = this.nodes[nodeId]; try { - node.getDB("admin").runCommand({replSetStepDown: 50, force: true}); + node.getDB("admin").runCommand({replSetStepDown: 300, secondaryCatchUpPeriodSecs: 60}); assert(false); } catch (ex) { print('Caught exception after stepDown cmd: ' + tojson(ex)); diff --git a/jstests/multiVersion/minor_version_tags_new_old_new.js b/jstests/multiVersion/minor_version_tags_new_old_new.js index f39b3da4c68..7adb4d868a9 100644 --- a/jstests/multiVersion/minor_version_tags_new_old_new.js +++ b/jstests/multiVersion/minor_version_tags_new_old_new.js @@ -1,271 +1,17 @@ - (function() { 'use strict'; + load("jstests/replsets/libs/tags.js"); + // 3.2.1 is the final version to use the old style replSetUpdatePosition command. var oldVersion = "3.2.1"; var newVersion = "latest"; - var nodes = { - n1: {binVersion: newVersion}, - n2: {binVersion: oldVersion}, - n3: {binVersion: newVersion}, - n4: {binVersion: oldVersion}, - n5: {binVersion: newVersion} - }; - var host = getHostName(); - var name = 'tags'; - - var replTest = new ReplSetTest({name: name, nodes: nodes, useBridge: true}); - var nodes = replTest.nodeList(); - var conns = replTest.startSet(); - var port = replTest.ports; - replTest.initiate({ - _id: name, - members: [ - { - _id: 0, - host: nodes[0], - tags: { - server: '0', - dc: 'ny', - ny: '1', - rack: 'ny.rk1', - }, - }, - { - _id: 1, - host: nodes[1], - priority: 2, - tags: { - server: '1', - dc: 'ny', - ny: '2', - rack: 'ny.rk1', - }, - }, - { - _id: 2, - host: nodes[2], - priority: 3, - tags: { - server: '2', - dc: 'ny', - ny: '3', - rack: 'ny.rk2', 2: 'this', - }, - }, - { - _id: 3, - host: nodes[3], - tags: { - server: '3', - dc: 'sf', - sf: '1', - rack: 'sf.rk1', - }, - }, - { - _id: 4, - host: nodes[4], - tags: { - server: '4', - dc: 'sf', - sf: '2', - rack: 'sf.rk2', - }, - }, - ], - settings: { - getLastErrorModes: { - '2 dc and 3 server': { - dc: 2, - server: 3, - }, - '1 and 2': { - 2: 1, - server: 1, - }, - '2': { - 2: 1, - }, - '3 and 4': { - sf: 2, - }, - '3 or 4': { - sf: 1, - }, - }, - }, - }); - - replTest.awaitReplication(); - - // Create collection to guard against timeouts due to file allocation. - assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); - replTest.awaitReplication(); - - var ensurePrimary = function(nodeId, expectedWritableNodes) { - jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.'); - replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000); - primary = replTest.getPrimary(); - primary.forceWriteMode('commands'); - var writeConcern = { - writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000} - }; - assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); - return primary; - }; - - // 2 should eventually stage a priority takeover from the primary. - var primary = ensurePrimary(2, 3); - - jsTestLog('primary is now 2'); - var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - jsTestLog('test configuration = ' + tojson(config)); - - jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); - conns[0].disconnect(conns[3]); - conns[0].disconnect(conns[4]); - conns[1].disconnect(conns[3]); - conns[1].disconnect(conns[4]); - conns[2].disconnect(conns[3]); - conns[2].disconnect(conns[4]); - conns[3].disconnect(conns[4]); - jsTestLog('Done setting up partitions'); - - jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); - jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); - - var doc = { - x: 1 - }; - - // This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs. - // Some writes are expected to block for this 'timeout' duration before failing. - // Depending on the order of heartbeats (containing last committed op time) received - // by a node, it might hang up on its sync source. This may cause some of the write concern - // tests to fail. - var timeout = 20 * 1000; - - jsTestLog('test1'); - primary = ensurePrimary(2, 3); - - jsTestLog('Non-existent write concern should be rejected.'); - options = { - writeConcern: {w: 'blahblah', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert.eq(ErrorCodes.UnknownReplWriteConcern, - result.getWriteConcernError().code, - tojson(result.getWriteConcernError())); - - jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = primary.getDB('foo').bar.insert(doc, options); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - conns[1].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3] ' + - '(all nodes besides node 3 can replicate from primary node 2)'); - primary = ensurePrimary(2, 4); - - jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + - primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError())); - - conns[3].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + - '(all secondaries can replicate from primary node 2)'); - primary = ensurePrimary(2, 5); - - jsTestLog('31004 should sync from 31001 (31026)'); - jsTestLog('31003 should sync from 31004 (31024)'); - jsTestLog('Write concern "3 and 4" should work - ' + - 'nodes 3 and 4 are connected to primary via node 1.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" - writes to primary only.'); - options = { - writeConcern: {w: '2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "1 and 2"'); - options = { - writeConcern: {w: '1 and 2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2 dc and 3 server"'); - primary = ensurePrimary(2, 5); - options = { - writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Bringing down current primary node 2 ' + primary.host + - ' to allow next higher priority node 1 ' + replTest.nodes[1].host + - ' to become primary.'); - - // Is this necessary since 3 will be connected to the new primary via node 4? - conns[1].reconnect(conns[3]); - - conns[2].disconnect(conns[0]); - conns[2].disconnect(conns[1]); - - // Is this necessary when we partition node 2 off from the rest of the nodes? - replTest.stop(2); - jsTestLog('partitions: [0-1] [2] [1-3-4] ' + - '(all secondaries except down node 2 can replicate from new primary node 1)'); - - // Node 1 with slightly higher priority will take over. - jsTestLog('1 must become primary here because otherwise the other members will take too ' + - 'long timing out their old sync threads'); - primary = ensurePrimary(1, 4); - - jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + - ' is down.'); - options = { - writeConcern: {w: '2', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - replTest.stopSet(); - jsTestLog('tags.js SUCCESS'); + var nodes = [ + {binVersion: newVersion}, + {binVersion: oldVersion}, + {binVersion: newVersion}, + {binVersion: oldVersion}, + {binVersion: newVersion} + ]; + new TagsTest({nodes: nodes, forceWriteMode: 'commands'}).run(); }()); diff --git a/jstests/multiVersion/minor_version_tags_old_new_old.js b/jstests/multiVersion/minor_version_tags_old_new_old.js index 056aab2972a..2c4eb59c245 100644 --- a/jstests/multiVersion/minor_version_tags_old_new_old.js +++ b/jstests/multiVersion/minor_version_tags_old_new_old.js @@ -1,271 +1,17 @@ - (function() { 'use strict'; + load("jstests/replsets/libs/tags.js"); + // 3.2.1 is the final version to use the old style replSetUpdatePosition command. var oldVersion = "3.2.1"; var newVersion = "latest"; - var nodes = { - n1: {binVersion: oldVersion}, - n2: {binVersion: newVersion}, - n3: {binVersion: oldVersion}, - n4: {binVersion: newVersion}, - n5: {binVersion: oldVersion} - }; - var host = getHostName(); - var name = 'tags'; - - var replTest = new ReplSetTest({name: name, nodes: nodes, useBridge: true}); - var nodes = replTest.nodeList(); - var conns = replTest.startSet(); - var port = replTest.ports; - replTest.initiate({ - _id: name, - members: [ - { - _id: 0, - host: nodes[0], - tags: { - server: '0', - dc: 'ny', - ny: '1', - rack: 'ny.rk1', - }, - }, - { - _id: 1, - host: nodes[1], - priority: 2, - tags: { - server: '1', - dc: 'ny', - ny: '2', - rack: 'ny.rk1', - }, - }, - { - _id: 2, - host: nodes[2], - priority: 3, - tags: { - server: '2', - dc: 'ny', - ny: '3', - rack: 'ny.rk2', 2: 'this', - }, - }, - { - _id: 3, - host: nodes[3], - tags: { - server: '3', - dc: 'sf', - sf: '1', - rack: 'sf.rk1', - }, - }, - { - _id: 4, - host: nodes[4], - tags: { - server: '4', - dc: 'sf', - sf: '2', - rack: 'sf.rk2', - }, - }, - ], - settings: { - getLastErrorModes: { - '2 dc and 3 server': { - dc: 2, - server: 3, - }, - '1 and 2': { - 2: 1, - server: 1, - }, - '2': { - 2: 1, - }, - '3 and 4': { - sf: 2, - }, - '3 or 4': { - sf: 1, - }, - }, - }, - }); - - replTest.awaitReplication(); - - // Create collection to guard against timeouts due to file allocation. - assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); - replTest.awaitReplication(); - - var ensurePrimary = function(nodeId, expectedWritableNodes) { - jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.'); - replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000); - primary = replTest.getPrimary(); - primary.forceWriteMode('commands'); - var writeConcern = { - writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000} - }; - assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); - return primary; - }; - - // 2 should eventually stage a priority takeover from the primary. - var primary = ensurePrimary(2, 3); - - jsTestLog('primary is now 2'); - var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - jsTestLog('test configuration = ' + tojson(config)); - - jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); - conns[0].disconnect(conns[3]); - conns[0].disconnect(conns[4]); - conns[1].disconnect(conns[3]); - conns[1].disconnect(conns[4]); - conns[2].disconnect(conns[3]); - conns[2].disconnect(conns[4]); - conns[3].disconnect(conns[4]); - jsTestLog('Done setting up partitions'); - - jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); - jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); - - var doc = { - x: 1 - }; - - // This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs. - // Some writes are expected to block for this 'timeout' duration before failing. - // Depending on the order of heartbeats (containing last committed op time) received - // by a node, it might hang up on its sync source. This may cause some of the write concern - // tests to fail. - var timeout = 20 * 1000; - - jsTestLog('test1'); - primary = ensurePrimary(2, 3); - - jsTestLog('Non-existent write concern should be rejected.'); - options = { - writeConcern: {w: 'blahblah', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert.eq(ErrorCodes.UnknownReplWriteConcern, - result.getWriteConcernError().code, - tojson(result.getWriteConcernError())); - - jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = primary.getDB('foo').bar.insert(doc, options); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - conns[1].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3] ' + - '(all nodes besides node 3 can replicate from primary node 2)'); - primary = ensurePrimary(2, 4); - - jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + - primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError())); - - conns[3].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + - '(all secondaries can replicate from primary node 2)'); - primary = ensurePrimary(2, 5); - - jsTestLog('31004 should sync from 31001 (31026)'); - jsTestLog('31003 should sync from 31004 (31024)'); - jsTestLog('Write concern "3 and 4" should work - ' + - 'nodes 3 and 4 are connected to primary via node 1.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" - writes to primary only.'); - options = { - writeConcern: {w: '2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "1 and 2"'); - options = { - writeConcern: {w: '1 and 2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2 dc and 3 server"'); - primary = ensurePrimary(2, 5); - options = { - writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Bringing down current primary node 2 ' + primary.host + - ' to allow next higher priority node 1 ' + replTest.nodes[1].host + - ' to become primary.'); - - // Is this necessary since 3 will be connected to the new primary via node 4? - conns[1].reconnect(conns[3]); - - conns[2].disconnect(conns[0]); - conns[2].disconnect(conns[1]); - - // Is this necessary when we partition node 2 off from the rest of the nodes? - replTest.stop(2); - jsTestLog('partitions: [0-1] [2] [1-3-4] ' + - '(all secondaries except down node 2 can replicate from new primary node 1)'); - - // Node 1 with slightly higher priority will take over. - jsTestLog('1 must become primary here because otherwise the other members will take too ' + - 'long timing out their old sync threads'); - primary = ensurePrimary(1, 4); - - jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + - ' is down.'); - options = { - writeConcern: {w: '2', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - replTest.stopSet(); - jsTestLog('tags.js SUCCESS'); + var nodes = [ + {binVersion: oldVersion}, + {binVersion: newVersion}, + {binVersion: oldVersion}, + {binVersion: newVersion}, + {binVersion: oldVersion} + ]; + new TagsTest({nodes: nodes, forceWriteMode: 'commands'}).run(); }()); diff --git a/jstests/noPassthrough/command_line_parsing.js b/jstests/noPassthrough/command_line_parsing.js index d2e02bb9865..06246e72084 100644 --- a/jstests/noPassthrough/command_line_parsing.js +++ b/jstests/noPassthrough/command_line_parsing.js @@ -25,7 +25,7 @@ var m2expected = { var m2result = m2.getDB("admin").runCommand("getCmdLineOpts"); // remove variables that depend on the way the test is started. -delete m2result.parsed.nopreallocj; +delete m2result.parsed.storage.mmapv1; delete m2result.parsed.setParameter; delete m2result.parsed.storage.engine; delete m2result.parsed.storage.journal; @@ -49,7 +49,7 @@ var m3expected = { var m3result = m3.getDB("admin").runCommand("getCmdLineOpts"); // remove variables that depend on the way the test is started. -delete m3result.parsed.nopreallocj; +delete m3result.parsed.storage.mmapv1; delete m3result.parsed.setParameter; delete m3result.parsed.storage.engine; delete m3result.parsed.storage.journal; diff --git a/jstests/noPassthrough/indexbg1.js b/jstests/noPassthrough/indexbg1.js index 62e3617fe9f..00670f3f2db 100644 --- a/jstests/noPassthrough/indexbg1.js +++ b/jstests/noPassthrough/indexbg1.js @@ -14,13 +14,15 @@ var resetParallel = function() { parallel().drop(); }; +// Return the PID to call `waitpid` on for clean shutdown. var doParallel = function(work) { resetParallel(); print("doParallel: " + work); - startMongoProgramNoConnect("mongo", - "--eval", - work + "; db." + baseName + "_parallelStatus.save( {done:1} );", - db.getMongo().host); + return startMongoProgramNoConnect( + "mongo", + "--eval", + work + "; db." + baseName + "_parallelStatus.save( {done:1} );", + db.getMongo().host); }; var doneParallel = function() { @@ -34,6 +36,7 @@ var waitParallel = function() { }; var size = 400 * 1000; +var bgIndexBuildPid; while (1) { // if indexing finishes before we can run checks, try indexing w/ more data print("size: " + size); @@ -48,7 +51,7 @@ while (1) { // if indexing finishes before we can run checks, try indexing w/ m assert.writeOK(bulk.execute()); assert.eq(size, t.count()); - doParallel(fullName + ".ensureIndex( {i:1}, {background:true} )"); + bgIndexBuildPid = doParallel(fullName + ".ensureIndex( {i:1}, {background:true} )"); try { // wait for indexing to start print("wait for indexing to start"); @@ -98,12 +101,17 @@ while (1) { // if indexing finishes before we can run checks, try indexing w/ m break; } print("indexing finished too soon, retrying..."); + // Although the index build finished, ensure the shell has exited. + waitProgram(bgIndexBuildPid); size *= 2; assert(size < 200000000, "unable to run checks in parallel with index creation"); } print("our tests done, waiting for parallel to finish"); waitParallel(); +// Ensure the shell has exited cleanly. Otherwise the test harness may send a SIGTERM which can lead +// to a false test failure. +waitProgram(bgIndexBuildPid); print("finished"); assert.eq(1, t.count({i: -10})); diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js index 5fcd975a98f..e9ac45c8b78 100644 --- a/jstests/noPassthrough/indexbg2.js +++ b/jstests/noPassthrough/indexbg2.js @@ -14,12 +14,14 @@ var resetParallel = function() { parallel().drop(); }; +// Return the PID to call `waitpid` on for clean shutdown. var doParallel = function(work) { resetParallel(); - startMongoProgramNoConnect("mongo", - "--eval", - work + "; db." + baseName + "_parallelStatus.save( {done:1} );", - db.getMongo().host); + return startMongoProgramNoConnect( + "mongo", + "--eval", + work + "; db." + baseName + "_parallelStatus.save( {done:1} );", + db.getMongo().host); }; var doneParallel = function() { @@ -35,6 +37,7 @@ var waitParallel = function() { var doTest = function() { "use strict"; var size = 10000; + var bgIndexBuildPid; while (1) { // if indexing finishes before we can run checks, try indexing w/ more data print("size: " + size); var fullName = "db." + baseName; @@ -46,7 +49,8 @@ var doTest = function() { } assert.eq(size, t.count()); - doParallel(fullName + ".ensureIndex( {i:1}, {background:true, unique:true} )"); + bgIndexBuildPid = + doParallel(fullName + ".ensureIndex( {i:1}, {background:true, unique:true} )"); try { // wait for indexing to start assert.soon(function() { @@ -59,13 +63,19 @@ var doTest = function() { // wait for parallel status to update to reflect indexing status sleep(1000); if (!doneParallel()) { + waitProgram(bgIndexBuildPid); throw e; } } if (!doneParallel()) { + // Ensure the shell has exited cleanly. Otherwise the test harness may send a SIGTERM + // which can lead to a false test failure. + waitProgram(bgIndexBuildPid); break; } print("indexing finished too soon, retrying..."); + // Although the index build finished, ensure the shell has exited. + waitProgram(bgIndexBuildPid); size *= 2; assert(size < 5000000, "unable to run checks in parallel with index creation"); } diff --git a/jstests/noPassthrough/read_majority_reads.js b/jstests/noPassthrough/read_majority_reads.js new file mode 100644 index 00000000000..5c169e3bed7 --- /dev/null +++ b/jstests/noPassthrough/read_majority_reads.js @@ -0,0 +1,270 @@ +/** + * Tests that read operations with readConcern majority only see committed data. + * + * The following read operations are tested: + * - find command + * - aggregation + * - distinct + * - group + * - count + * - parallelCollectionScan + * - geoNear + * - geoSearch + * + * Each operation is tested on a single node, and (if supported) through mongos on both sharded and + * unsharded collections. Mongos doesn't directly handle readConcern majority, but these tests + * should ensure that it correctly propagates the setting to the shards when running commands. + */ + +(function() { + 'use strict'; + + function makeCursor(mongo, result) { + return new DBCommandCursor(mongo, result); + } + + // These test cases are functions that return a cursor of the documents in collections without + // fetching them yet. + var cursorTestCases = { + find: function(coll) { + return makeCursor(coll.getMongo(), + assert.commandWorked(coll.runCommand( + 'find', {readConcern: {level: 'majority'}, batchSize: 0}))); + }, + aggregate: function(coll) { + return makeCursor( + coll.getMongo(), + assert.commandWorked(coll.runCommand( + 'aggregate', + {readConcern: {level: 'majority'}, cursor: {batchSize: 0}, pipeline: []}))); + }, + parallelCollectionScan: function(coll) { + var res = coll.runCommand('parallelCollectionScan', + {readConcern: {level: 'majority'}, numCursors: 1}); + assert.commandWorked(res); + assert.eq(res.cursors.length, 1, tojson(res)); + return makeCursor(coll.getMongo(), res.cursors[0]); + }, + }; + + // These test cases have a run method that will be passed a collection with a single object with + // _id: 1 and a state field that equals either "before" or "after". The collection will also + // contain both a 2dsphere and a geoHaystack index to enable testing commands that depend on + // them. The return value from the run method is expected to be the value of expectedBefore or + // expectedAfter depending on the state of the state field. + var nonCursorTestCases = { + count_before: { + run: function(coll) { + var res = coll.runCommand( + 'count', {readConcern: {level: 'majority'}, query: {state: 'before'}}); + assert.commandWorked(res); + return res.n; + }, + expectedBefore: 1, + expectedAfter: 0, + }, + count_after: { + run: function(coll) { + var res = coll.runCommand( + 'count', {readConcern: {level: 'majority'}, query: {state: 'after'}}); + assert.commandWorked(res); + return res.n; + }, + expectedBefore: 0, + expectedAfter: 1, + }, + distinct: { + run: function(coll) { + var res = + coll.runCommand('distinct', {readConcern: {level: 'majority'}, key: 'state'}); + assert.commandWorked(res); + assert.eq(res.values.length, 1, tojson(res)); + return res.values[0]; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + geoNear: { + run: function(coll) { + var res = coll.runCommand('geoNear', + { + readConcern: {level: 'majority'}, + near: [0, 0], + spherical: true, + }); + assert.commandWorked(res); + assert.eq(res.results.length, 1, tojson(res)); + return res.results[0].obj.state; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + geoSearch: { + run: function(coll) { + var res = coll.runCommand('geoSearch', + { + readConcern: {level: 'majority'}, + near: [0, 0], + search: {_id: 1}, // Needed due to SERVER-23158. + maxDistance: 1, + }); + assert.commandWorked(res); + assert.eq(res.results.length, 1, tojson(res)); + return res.results[0].state; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + group: { + run: function(coll) { + var res = coll.runCommand({ + 'group': { + ns: coll.getName(), + key: {_id: 1}, + initial: {}, + $reduce: function(curr, result) { + result.state = curr.state; + }, + }, + readConcern: {level: 'majority'}, + }); + assert.commandWorked(res); + assert.eq(res.retval.length, 1, tojson(res)); + return res.retval[0].state; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + }; + + function runTests(coll, mongodConnection) { + function makeSnapshot() { + return assert.commandWorked(mongodConnection.adminCommand("makeSnapshot")).name; + } + function setCommittedSnapshot(snapshot) { + assert.commandWorked(mongodConnection.adminCommand({"setCommittedSnapshot": snapshot})); + } + + for (var testName in cursorTestCases) { + jsTestLog('Running ' + testName + ' against ' + coll.toString()); + var getCursor = cursorTestCases[testName]; + + // Setup initial state. + assert.writeOK(coll.remove({})); + assert.writeOK(coll.save({_id: 1, state: 'before'})); + setCommittedSnapshot(makeSnapshot()); + + // Check initial conditions. + assert.eq(getCursor(coll).next().state, 'before'); + + // Change state without making it committed. + assert.writeOK(coll.save({_id: 1, state: 'after'})); + + // Cursor still sees old state. + assert.eq(getCursor(coll).next().state, 'before'); + + // Create a cursor before the update is visible. + var oldCursor = getCursor(coll); + + // Making a snapshot doesn't make the update visible yet. + var snapshot = makeSnapshot(); + assert.eq(getCursor(coll).next().state, 'before'); + + // Setting it as committed does for both new and old cursors. + setCommittedSnapshot(snapshot); + assert.eq(getCursor(coll).next().state, 'after'); + assert.eq(oldCursor.next().state, 'after'); + } + + assert.commandWorked(coll.ensureIndex({point: '2dsphere'})); + assert.commandWorked(coll.ensureIndex({point: 'geoHaystack', _id: 1}, {bucketSize: 1})); + for (var testName in nonCursorTestCases) { + jsTestLog('Running ' + testName + ' against ' + coll.toString()); + var getResult = nonCursorTestCases[testName].run; + var expectedBefore = nonCursorTestCases[testName].expectedBefore; + var expectedAfter = nonCursorTestCases[testName].expectedAfter; + + // Setup initial state. + assert.writeOK(coll.remove({})); + assert.writeOK(coll.save({_id: 1, state: 'before', point: [0, 0]})); + setCommittedSnapshot(makeSnapshot()); + + // Check initial conditions. + assert.eq(getResult(coll), expectedBefore); + + // Change state without making it committed. + assert.writeOK(coll.save({_id: 1, state: 'after', point: [0, 0]})); + + // Cursor still sees old state. + assert.eq(getResult(coll), expectedBefore); + + // Making a snapshot doesn't make the update visible yet. + var snapshot = makeSnapshot(); + assert.eq(getResult(coll), expectedBefore); + + // Setting it as committed does. + setCommittedSnapshot(snapshot); + assert.eq(getResult(coll), expectedAfter); + } + } + + var mongod = MongoRunner.runMongod({setParameter: 'testingSnapshotBehaviorInIsolation=true'}); + assert.neq( + null, + mongod, + 'mongod was unable to start with the testingSnapshotBehaviorInIsolation parameter enabled'); + + if (!mongod.adminCommand('serverStatus').storageEngine.supportsCommittedReads) { + print("Skipping read_majority_reads.js since storageEngine doesn't support it."); + return; + } + + (function testSingleNode() { + var db = mongod.getDB("singleNode"); + runTests(db.collection, mongod); + })(); + + var shardingTest = new ShardingTest({ + shards: 0, // We use the existing mongod. + mongos: 1, + }); + shardingTest.adminCommand({addShard: mongod.host}); + + // Remove tests of commands that aren't supported at all through mongos, even on unsharded + // collections. + ['parallelCollectionScan', 'geoSearch'].forEach(function(cmd) { + // Make sure it really isn't supported. + assert.eq(shardingTest.getDB('test').coll.runCommand(cmd).code, ErrorCodes.CommandNotFound); + delete cursorTestCases[cmd]; + delete nonCursorTestCases[cmd]; + }); + + (function testUnshardedDBThroughMongos() { + var db = shardingTest.getDB("throughMongos"); + runTests(db.unshardedDB, mongod); + })(); + + shardingTest.adminCommand({enableSharding: 'throughMongos'}); + + (function testUnshardedCollectionThroughMongos() { + var db = shardingTest.getDB("throughMongos"); + runTests(db.unshardedCollection, mongod); + })(); + + (function testShardedCollectionThroughMongos() { + var db = shardingTest.getDB("throughMongos"); + var collection = db.shardedCollection; + shardingTest.adminCommand({shardCollection: collection.getFullName(), key: {_id: 1}}); + + // The group command isn't supported on sharded collections. It also uses a weird syntax so + // code to check that it isn't supported can't be reused with other commands. + assert.eq(collection.runCommand({'group': {ns: collection.getName()}}).code, + ErrorCodes.IllegalOperation); + delete nonCursorTestCases.group; + + runTests(collection, mongod); + })(); + + shardingTest.stop(); + MongoRunner.stopMongod(mongod); +})(); diff --git a/jstests/noPassthrough/shell_history.js b/jstests/noPassthrough/shell_history.js new file mode 100644 index 00000000000..bbc8f9538af --- /dev/null +++ b/jstests/noPassthrough/shell_history.js @@ -0,0 +1,126 @@ +// Test that when running the shell for the first time creates the ~/.dbshell file, and it has +// appropriate permissions (where relevant). + +(function() { + "use strict"; + + // Use dataPath because it includes the trailing "/" or "\". + var tmpHome = MongoRunner.dataPath; + // Ensure it exists and is a dir (eg. if running without resmoke.py and /data/db doesn't exist). + mkdir(tmpHome); + removeFile(tmpHome + ".dbshell"); + + var args = []; + var cmdline = "mongo --nodb"; + var redirection = ""; + var env = {}; + if (_isWindows()) { + args.push("cmd.exe"); + args.push("/c"); + + // Input is set to NUL. The output must also be redirected to NUL, otherwise running the + // jstest manually has strange terminal IO behaviour. + redirection = "< NUL > NUL"; + + // USERPROFILE set to the tmp homedir. + // Since NUL is a character device, isatty() will return true, which means that .mongorc.js + // will be created in the HOMEDRIVE + HOMEPATH location, so we must set them also. + if (tmpHome.match("^[a-zA-Z]:")) { + var tmpHomeDrive = tmpHome.substr(0, 2); + var tmpHomePath = tmpHome.substr(2); + } else { + var _pwd = pwd(); + assert(_pwd.match("^[a-zA-Z]:"), "pwd must include drive"); + var tmpHomeDrive = _pwd.substr(0, 2); + var tmpHomePath = tmpHome; + } + env = { + USERPROFILE: tmpHome, + HOMEDRIVE: tmpHomeDrive, + HOMEPATH: tmpHomePath + }; + + } else { + args.push("sh"); + args.push("-c"); + + // Use the mongo shell from the current dir, same as resmoke.py does. + // Doesn't handle resmoke's --mongo= option. + cmdline = "./" + cmdline; + + // Set umask to 0 prior to running the shell. + cmdline = "umask 0 ; " + cmdline; + + // stdin is /dev/null. + redirection = "< /dev/null"; + + // HOME set to the tmp homedir. + if (!tmpHome.startsWith("/")) { + tmpHome = pwd() + "/" + tmpHome; + } + env = { + HOME: tmpHome + }; + } + + // Workaround for SERVER-18877 not being fixed in earlier versions. + if (_isWindows()) { + for (var envvar in env) { + cmdline = "set " + envvar + "=" + env[envvar] + "&&" + cmdline; + } + } else { + for (var envvar in env) { + cmdline = envvar + "='" + env[envvar] + "' ; " + cmdline; + } + } + + // Add redirection to cmdline, and add cmdline to args. + cmdline += " " + redirection; + args.push(cmdline); + jsTestLog("Running args:\n " + tojson(args) + "\nwith env:\n " + tojson(env)); + var rc = runProgram.apply(this, args); + + assert.eq(rc, 0); + + var files = listFiles(tmpHome); + jsTestLog(tojson(files)); + + var findFile = function(baseName) { + for (var i = 0; i < files.length; i++) { + if (files[i].baseName === baseName) { + return files[i]; + } + } + return undefined; + }; + + var targetFile = ".dbshell"; + var file = findFile(targetFile); + + assert.neq(typeof(file), "undefined", targetFile + " should exist, but it doesn't"); + assert.eq(file.isDirectory, false, targetFile + " should not be a directory, but it is"); + assert.eq(file.size, 0, targetFile + " should be empty, but it isn't"); + + if (!_isWindows()) { + // On Unix, check that the file has the correct mode (permissions). + // The shell has no way to stat a file. + // There is no stat utility in POSIX. + // `ls -l` is POSIX, so this is the best that we have. + // Check for exactly "-rw-------". + clearRawMongoProgramOutput(); + var rc = runProgram("ls", "-l", file.name); + assert.eq(rc, 0); + // Before SERVER-22992 is fixed: + var output = null; + assert.soon(function() { + output = rawMongoProgramOutput(); + return output != ""; + }); + // After SERVER-22992 is fixed: + // var output = rawMongoProgramOutput(); + var fields = output.split(" "); + // First field is the prefix, second field is the `ls -l` permissions. + assert.eq(fields[1].substr(0, 10), "-rw-------", targetFile + " has bad permissions"); + } + +})(); diff --git a/jstests/noPassthrough/wt_index_option_defaults.js b/jstests/noPassthrough/wt_index_option_defaults.js index 2516cc28d24..b7574d3206a 100644 --- a/jstests/noPassthrough/wt_index_option_defaults.js +++ b/jstests/noPassthrough/wt_index_option_defaults.js @@ -31,9 +31,9 @@ // Use different values for the same configuration string key to test that index-specific // options override collection-wide options, and that collection-wide options override // system-wide options. - var systemWideConfigString = 'split_pct=25,'; - var collectionWideConfigString = 'split_pct=30,'; - var indexSpecificConfigString = 'split_pct=35,'; + var systemWideConfigString = 'split_pct=70,'; + var collectionWideConfigString = 'split_pct=75,'; + var indexSpecificConfigString = 'split_pct=80,'; // Start up a mongod with system-wide defaults for index options and create a collection without // any additional options. Tests than an index without any additional options should take on the diff --git a/jstests/noPassthroughWithMongod/connections_opened.js b/jstests/noPassthroughWithMongod/connections_opened.js index 2ec192ed1e2..d3d141ab5a1 100644 --- a/jstests/noPassthroughWithMongod/connections_opened.js +++ b/jstests/noPassthroughWithMongod/connections_opened.js @@ -20,7 +20,8 @@ var signalCollection = 'keepRunning'; function createPersistentConnection() { assert.soon(function() { try { - return new Mongo(db.getMongo().host); + permConns.push(new Mongo(db.getMongo().host)); + return true; } catch (x) { return false; } @@ -95,7 +96,4 @@ jsTestLog("Testing that current connections counter went down after temporary co waitForConnections(originalConnInfo.current + numPerTypeToCreate, originalConnInfo.totalCreated + numPerTypeToCreate * 2); -persistent = null; -gc(); - MongoRunner.stopMongod(mongo); diff --git a/jstests/noPassthroughWithMongod/ftdc_params.js b/jstests/noPassthroughWithMongod/ftdc_params.js index 5fae9e77c49..fd029a264c2 100644 --- a/jstests/noPassthroughWithMongod/ftdc_params.js +++ b/jstests/noPassthroughWithMongod/ftdc_params.js @@ -19,7 +19,7 @@ // Verify the defaults are as we documented them assert.eq(getparam("diagnosticDataCollectionEnabled"), true); assert.eq(getparam("diagnosticDataCollectionPeriodMillis"), 1000); - assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 100); + assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 200); assert.eq(getparam("diagnosticDataCollectionFileSizeMB"), 10); assert.eq(getparam("diagnosticDataCollectionSamplesPerChunk"), 300); assert.eq(getparam("diagnosticDataCollectionSamplesPerInterimUpdate"), 10); @@ -53,7 +53,7 @@ // Reset assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 10})); - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 100})); + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 200})); assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 1000})); assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 300})); assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 10})); diff --git a/jstests/noPassthroughWithMongod/indexbg_drop.js b/jstests/noPassthroughWithMongod/indexbg_drop.js index ac5783525a6..a81f2f97e61 100644 --- a/jstests/noPassthroughWithMongod/indexbg_drop.js +++ b/jstests/noPassthroughWithMongod/indexbg_drop.js @@ -51,7 +51,7 @@ var bulk = masterDB.getCollection(collection).initializeUnorderedBulkOp(); for (i = 0; i < size; ++i) { bulk.insert({i: Random.rand()}); } -assert.writeOK(bulk.execute()); +assert.writeOK(bulk.execute({w: 2, wtimeout: replTest.kDefaultTimeoutMS})); jsTest.log("Starting background indexing for test of: " + tojson(dc)); // Add another index to be sure the drop command works. diff --git a/jstests/noPassthroughWithMongod/ttl_sharded.js b/jstests/noPassthroughWithMongod/ttl_sharded.js index 577a968eed9..8f37c616f6f 100644 --- a/jstests/noPassthroughWithMongod/ttl_sharded.js +++ b/jstests/noPassthroughWithMongod/ttl_sharded.js @@ -36,15 +36,14 @@ t.ensureIndex({x: 1}, {expireAfterSeconds: 20000}); s.adminCommand({split: ns, middle: {_id: 12}}); s.adminCommand({moveChunk: ns, find: {_id: 0}, to: s.getOther(s.getServer(dbname)).name}); -// one shard will lose 12/12 docs, the other 6/12, so count will go -// from 24 -> 18 or 12 -> 6 -assert.soon(function() { - return t.count() < 7; -}, "TTL index on x didn't delete enough", 70 * 1000); - -// ensure that count ultimately ends up at 6 -assert.eq(0, t.find({x: {$lt: new Date(now - 20000000)}}).count()); -assert.eq(6, t.count()); +// Check that all expired documents are deleted. +assert.soon( + function() { + return t.count() === 6 && t.find({x: {$lt: new Date(now - 20000000)}}).count() === 0; + }, + "TTL index did not successfully delete expired documents, all documents: " + + tojson(t.find().toArray()), + 70 * 1000); // now lets check things explicily on each shard var shard0 = s._connections[0].getDB(dbname); @@ -73,10 +72,13 @@ s.getDB(dbname).runCommand({collMod: coll, index: {keyPattern: {x: 1}, expireAft assert.eq(10000, getTTLTime(shard0.getCollection(coll), {x: 1})); assert.eq(10000, getTTLTime(shard1.getCollection(coll), {x: 1})); -assert.soon(function() { - return t.count() < 6; -}, "new expireAfterSeconds value not taking effect", 70 * 1000); -assert.eq(0, t.find({x: {$lt: new Date(now - 10000000)}}).count()); -assert.eq(3, t.count()); +// Check that all expired documents are deleted. +assert.soon( + function() { + return t.count() === 3 && t.find({x: {$lt: new Date(now - 10000000)}}).count() === 0; + }, + "new expireAfterSeconds did not successfully delete expired documents, all documents: " + + tojson(t.find().toArray()), + 70 * 1000); s.stop(); diff --git a/jstests/replsets/chaining_removal.js b/jstests/replsets/chaining_removal.js index dbc80148745..2c42b3f83e4 100644 --- a/jstests/replsets/chaining_removal.js +++ b/jstests/replsets/chaining_removal.js @@ -2,6 +2,8 @@ (function() { "use strict"; + load("jstests/replsets/rslib.js"); + var numNodes = 5; var host = getHostName(); var name = "chaining_removal"; @@ -19,33 +21,25 @@ {_id: 4, host: nodes[4].host, priority: 0}, ], }); - replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - replTest.awaitNodesAgreeOnPrimary(); + replTest.awaitNodesAgreeOnPrimary(replTest.kDefaultTimeoutMS, nodes, 0); var primary = replTest.getPrimary(); replTest.awaitReplication(); + // When setting up chaining on slow machines, we do not want slow writes or delayed heartbeats + // to cause our nodes to invalidate the sync source provided in the 'replSetSyncFrom' command. + // To achieve this, we disable the server parameter 'maxSyncSourceLagSecs' (see + // repl_settings_init.cpp and TopologyCoordinatorImpl::Options) in + // TopologyCoordinatorImpl::shouldChangeSyncSource(). + assert.commandWorked(nodes[1].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'})); + assert.commandWorked(nodes[4].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'})); + // Force node 1 to sync directly from node 0. - assert.commandWorked(nodes[1].getDB("admin").runCommand({"replSetSyncFrom": nodes[0].host})); - var res; - assert.soon( - function() { - res = nodes[1].getDB("admin").runCommand({"replSetGetStatus": 1}); - return res.syncingTo === nodes[0].host; - }, - function() { - return "node 1 failed to start syncing from node 0: " + tojson(res); - }); + syncFrom(nodes[1], nodes[0], replTest); // Force node 4 to sync through node 1. - assert.commandWorked(nodes[4].getDB("admin").runCommand({"replSetSyncFrom": nodes[1].host})); - assert.soon( - function() { - res = nodes[4].getDB("admin").runCommand({"replSetGetStatus": 1}); - return res.syncingTo === nodes[1].host; - }, - function() { - return "node 4 failed to start chaining through node 1: " + tojson(res); - }); + syncFrom(nodes[4], nodes[1], replTest); // write that should reach all nodes var timeout = 60 * 1000; @@ -54,6 +48,12 @@ }; assert.writeOK(primary.getDB(name).foo.insert({x: 1}, options)); + // Re-enable 'maxSyncSourceLagSecs' checking on sync source. + assert.commandWorked(nodes[1].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'})); + assert.commandWorked(nodes[4].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'})); + var config = primary.getDB("local").system.replset.findOne(); config.members.pop(); config.version++; diff --git a/jstests/replsets/disallow_adding_initialized_node1.js b/jstests/replsets/disallow_adding_initialized_node1.js index 8d4491975b6..ba39e6010ad 100644 --- a/jstests/replsets/disallow_adding_initialized_node1.js +++ b/jstests/replsets/disallow_adding_initialized_node1.js @@ -3,8 +3,10 @@ // Initialize two replica sets A and B with the same name: A_0; B_0 // Add B_0 to the replica set A. This operation should fail on replica set A should fail on // detecting an inconsistent replica set ID in the heartbeat response metadata from B_0. + (function() { 'use strict'; + load("jstests/libs/check_log.js"); var name = 'disallow_adding_initialized_node1'; var replSetA = new ReplSetTest({name: name, nodes: [{rsConfig: {_id: 10}}, ]}); @@ -45,20 +47,9 @@ assert.eq(primaryB, newPrimaryB); // Mismatch replica set IDs in heartbeat responses should be logged. - var checkLog = function(node, msg) { - assert.soon(function() { - var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - return true; - } - } - return false; - }, 'Did not see a log entry containing the following message: ' + msg, 10000, 1000); - }; var msgB = "replica set IDs do not match, ours: " + configB.settings.replicaSetId + "; remote node's: " + configA.settings.replicaSetId; - checkLog(primaryB, msgB); + checkLog.contains(primaryB, msgB); var statusA = assert.commandWorked(primaryA.adminCommand({replSetGetStatus: 1})); var statusB = assert.commandWorked(primaryB.adminCommand({replSetGetStatus: 1})); diff --git a/jstests/replsets/disallow_adding_initialized_node2.js b/jstests/replsets/disallow_adding_initialized_node2.js index c4125f7c069..f54ef947e56 100644 --- a/jstests/replsets/disallow_adding_initialized_node2.js +++ b/jstests/replsets/disallow_adding_initialized_node2.js @@ -8,8 +8,10 @@ // This test requires users to persist across a restart. // @tags: [requires_persistence] + (function() { 'use strict'; + load("jstests/libs/check_log.js"); var name = 'disallow_adding_initialized_node2'; var replSetA = new ReplSetTest( @@ -52,23 +54,12 @@ assert.eq(primaryB, newPrimaryB); // Mismatch replica set IDs in heartbeat responses should be logged. - var checkLog = function(node, msg) { - assert.soon(function() { - var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - return true; - } - } - return false; - }, 'Did not see a log entry containing the following message: ' + msg, 10000, 1000); - }; var msgA = "replica set IDs do not match, ours: " + configA.settings.replicaSetId + "; remote node's: " + configB.settings.replicaSetId; var msgB = "replica set IDs do not match, ours: " + configB.settings.replicaSetId + "; remote node's: " + configA.settings.replicaSetId; - checkLog(primaryA, msgA); - checkLog(primaryB, msgB); + checkLog.contains(primaryA, msgA); + checkLog.contains(primaryB, msgB); var statusA = assert.commandWorked(primaryA.adminCommand({replSetGetStatus: 1})); var statusB = assert.commandWorked(primaryB.adminCommand({replSetGetStatus: 1})); @@ -83,9 +74,6 @@ assert.eq(12, statusA.members[2]._id); assert.eq(primaryB.host, statusA.members[2].name); assert.eq(ReplSetTest.State.DOWN, statusA.members[2].state); - assert(statusA.members[2].lastHeartbeatMessage.indexOf(msgA) != -1 || - statusA.members[2].lastHeartbeatMessage.indexOf( - 'no response within election timeout period') != -1); // Replica set B's config should remain unchanged. assert.eq(1, statusB.members.length); diff --git a/jstests/replsets/do_not_sync_from_stale_sync_source.js b/jstests/replsets/do_not_sync_from_stale_sync_source.js new file mode 100644 index 00000000000..bf6957a0e06 --- /dev/null +++ b/jstests/replsets/do_not_sync_from_stale_sync_source.js @@ -0,0 +1,44 @@ +/* + * This tests that nodes do not sync from other nodes that are behind them. The test sets up a + * 3-node replica set and then stops replication at one node so it starts to lag. The test then + * uses 'replSetSyncFrom' to force the up to date node to sync from the lagging node. After it + * receives its first batch, it errors saying that it cannot sync from a node behind it. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var name = "do_not_sync_from_stale_sync_source"; + var collName = "test.coll"; + + var rst = new ReplSetTest({ + name: name, + nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}], + useBridge: true, + settings: {chainingAllowed: false} + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 0}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Stop node 2 from syncing so it starts lagging."); + assert.commandWorked(nodes[2].getDB('admin').runCommand( + {configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'})); + checkLog.contains(nodes[2], 'stopReplProducer fail point enabled'); + + jsTestLog("Do a write that replicates to [0,1]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Tell node 1 to sync from node 0 which is now behind."); + assert.commandWorked(nodes[1].adminCommand({"replSetSyncFrom": nodes[0].host})); + checkLog.contains(nodes[1], "is not greater than our last fetched OpTime"); +}()); diff --git a/jstests/replsets/double_rollback.js b/jstests/replsets/double_rollback.js index 2286b80e315..1f37bd93349 100644 --- a/jstests/replsets/double_rollback.js +++ b/jstests/replsets/double_rollback.js @@ -9,8 +9,13 @@ * it rolled back, which could then lead to a double-rollback when node 2 was reconnected * to node 1 and tried to apply its oplog despite not being in a consistent state. */ + (function() { 'use strict'; + load("jstests/libs/check_log.js"); + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); var name = "double_rollback"; var dbName = "test"; @@ -40,16 +45,8 @@ }); } - function stepUp(rst, node) { - var primary = rst.getPrimary(); - if (primary != node) { - assert.commandWorked(primary.adminCommand({replSetStepDown: 1})); - } - waitForState(node, ReplSetTest.State.PRIMARY); - } - jsTestLog("Make sure node 0 is primary."); - stepUp(rst, nodes[0]); + rst.stepUp(nodes[0]); assert.eq(nodes[0], rst.getPrimary()); // Wait for all data bearing nodes to get up to date. assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( @@ -100,18 +97,7 @@ jsTestLog("Wait for failpoint on node 2 to pause rollback before it finishes"); // Wait for fail point message to be logged. - var checkLog = function(node, msg) { - assert.soon(function() { - var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - return true; - } - } - return false; - }, 'Did not see a log entry containing the following message: ' + msg, timeout); - }; - checkLog(nodes[2], 'rollback - rollbackHangBeforeFinish fail point enabled'); + checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeFinish fail point enabled'); jsTestLog("Repartition to: [1,3,4] and [0,2]."); nodes[2].disconnect(nodes[1]); @@ -128,8 +114,8 @@ // for a sync source it can use to reach minvalid and get back into SECONDARY state. Node 0 // is the only node it can reach, but since node 0 doesn't contain node 2's minvalid oplog entry // node 2 will refuse to use it as a sync source. - checkLog(nodes[2], - "remote oplog does not contain entry with optime matching our required optime"); + checkLog.contains( + nodes[2], "remote oplog does not contain entry with optime matching our required optime"); var node0RBID = nodes[0].adminCommand('replSetGetRBID').rbid; var node1RBID = nodes[1].adminCommand('replSetGetRBID').rbid; diff --git a/jstests/replsets/double_rollback_early.js b/jstests/replsets/double_rollback_early.js new file mode 100644 index 00000000000..4af501701c6 --- /dev/null +++ b/jstests/replsets/double_rollback_early.js @@ -0,0 +1,139 @@ +/* SERVER-27050 This test causes node 2 to enter rollback, then fail after setting minValid, but + * before truncating the oplog. It will then choose the same sync source (1) and retry the rollback. + * The upstream node itself rolls back at this point. Node 2 should detect this case and fail the + * rollback and refuse to choose node 1 as its sync source because it doesn't have the minValid. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var collName = "test.coll"; + var counter = 0; + + var rst = new ReplSetTest({ + nodes: [ + {}, + {}, + {rsConfig: {priority: 0}}, + {rsConfig: {arbiterOnly: true}}, + {rsConfig: {arbiterOnly: true}} + ], + useBridge: true + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMs}})); + + jsTestLog("Create two partitions: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[0]); + nodes[1].disconnect(nodes[2]); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + + jsTestLog("Do a write that is replicated to [0,2,3,4]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMs}})); + + jsTestLog("Repartition to: [0,2] and [1,3,4]."); + nodes[1].reconnect(nodes[3]); + nodes[1].reconnect(nodes[4]); + nodes[3].disconnect(nodes[0]); + nodes[3].disconnect(nodes[2]); + nodes[4].disconnect(nodes[0]); + nodes[4].disconnect(nodes[2]); + + jsTestLog("Ensure that 0 steps down and that 1 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + waitForState(nodes[1], ReplSetTest.State.PRIMARY); + assert.eq(nodes[1], rst.getPrimary()); + + jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition."); + // We track this object to ensure it gets rolled back on node 1 later. + assert.writeOK(nodes[1].getCollection(collName).insert({a: counter++, tracked: true})); + + // Turn on failpoint on node 2 to pause rollback after oplog is truncated and minValid is set. + assert.commandWorked(nodes[2].adminCommand( + {configureFailPoint: 'rollbackHangThenFailAfterWritingMinValid', mode: 'alwaysOn'})); + + jsTestLog("Repartition to: [0] and [1,2,3,4]."); + nodes[2].disconnect(nodes[0]); + nodes[2].reconnect(nodes[1]); + nodes[2].reconnect(nodes[3]); + nodes[2].reconnect(nodes[4]); + + jsTestLog("Wait for node 2 to go into ROLLBACK and start syncing from node 1."); + // Since nodes 1 and 2 have now diverged, node 2 should go into rollback. + waitForState(nodes[2], ReplSetTest.State.ROLLBACK); + rst.awaitSyncSource(nodes[2], nodes[1]); + + jsTestLog("Wait for failpoint on node 2 to pause rollback after it writes minValid"); + // Wait for fail point message to be logged. + checkLog.contains(nodes[2], + 'rollback - rollbackHangThenFailAfterWritingMinValid fail point enabled'); + + // Switch failpoints, causing rollback to fail then pause when it retries. It is important to + // enable the new one before disabling the current one. + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); + assert.commandWorked(nodes[2].adminCommand( + {configureFailPoint: 'rollbackHangThenFailAfterWritingMinValid', mode: 'off'})); + jsTestLog("Wait for failpoint on node 2 to pause rollback after it restarts"); + // Wait for fail point message to be logged. + checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeStart fail point enabled'); + + jsTestLog("Repartition to: [0,3,4] and [1,2]."); + nodes[3].disconnect(nodes[1]); + nodes[3].reconnect(nodes[0]); + nodes[4].disconnect(nodes[1]); + nodes[4].reconnect(nodes[0]); + + jsTestLog("Ensure that 0 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.PRIMARY); + waitForState(nodes[1], ReplSetTest.State.SECONDARY); + assert.eq(nodes[0], rst.getPrimary()); + // Do a write so that node 0 is definitely ahead of node 1. + assert.writeOK(nodes[0].getCollection(collName).insert({a: counter++})); + + jsTestLog("Repartition to: [0,1,3,4] and [2] so 1 rolls back and replicates from 0."); + assert.eq(nodes[1].getCollection(collName).count({tracked: true}), 1); + nodes[1].reconnect(nodes[0]); + waitForState(nodes[1], ReplSetTest.State.SECONDARY); + jsTestLog("w:2 write to node 0"); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMs}})); + reconnect(nodes[1]); // rollback drops connections. + assert.eq(nodes[1].getCollection(collName).count({tracked: true}), 0); + + // Turn off failpoint on node 2 to allow rollback to finish its attempt at rollback from node 1. + // It should fail with a rbid error and get stuck. + jsTestLog("Repartition to: [0,3,4] and [1,2]."); + nodes[1].reconnect(nodes[2]); + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); + + jsTestLog("Wait for node 2 exit ROLLBACK state and go into RECOVERING"); + waitForState(nodes[2], ReplSetTest.State.RECOVERING); + + // At this point node 2 has truncated its oplog back to the common point and is looking + // for a sync source it can use to reach minvalid and get back into SECONDARY state. Node 1 + // is the only node it can reach, but since node 1 doesn't contain node 2's minvalid oplog entry + // node 2 will refuse to use it as a sync source. + checkLog.contains(nodes[2], "Upstream node rolled back. Need to retry our rollback."); + waitForState(nodes[2], ReplSetTest.State.RECOVERING); + + // This log message means that it will not be willing to use node 1 as the sync source when it + // retries. + checkLog.contains( + nodes[2], "remote oplog does not contain entry with optime matching our required optime"); + + rst.stopSet(); +}()); diff --git a/jstests/replsets/drain.js b/jstests/replsets/drain.js index 7f170c485d9..59082255c1d 100644 --- a/jstests/replsets/drain.js +++ b/jstests/replsets/drain.js @@ -25,7 +25,6 @@ var primary = replSet.getPrimary(); var secondary = replSet.getSecondary(); - var isPV0 = replSet.getReplSetConfigFromNode().protocolVersion != 1; // Do an initial insert to prevent the secondary from going into recovery var numDocuments = 20; @@ -43,7 +42,7 @@ } assert.writeOK(bulk.execute()); jsTestLog('Number of documents inserted into collection on primary: ' + numDocuments); - assert.eq(numDocuments, primary.getDB("foo").foo.find().itcount()); + assert.eq(numDocuments, primary.getDB("foo").foo.count()); assert.soon(function() { var serverStatus = secondary.getDB('foo').serverStatus(); @@ -52,13 +51,12 @@ jsTestLog('Number of operations buffered on secondary since stopping applier: ' + bufferCountChange); return bufferCountChange >= numDocuments - 1; - }, 'secondary did not buffer operations for new inserts on primary', 30000, 1000); + }, 'secondary did not buffer operations for new inserts on primary', 300000, 1000); // Kill primary; secondary will enter drain mode to catch up primary.getDB("admin").shutdownServer({force: true}); - var electionTimeout = (isPV0 ? 60 : 20) * 1000; // Timeout in milliseconds - replSet.waitForState(secondary, ReplSetTest.State.PRIMARY, electionTimeout); + replSet.waitForState(secondary, ReplSetTest.State.PRIMARY); // Ensure new primary is not yet writable jsTestLog('New primary should not be writable yet'); @@ -75,12 +73,7 @@ "find failed with unexpected error code: " + tojson(res)); // Nor should it be readable with the slaveOk bit. secondary.slaveOk = true; - res = secondary.getDB("foo").runCommand({find: "foo"}); - assert.commandFailed(res); - assert.eq(ErrorCodes.NotMasterOrSecondary, - res.code, - "find failed with unexpected error code: " + tojson(res)); - secondary.slaveOk = false; + assert.commandWorked(secondary.getDB("foo").runCommand({find: "foo"})); assert.commandFailedWithCode( secondary.adminCommand({ @@ -100,7 +93,7 @@ assert.commandWorked( secondary.adminCommand({ replSetTest: 1, - waitForDrainFinish: 5000, + waitForDrainFinish: 30000, }), 'replSetTest waitForDrainFinish should work when draining is allowed to complete'); @@ -110,5 +103,5 @@ // Check for at least two entries. There was one prior to freezing op application on the // secondary and we cannot guarantee all writes reached the secondary's op queue prior to // shutting down the original primary. - assert.gte(primary.getDB("foo").foo.find().itcount(), 2); + assert.gte(primary.getDB("foo").foo.count(), 2); })(); diff --git a/jstests/replsets/election_timing.js b/jstests/replsets/election_timing.js deleted file mode 100644 index 48139e6bac0..00000000000 --- a/jstests/replsets/election_timing.js +++ /dev/null @@ -1,143 +0,0 @@ -// Run a small set of tests using the ElectionTimingTest framework. While this -// reports the timing of the election, we are using it to check if any errors happen -// during different election cycles. -(function() { - "use strict"; - load("jstests/libs/election_timing_test.js"); - var testStart = Date.now(); - - var testCases = [ - { - name: "testV1Stop", - description: "protocolVersion 1, primary is stopped", - protocolVersion: 1, - // testRuns is the number of times a new ReplSetTest will be used. - testRuns: 1, - // testCycles is the number of election cycles that will be run per ReplSetTest lifespan. - testCycles: 5, - // testSetup is run after the replSet is initiated. - // Function.prototype is the default. - testSetup: Function.prototype, - // Trigger an election by stepping down, stopping, or partitioning the primary. - // stopPrimary is the default. - electionTrigger: ElectionTimingTest.prototype.stopPrimary, - // After the election has completed, make the old primary available again. - // stopPrimaryReset is the default. - testReset: ElectionTimingTest.prototype.stopPrimaryReset - }, - - /* - This test case is flakey since waiting for the old primary to shutdown can take longer than - the - allowed timeout, even if a new primary was elected during the shutdown time. - - { - name: "testV1StopTimeout1500", - description: "protocolVersion 1, primary is stopped, electionTimeoutMillis set to 1500", - protocolVersion: 1, - testRuns: 1, - testCycles: 5, - // The settings object is merged into the replset config settings object. - settings: {electionTimeoutMillis: 1500} - }, - */ - - { - name: "testV1StepDown", - description: "protocolVersion 1, primary is stepped down", - protocolVersion: 1, - testRuns: 1, - testCycles: 5, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: ElectionTimingTest.prototype.stepDownPrimaryReset, - }, - - { - name: "testV1StepDown1500", - description: "protocolVersion 1, primary is stepped down", - protocolVersion: 1, - testRuns: 1, - testCycles: 5, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: ElectionTimingTest.prototype.stepDownPrimaryReset, - // The settings object is merged into the replset config settings object. - settings: {electionTimeoutMillis: 1500} - }, - - { - name: "testV1StepDownLargeCluster", - description: "protocolVersion 1, primary is stepped down, 7 electable nodes", - protocolVersion: 1, - nodes: 7, - testRuns: 1, - testCycles: 5, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: function() {}, - waitForNewPrimary: function(rst, secondary) { - rst.getPrimary(); - } - }, - - { - name: "testV0Stop", - description: "protocolVersion 0, primary is stopped", - protocolVersion: 0, - testRuns: 1, - testCycles: 1 - }, - - { - name: "testV0StepDown", - description: "protocolVersion 0, primary is stepped down", - protocolVersion: 0, - testRuns: 1, - testCycles: 2, - stepDownGuardTime: 30, - // There is a guard time in pv0 that prevents an election right - // after initiating. - testSetup: function() { - sleep(30 * 1000); - }, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: ElectionTimingTest.prototype.stepDownPrimaryReset - }, - - ]; - - testCases.forEach(function(tc) { - var testRun = new ElectionTimingTest(tc); - tc.testResults = testRun.testResults; - tc.electionTimeoutLimitMillis = testRun.electionTimeoutLimitMillis; - - if (testRun.testErrors.length) { - // Stop tests if we encounter an error. - // Dump available information for debugging. - jsTestLog("Errors from: " + tc.name); - printjson(tc); - printjson(testRun.testErrors); - throw new Error(testRun.testErrors[0].status); - } - // Print results of current test in case - // we need to analyze a failed test later. - jsTestLog("Raw Results: " + tc.name); - printjson(tc.testResults); - }); - - testCases.forEach(function(tc) { - var allResults = []; - tc.testResults.forEach(function(tr) { - allResults = allResults.concat(tr.results); - }); - - var resAvg = Array.avg(allResults); - var resMin = Math.min(... allResults); - var resMax = Math.max(... allResults); - var resStdDev = Array.stdDev(allResults); - - jsTestLog("Results: " + tc.name + " Average over " + allResults.length + " runs: " + - resAvg + " Min: " + resMin + " Max: " + resMax + " StdDev: " + - resStdDev.toFixed(4)); - }); - - jsTestLog("Tests completed in: " + (Date.now() - testStart) / 1000 + " seconds"); -}()); diff --git a/jstests/replsets/last_vote.js b/jstests/replsets/last_vote.js new file mode 100644 index 00000000000..9b05766cae6 --- /dev/null +++ b/jstests/replsets/last_vote.js @@ -0,0 +1,229 @@ +// Tests that the last vote document is stored during elections and that it is loaded and used on +// startup. +// +// The test first runs a few elections and checks that the lastVote document is set correctly +// after each one. +// +// The test then restarts one node as a standalone, changes its last vote doc, and stops the +// other node. It then restarts the first node as a replicaset and manually runs +// replSetRequestVotes commands against it and checks that its response is correct. +// +// @tags: [requires_persistence] + +(function() { + "use strict"; + load("jstests/replsets/rslib.js"); // For getLatestOp() + + var name = "last_vote"; + var rst = new ReplSetTest({ + name: name, + nodes: 2, + }); + rst.startSet(); + + // Lower the election timeout to make the test run faster since it waits for multiple elections. + var conf = rst.getReplSetConfig(); + conf.settings = { + electionTimeoutMillis: 6000, + }; + rst.initiate(conf); + + const lastVoteNS = 'local.replset.election'; + + function getLastVoteDoc(conn) { + assert.eq( + conn.getCollection(lastVoteNS).find().itcount(), 1, 'last vote should be singleton'); + return conn.getCollection(lastVoteNS).findOne(); + } + + function setLastVoteDoc(conn, term, candidate) { + var newLastVote = { + term: term, + candidateIndex: rst.getNodeId(candidate) + }; + return assert.writeOK(conn.getCollection(lastVoteNS).update({}, newLastVote)); + } + + function assertNodeHasLastVote(node, term, candidate) { + var lastVoteDoc = getLastVoteDoc(node); + assert.eq(lastVoteDoc.term, term, node.host + " had wrong last vote term."); + assert.eq(lastVoteDoc.candidateIndex, + rst.getNodeId(candidate), + node.host + " had wrong last vote candidate."); + } + + function assertCurrentTerm(node, term) { + var stat = assert.commandWorked(node.adminCommand({replSetGetStatus: 1})); + assert.eq(stat.term, term, "Term changed when it should not have"); + } + + jsTestLog("Test that last vote is set on successive elections"); + + // Run a few successive elections, alternating who becomes primary. + var numElections = 3; + for (var i = 0; i < numElections; i++) { + var primary = rst.getPrimary(); + var secondary = rst.getSecondary(); + var term = getLatestOp(primary).t; + + jsTestLog("Last vote should have term: " + term + " and candidate: " + primary.host + + ", index: " + rst.getNodeId(primary)); + rst.nodes.forEach(function(node) { + assertNodeHasLastVote(node, term, primary); + }); + + assert.throws(function() { + primary.adminCommand({replSetStepDown: 60 * 10, force: true}); + }); + + // Make sure a new primary has been established. + rst.waitForState(primary, ReplSetTest.State.SECONDARY); + rst.waitForState(secondary, ReplSetTest.State.PRIMARY); + + // Reset election timeout for the old primary. + assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); + } + + var term = getLatestOp(rst.getPrimary()).t + 100; + + jsTestLog("Test that last vote is loaded on startup"); + + // Ensure that all ops are replicated before stepping up node 1. + rst.awaitReplication(); + + // We cannot reconfig node 0 to have priority 0 if it is currently the primary, + // so we make sure node 1 is primary. + jsTestLog("Stepping up node 1"); + rst.stepUp(rst.nodes[1]); + + jsTestLog("Reconfiguring cluster to make node 0 unelectable so it stays SECONDARY on restart"); + conf = rst.getReplSetConfigFromNode(); + conf.version++; + conf.members[0].priority = 0; + reconfig(rst, conf); + rst.awaitNodesAgreeOnConfigVersion(); + + jsTestLog("Restarting node 0 as a standalone"); + var node0 = rst.restart(0, {noReplSet: true}); // Restart as a standalone node. + jsTestLog("Stopping node 1"); + rst.stop(1); // Stop node 1 so that node 0 controls the term by itself. + jsTestLog("Setting the lastVote on node 0 to term: " + term + " candidate: " + + rst.nodes[0].host + ", index: 0"); + setLastVoteDoc(node0, term, rst.nodes[0]); + + jsTestLog("Restarting node 0 in replica set mode"); + node0 = rst.restart(0); // Restart in replSet mode again. + assertCurrentTerm(node0, term); + + jsTestLog("Manually sending node 0 a dryRun replSetRequestVotes command, " + + "expecting failure in old term"); + var response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: true, + term: term - 1, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(!response.voteGranted, + "node granted vote in term before last vote doc: " + tojson(response)); + assert.eq(response.reason, + "candidate's term is lower than mine", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term, rst.nodes[0]); + assertCurrentTerm(node0, term); + + jsTestLog("Manually sending node 0 a dryRun replSetRequestVotes command in same term, " + + "expecting success but no recording of lastVote"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: true, + term: term, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(response.voteGranted, + "node failed to grant dryRun vote in term equal to last vote doc: " + tojson(response)); + assert.eq(response.reason, + "", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term, rst.nodes[0]); + assertCurrentTerm(node0, term); + + jsTestLog( + "Manually sending node 0 a replSetRequestVotes command, expecting failure in same term"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: false, + term: term, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(!response.voteGranted, + "node granted vote in term of last vote doc: " + tojson(response)); + assert.eq(response.reason, + "already voted for another candidate this term", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term, rst.nodes[0]); + assertCurrentTerm(node0, term); + + jsTestLog("Manually sending node 0 a replSetRequestVotes command, " + + "expecting success with a recording of the new lastVote"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: false, + term: term + 1, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term + 1, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(response.voteGranted, + "node failed to grant vote in term greater than last vote doc: " + tojson(response)); + assert.eq(response.reason, + "", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term + 1, rst.nodes[1]); + assertCurrentTerm(node0, term + 1); + + jsTestLog("Manually sending node 0 a dryRun replSetRequestVotes command in future term, " + + "expecting success but no recording of lastVote"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: true, + term: term + 2, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term + 2, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(response.voteGranted, + "node failed to grant vote in term greater than last vote doc: " + tojson(response)); + assert.eq(response.reason, + "", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term + 1, rst.nodes[1]); + assertCurrentTerm(node0, term + 2); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/libs/tags.js b/jstests/replsets/libs/tags.js new file mode 100644 index 00000000000..80ea4264fc3 --- /dev/null +++ b/jstests/replsets/libs/tags.js @@ -0,0 +1,323 @@ +/** + * Sets up a test for replica set tags sets. + * + * https://docs.mongodb.com/v3.0/tutorial/configure-replica-set-tag-sets/ + */ +var TagsTest = function(options) { + 'use strict'; + + if (!(this instanceof TagsTest)) { + return new TagsTest(options); + } + + // Capture the 'this' reference + var self = this; + + self.options = options; + + /** + * Runs the test. + */ + this.run = function() { + var options = this.options; + + load('jstests/replsets/rslib.js'); + + var nodes = options.nodes; + var host = getHostName(); + var name = 'tags'; + var forceWriteMode = options.forceWriteMode; + + var replTest = new ReplSetTest({name: name, nodes: {n0: nodes[0]}, useBridge: true}); + replTest.startSet(); + replTest.initiate(); + + // If provided in 'options', we set the featureCompatibilityVersion. We do this prior to + // adding any other members to the replica set. This effectively allows us to emulate + // upgrading some of our nodes to the latest version while performing write operations under + // different network partition scenarios. + if (options.setFeatureCompatibilityVersion) { + assert.commandWorked(replTest.getPrimary().adminCommand( + {setFeatureCompatibilityVersion: options.setFeatureCompatibilityVersion})); + } + + for (var i = 1; i < nodes.length; ++i) { + replTest.add(nodes[i]); + } + + const conns = replTest.nodes; + nodes = replTest.nodeList(); + var port = replTest.ports; + var nextVersion = replTest.getReplSetConfigFromNode().version + 1; + const replSetConfig = { + _id: name, + members: [ + { + _id: 0, + host: nodes[0], + tags: { + server: '0', + dc: 'ny', + ny: '1', + rack: 'ny.rk1', + }, + }, + { + _id: 1, + host: nodes[1], + priority: 2, + tags: { + server: '1', + dc: 'ny', + ny: '2', + rack: 'ny.rk1', + }, + }, + { + _id: 2, + host: nodes[2], + priority: 3, + tags: { + server: '2', + dc: 'ny', + ny: '3', + rack: 'ny.rk2', 2: 'this', + }, + }, + { + _id: 3, + host: nodes[3], + tags: { + server: '3', + dc: 'sf', + sf: '1', + rack: 'sf.rk1', + }, + }, + { + _id: 4, + host: nodes[4], + tags: { + server: '4', + dc: 'sf', + sf: '2', + rack: 'sf.rk2', + }, + }, + ], + settings: { + getLastErrorModes: { + '2 dc and 3 server': { + dc: 2, + server: 3, + }, + '1 and 2': { + 2: 1, + server: 1, + }, + '2': { + 2: 1, + }, + '3 and 4': { + sf: 2, + }, + '3 or 4': { + sf: 1, + }, + }, + }, + version: nextVersion, + }; + + reconfig(replTest, replSetConfig); + + replTest.waitForState(replTest.nodes[2], ReplSetTest.State.PRIMARY); + replTest.awaitReplication(); + + // Create collection to guard against timeouts due to file allocation. + assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); + replTest.awaitReplication(); + + // nodeId is the index of the node that we expect to see as primary. + // expectedNodesAgreeOnPrimary is a set of nodes that should agree that 'nodeId' is the + // primary. + // expectedWritableNodesCount is the number of nodes we can expect to write to. Defaults to + // expectedNodesAgreeOnPrimary.length. + var ensurePrimary = function( + nodeId, expectedNodesAgreeOnPrimary, expectedWritableNodesCount) { + expectedWritableNodesCount = + expectedWritableNodesCount || expectedNodesAgreeOnPrimary.length; + jsTestLog('ensurePrimary - Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + + ') should be primary.'); + replTest.awaitNodesAgreeOnPrimary( + replTest.kDefaultTimeoutMS, expectedNodesAgreeOnPrimary, nodeId); + jsTestLog('ensurePrimary - Nodes ' + tojson(expectedNodesAgreeOnPrimary) + + ' agree that ' + nodeId + ' (' + replTest.nodes[nodeId].host + + ') should be primary.'); + primary = replTest.getPrimary(); + if (forceWriteMode) { + primary.forceWriteMode(forceWriteMode); + } + var writeConcern = { + writeConcern: + {w: expectedWritableNodesCount, wtimeout: replTest.kDefaultTimeoutMS} + }; + assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); + jsTestLog('ensurePrimary - Successfully written a document to primary node (' + + replTest.nodes[nodeId].host + ') using a write concern of w:' + + expectedWritableNodesCount); + return primary; + }; + + // 2 should eventually stage a priority takeover from the primary. + var primary = ensurePrimary(2, replTest.nodes); + + jsTestLog('primary is now 2'); + var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; + jsTestLog('test configuration = ' + tojson(config)); + + jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); + conns[0].disconnect(conns[3]); + conns[0].disconnect(conns[4]); + conns[1].disconnect(conns[3]); + conns[1].disconnect(conns[4]); + conns[2].disconnect(conns[3]); + conns[2].disconnect(conns[4]); + conns[3].disconnect(conns[4]); + jsTestLog('Done setting up partitions'); + + jsTestLog( + 'partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); + jsTestLog( + 'partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); + + var doc = { + x: 1 + }; + + // This timeout should be shorter in duration than the server parameter + // maxSyncSourceLagSecs. + // Some writes are expected to block for this 'timeout' duration before failing. + // Depending on the order of heartbeats (containing last committed op time) received + // by a node, it might hang up on its sync source. This may cause some of the write concern + // tests to fail. + var timeout = 20 * 1000; + + jsTestLog('test1'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3)); + + jsTestLog('Non-existent write concern should be rejected.'); + options = { + writeConcern: {w: 'blahblah', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); + assert.neq(null, result.getWriteConcernError()); + assert.eq(ErrorCodes.UnknownReplWriteConcern, + result.getWriteConcernError().code, + tojson(result.getWriteConcernError())); + + jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); + var options = { + writeConcern: {w: '3 or 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + result = primary.getDB('foo').bar.insert(doc, options); + assert.neq(null, result.getWriteConcernError()); + assert(result.getWriteConcernError().errInfo.wtimeout); + + conns[1].reconnect(conns[4]); + jsTestLog('partitions: [0-1-2] [1-4] [3] ' + + '(all nodes besides node 3 can replicate from primary node 2)'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3), 4); + + jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + + primary.host + ' via node 1 ' + replTest.nodes[1].host); + options = { + writeConcern: {w: '3 or 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); + options = { + writeConcern: {w: '3 and 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); + assert.neq(null, result.getWriteConcernError()); + assert(result.getWriteConcernError().errInfo.wtimeout, + tojson(result.getWriteConcernError())); + + conns[3].reconnect(conns[4]); + jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + + '(all secondaries can replicate from primary node 2)'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3), replTest.nodes.length); + + jsTestLog('Write concern "3 and 4" should work - ' + + 'nodes 3 and 4 are connected to primary via node 1.'); + options = { + writeConcern: {w: '3 and 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "2" - writes to primary only.'); + options = { + writeConcern: {w: '2', wtimeout: 0} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "1 and 2"'); + options = { + writeConcern: {w: '1 and 2', wtimeout: 0} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "2 dc and 3 server"'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3), replTest.nodes.length); + options = { + writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Bringing down current primary node 2 ' + primary.host + + ' to allow next higher priority node 1 ' + replTest.nodes[1].host + + ' to become primary.'); + + conns[1].reconnect(conns[3]); + conns[2].disconnect(conns[0]); + conns[2].disconnect(conns[1]); + jsTestLog('partitions: [0-1] [2] [1-3-4] ' + + '(all secondaries except down node 2 can replicate from new primary node 1)'); + + // Node 1 with slightly higher priority will take over. + jsTestLog('1 must become primary here because otherwise the other members will take too ' + + 'long timing out their old sync threads'); + primary = ensurePrimary(1, replTest.nodes.slice(0, 2), 4); + + jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + + primary.host); + options = { + writeConcern: {w: '3 and 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + + ' is down.'); + options = { + writeConcern: {w: '2', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); + assert.neq(null, result.getWriteConcernError()); + assert(result.getWriteConcernError().errInfo.wtimeout); + + replTest.stopSet(); + }; + +}; diff --git a/jstests/replsets/maintenance.js b/jstests/replsets/maintenance.js index 7e49e07e396..b1fe94efc0e 100644 --- a/jstests/replsets/maintenance.js +++ b/jstests/replsets/maintenance.js @@ -75,6 +75,10 @@ assert.soon(function() { return !im.secondary && !im.ismaster; }); +var recv = conns[1].getDB("admin").runCommand({find: "foo"}); +assert.commandFailed(recv); +assert.eq(recv.errmsg, "node is recovering"); + print("now getmore shouldn't work"); var ex = assert.throws(function() { lastDoc = null; diff --git a/jstests/replsets/maxSyncSourceLagSecs.js b/jstests/replsets/maxSyncSourceLagSecs.js index 6bcec4a3f71..a00ad8a7bef 100644 --- a/jstests/replsets/maxSyncSourceLagSecs.js +++ b/jstests/replsets/maxSyncSourceLagSecs.js @@ -4,6 +4,8 @@ // @tags: [requires_fsync] (function() { "use strict"; + load("jstests/replsets/rslib.js"); + var name = "maxSyncSourceLagSecs"; var replTest = new ReplSetTest({ name: name, @@ -23,36 +25,33 @@ }); replTest.awaitNodesAgreeOnPrimary(); + // Disable maxSyncSourceLagSecs behavior until we've established the spanning tree we want. + replTest.nodes.forEach(function(node) { + assert.commandWorked(node.getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'})); + }); + var master = replTest.getPrimary(); var slaves = replTest.liveNodes.slaves; - assert.commandWorked(slaves[0].getDB("admin").runCommand({replSetSyncFrom: master.name})); - assert.commandWorked(slaves[1].getDB("admin").runCommand({replSetSyncFrom: master.name})); + syncFrom(slaves[0], master, replTest); + syncFrom(slaves[1], master, replTest); master.getDB("foo").bar.save({a: 1}); replTest.awaitReplication(); + jsTestLog("Setting sync target of slave 2 to slave 1"); + syncFrom(slaves[1], slaves[0], replTest); + printjson(replTest.status()); + // need to put at least maxSyncSourceLagSecs b/w first op and subsequent ops // so that the shouldChangeSyncSource logic goes into effect sleep(4000); - jsTestLog("Setting sync target of slave 2 to slave 1"); - assert.soon(function() { - // We do a write each time and have this in a try...catch block due to the fallout of - // SERVER-24114. If that timeout occurs, then we search for another sync source, however we - // will not find one unless more writes have come in. Additionally, it is possible that - // slaves[1] will switch to sync from slaves[0] after slaves[1] replicates a write from - // the primary but before slaves[0] replicates it. slaves[1] will then have to roll back - // which would cause a network error. - try { - slaves[1].getDB("admin").runCommand({replSetSyncFrom: slaves[0].name}); - var res = slaves[1].getDB("admin").runCommand({"replSetGetStatus": 1}); - master.getDB("foo").bar.insert({a: 1}); - return res.syncingTo === slaves[0].name; - } catch (e) { - print("Exception in assert.soon, retrying: " + e); - return false; - } - }, "sync target not changed to other slave", 100 * 1000, 2 * 1000); - printjson(replTest.status()); + // Re-enable maxSyncSourceLagSecs behavior now that we have the spanning tree we want and are + // ready to test that behavior. + replTest.nodes.forEach(function(node) { + assert.commandWorked(node.getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'})); + }); jsTestLog("Lock slave 1 and add some docs. Force sync target for slave 2 to change to primary"); assert.commandWorked(slaves[0].getDB("admin").runCommand({fsync: 1, lock: 1})); diff --git a/jstests/replsets/no_chaining.js b/jstests/replsets/no_chaining.js index ad086c72f9a..07a67c3ea2b 100644 --- a/jstests/replsets/no_chaining.js +++ b/jstests/replsets/no_chaining.js @@ -1,3 +1,4 @@ +load("jstests/replsets/rslib.js"); function myprint(x) { print("chaining output: " + x); @@ -38,14 +39,7 @@ var checkNoChaining = function() { }; var forceSync = function() { - var config; - try { - config = nodes[2].getDB("local").system.replset.findOne(); - } catch (e) { - config = nodes[2].getDB("local").system.replset.findOne(); - } - var targetHost = config.members[1].host; - printjson(nodes[2].getDB("admin").runCommand({replSetSyncFrom: targetHost})); + syncFrom(nodes[2], nodes[1], replTest); assert.soon(function() { return nodes[2].getDB("test").foo.findOne() != null; }, 'Check for data after force sync'); diff --git a/jstests/replsets/no_flapping_during_network_partition.js b/jstests/replsets/no_flapping_during_network_partition.js new file mode 100644 index 00000000000..1fb987aef99 --- /dev/null +++ b/jstests/replsets/no_flapping_during_network_partition.js @@ -0,0 +1,60 @@ +/* + * Test that arbiters vote no in elections if they can see a healthy primary of equal or greater + * priority to the candidate, preventing flapping during certain kinds of network partitions. + * + * 1. Initiate a 3-node replica set with one arbiter (PSA) and a higher priority primary. + * 2. Create a network partition between the primary and secondary. + * 3. Wait long enough for the secondary to call for an election. + * 4. Verify the primary and secondary did not change. + * 5. Heal the partition. + * 6. Verify the primary and secondary did not change and are in the initial term. + */ + +(function() { + "use strict"; + + load("jstests/libs/check_log.js"); + + var name = "no_flapping_during_network_partition"; + + var replTest = new ReplSetTest({name: name, nodes: 3, useBridge: true}); + var nodes = replTest.startSet(); + var config = replTest.getReplSetConfig(); + config.members[0].priority = 5; + config.members[2].arbiterOnly = true; + config.settings = { + electionTimeoutMillis: 2000 + }; + replTest.initiate(config); + + function getTerm(node) { + return node.adminCommand({replSetGetStatus: 1}).term; + } + + replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY); + + var primary = replTest.getPrimary(); + var secondary = replTest.getSecondary(); + var initialTerm = getTerm(primary); + + jsTestLog("Create a network partition between the primary and secondary."); + primary.disconnect(secondary); + + jsTestLog("Wait long enough for the secondary to call for an election."); + checkLog.contains(secondary, "can see a healthy primary of equal or greater priority"); + + jsTestLog("Verify the primary and secondary do not change during the partition."); + assert.eq(primary, replTest.getPrimary()); + assert.eq(secondary, replTest.getSecondary()); + + jsTestLog("Heal the partition."); + primary.reconnect(secondary); + + jsTestLog("Verify the primary and secondary did not change and are in the initial term."); + assert.eq(primary, replTest.getPrimary()); + assert.eq(secondary, replTest.getSecondary()); + assert.eq(initialTerm, getTerm(primary)); + assert.eq(initialTerm, getTerm(secondary)); + + replTest.stopSet(); +})(); diff --git a/jstests/replsets/oplog_replay_on_startup_update_and_delete.js b/jstests/replsets/oplog_replay_on_startup_update_and_delete.js new file mode 100644 index 00000000000..14d99b330a4 --- /dev/null +++ b/jstests/replsets/oplog_replay_on_startup_update_and_delete.js @@ -0,0 +1,48 @@ +// SERVER-7200 On startup, replica set nodes delete oplog state past the oplog delete point and +// apply any remaining unapplied ops before coming up as a secondary. This test specifically tests +// having an update and a delete of the same document in the same batch. This is a bit of an edge +// case because if the delete has been applied already, the update won't find any documents. +// +// @tags: [requires_persistence] +(function() { + "use strict"; + + var ns = "test.coll"; + var id = ObjectId(); + + var rst = new ReplSetTest({ + nodes: 1, + }); + + rst.startSet(); + rst.initiate(); + + var conn = rst.getPrimary(); // Waits for PRIMARY state. + + // Do the insert update and delete operations. + var coll = conn.getCollection(ns); + assert.writeOK(coll.insert({_id: id})); + assert.writeOK(coll.update({_id: id}, {$inc: {a: 1}})); + assert.writeOK(coll.remove({_id: id})); + assert.eq(coll.findOne({_id: id}), null); + + // Set the appliedThrough point back to the insert so the update and delete are replayed. + conn = rst.restart(0, {noReplSet: true}); // Restart as a standalone node. + assert.neq(null, conn, "failed to restart"); + var oplog = conn.getCollection('local.oplog.rs'); + oplog.find().forEach(printjsononeline); + assert.eq(oplog.count({ns: ns, op: 'i'}), 1); + var insertOp = oplog.findOne({ns: ns, op: 'i'}); + var term = 't' in insertOp ? insertOp.t : -1; + var minValidColl = conn.getCollection('local.replset.minvalid'); + assert.writeOK(minValidColl.update({}, {$set: {begin: {ts: insertOp.ts, t: term}}})); + printjson({minValidDoc: minValidColl.findOne()}); + + // Make sure it starts up fine again and doesn't have the document. + conn = rst.restart(0); // Restart in replSet mode again. + conn = rst.getPrimary(); // Waits for PRIMARY state. + coll = conn.getCollection(ns); + assert.eq(coll.findOne({_id: id}), null); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/priority_takeover_cascading_priorities.js b/jstests/replsets/priority_takeover_cascading_priorities.js index df119abb669..e210d250f10 100644 --- a/jstests/replsets/priority_takeover_cascading_priorities.js +++ b/jstests/replsets/priority_takeover_cascading_priorities.js @@ -21,25 +21,15 @@ replSet.startSet(); replSet.initiate(); - var waitForPrimary = function(i) { - // Refresh connections to nodes. - replSet.status(); - assert.commandWorked( - replSet.nodes[i].adminCommand({ - replSetTest: 1, - waitForMemberState: ReplSetTest.State.PRIMARY, - timeoutMillis: 60 * 1000, - }), - 'node ' + i + ' ' + replSet.nodes[i].host + ' failed to become primary'); - }; - - waitForPrimary(0); + replSet.waitForState(0, ReplSetTest.State.PRIMARY); + // Wait until all nodes get the "no-op" of "new primary" after initial sync. + waitUntilAllNodesCaughtUp(replSet.nodes); replSet.stop(0); - waitForPrimary(1); + replSet.waitForState(1, ReplSetTest.State.PRIMARY); replSet.stop(1); - waitForPrimary(2); + replSet.waitForState(2, ReplSetTest.State.PRIMARY); // Cannot stop any more nodes because we will not have a majority. })(); diff --git a/jstests/replsets/priority_takeover_two_nodes_equal_priority.js b/jstests/replsets/priority_takeover_two_nodes_equal_priority.js index 42134f9430e..10f31796b2a 100644 --- a/jstests/replsets/priority_takeover_two_nodes_equal_priority.js +++ b/jstests/replsets/priority_takeover_two_nodes_equal_priority.js @@ -1,25 +1,29 @@ -// 2 nodes with non-default priority. -// 3-node replica set with priorities 3, 3 and 1 (default) -// Start replica set. Ensure that highest priority node becomes primary eventually. -// Step down the primary and confirm that the next highest priority node becomes primary. +/** + * Test to ensure that nodes with the highest priorities eventually become PRIMARY. + * + * 1. Initiate a 3 node replica set with node priorities of 3, 3 and 1 (default) + * 2. Make sure that one of the highest priority nodes becomes PRIMARY. + * 3. Step down the PRIMARY and confirm that the other high priority node becomes PRIMARY. + */ load('jstests/replsets/rslib.js'); (function() { 'use strict'; var name = 'priority_takeover_two_nodes_equal_priority'; - var replSet = new ReplSetTest( - {name: name, nodes: [{rsConfig: {priority: 3}}, {rsConfig: {priority: 3}}, {}, ]}); - replSet.startSet(); - replSet.initiate(); + var replTest = new ReplSetTest( + {name: name, nodes: [{rsConfig: {priority: 3}}, {rsConfig: {priority: 3}}, {}]}); + replTest.startSet(); + replTest.initiate(); + jsTestLog("Waiting for one of the high priority nodes to become PRIMARY."); var primary; var primaryIndex = -1; var defaultPriorityNodeIndex = 2; assert.soon( function() { - primary = replSet.getPrimary(); - replSet.nodes.find(function(node, index, array) { + primary = replTest.getPrimary(); + replTest.nodes.find(function(node, index, array) { if (primary.host == node.host) { primaryIndex = index; return true; @@ -28,27 +32,25 @@ load('jstests/replsets/rslib.js'); }); return primaryIndex !== defaultPriorityNodeIndex; }, - 'neither of the priority 3 nodes was elected primary', - 60000, // timeout - 1000 // interval + 'Neither of the high priority nodes was elected primary.', + replTest.kDefaultTimeoutMS, // timeout + 1000 // interval ); - try { - assert.commandWorked(primary.getDB('admin').runCommand({replSetStepDown: 90})); - } catch (x) { - // expected - } - var newPrimaryIndex = primaryIndex === 0 ? 1 : 0; - - // Refresh connections to nodes. - replSet.status(); - - assert.commandWorked(replSet.nodes[newPrimaryIndex].adminCommand({ - replSetTest: 1, - waitForMemberState: ReplSetTest.State.PRIMARY, - timeoutMillis: 60 * 1000, - }), - 'node ' + newPrimaryIndex + ' ' + replSet.nodes[newPrimaryIndex].host + - ' failed to become primary'); + jsTestLog("Stepping down the current primary."); + assert.throws(function() { + assert.commandWorked( + primary.adminCommand({replSetStepDown: 10 * 60, secondaryCatchUpPeriodSecs: 10 * 60})); + }); + + // Make sure the primary has stepped down. + assert.neq(primary, replTest.getPrimary()); + + // We expect the other high priority node to eventually become primary. + var expectedNewPrimaryIndex = (primaryIndex === 0) ? 1 : 0; + + jsTestLog("Waiting for the other high priority node to become PRIMARY."); + var expectedNewPrimary = replTest.nodes[expectedNewPrimaryIndex]; + replTest.waitForState(expectedNewPrimary, ReplSetTest.State.PRIMARY); })(); diff --git a/jstests/replsets/protocol_version_upgrade_downgrade.js b/jstests/replsets/protocol_version_upgrade_downgrade.js index 58210853c91..06bd26f2d94 100644 --- a/jstests/replsets/protocol_version_upgrade_downgrade.js +++ b/jstests/replsets/protocol_version_upgrade_downgrade.js @@ -35,7 +35,8 @@ load("jstests/replsets/rslib.js"); // Do a write, this will set up sync sources on secondaries. print("do a write"); - assert.writeOK(primaryColl.bar.insert({x: 1}, {writeConcern: {w: 3}})); + assert.writeOK( + primaryColl.bar.insert({x: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); // Check optime format in protocol version 0, which is a Timestamp. var res = primary.adminCommand({replSetGetStatus: 1}); assert.commandWorked(res); @@ -56,7 +57,8 @@ load("jstests/replsets/rslib.js"); // This write may block until all nodes finish upgrade, because replSetUpdatePosition may be // rejected by the primary for mismatched config version before secondaries get reconfig. // This will make secondaries wait for 0.5 seconds and retry. - assert.writeOK(primaryColl.bar.insert({x: 2}, {writeConcern: {w: 3}})); + assert.writeOK( + primaryColl.bar.insert({x: 2}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); // Check optime format in protocol version 1, which is an object including the term. res = primary.adminCommand({replSetGetStatus: 1}); @@ -79,7 +81,8 @@ load("jstests/replsets/rslib.js"); conf.protocolVersion = 0; conf.version++; reconfig(rst, conf); - assert.writeOK(primaryColl.bar.insert({x: 3}, {writeConcern: {w: 3}})); + assert.writeOK( + primaryColl.bar.insert({x: 3}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); // Check optime format in protocol version 0, which is a Timestamp. res = primary.adminCommand({replSetGetStatus: 1}); diff --git a/jstests/replsets/read_after_optime.js b/jstests/replsets/read_after_optime.js index 30cf7782679..7c5217660f6 100644 --- a/jstests/replsets/read_after_optime.js +++ b/jstests/replsets/read_after_optime.js @@ -2,6 +2,7 @@ (function() { "use strict"; + load("jstests/libs/check_log.js"); var replTest = new ReplSetTest({nodes: 2}); replTest.startSet(); @@ -33,30 +34,6 @@ assert.gt(timeoutResult.waitedMS, 500); }; - var countLogMessages = function(msg) { - var total = 0; - var logMessages = assert.commandWorked(testDB.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - total++; - } - } - return total; - }; - - var checkLog = function(msg, expectedCount) { - var count; - assert.soon( - function() { - count = countLogMessages(msg); - return expectedCount == count; - }, - 'Expected ' + expectedCount + ', but instead saw ' + count + - ' log entries containing the following message: ' + msg, - 60000, - 300); - }; - // Run the time out test 3 times with replication debug log level increased to 2 // for first and last run. The time out message should be logged twice. testDB.setLogLevel(2, 'command'); @@ -65,7 +42,7 @@ var msg = 'Command on database ' + testDB.getName() + ' timed out waiting for read concern to be satisfied. Command:'; - checkLog(msg, 1); + checkLog.containsWithCount(testDB.getMongo(), msg, 1); // Read concern timed out message should not be logged. runTimeoutTest(); @@ -74,7 +51,7 @@ runTimeoutTest(); testDB.setLogLevel(0, 'command'); - checkLog(msg, 2); + checkLog.containsWithCount(testDB.getMongo(), msg, 2); // Test read on future afterOpTime that will eventually occur. var insertFunc = startParallelShell( diff --git a/jstests/replsets/read_committed.js b/jstests/replsets/read_committed.js index 26611a04f0a..5d1d906c823 100644 --- a/jstests/replsets/read_committed.js +++ b/jstests/replsets/read_committed.js @@ -1,6 +1,4 @@ /** - * @tags: [requires_journaling] - * * Test basic read committed functionality, including: * - Writes with writeConcern 'majority' should be visible once the write completes. * - With the only data-bearing secondary down, committed reads should not include newly inserted @@ -11,6 +9,46 @@ (function() { "use strict"; + const majorityWriteConcern = { + writeConcern: {w: "majority", wtimeout: 60 * 1000} + }; + + // Each test case includes a 'prepareCollection' method that sets up the initial state starting + // with an empty collection, a 'write' method that does some write, and two arrays, + // 'expectedBefore' and 'expectedAfter' that describe the expected contents of the collection + // before and after the write. The 'prepareCollection' and 'write' methods should leave the + // collection either empty or with a single document with _id: 1. + const testCases = { + insert: { + prepareCollection: function(coll) {}, // No-op + write: function(coll, writeConcern) { + assert.writeOK(coll.insert({_id: 1}, writeConcern)); + }, + expectedBefore: [], + expectedAfter: [{_id: 1}], + }, + update: { + prepareCollection: function(coll) { + assert.writeOK(coll.insert({_id: 1, state: 'before'}, majorityWriteConcern)); + }, + write: function(coll, writeConcern) { + assert.writeOK(coll.update({_id: 1}, {$set: {state: 'after'}}, writeConcern)); + }, + expectedBefore: [{_id: 1, state: 'before'}], + expectedAfter: [{_id: 1, state: 'after'}], + }, + remove: { + prepareCollection: function(coll) { + assert.writeOK(coll.insert({_id: 1}, majorityWriteConcern)); + }, + write: function(coll, writeConcern) { + assert.writeOK(coll.remove({_id: 1}, writeConcern)); + }, + expectedBefore: [{_id: 1}], + expectedAfter: [], + }, + }; + // Set up a set and grab things for later. var name = "read_committed"; var replTest = @@ -29,56 +67,117 @@ throw e; } - replTest.initiate({ + var nodes = replTest.nodeList(); + var config = { "_id": name, "members": [ {"_id": 0, "host": nodes[0]}, {"_id": 1, "host": nodes[1], priority: 0}, {"_id": 2, "host": nodes[2], arbiterOnly: true} ] - }); + }; + replTest.initiate(config); // Get connections and collection. var primary = replTest.getPrimary(); var secondary = replTest.liveNodes.slaves[0]; - var secondaryId = replTest.getNodeId(secondary); - var db = primary.getDB(name); - var t = db[name]; + var coll = primary.getDB(name)[name]; + var secondaryColl = secondary.getDB(name)[name]; - function doDirtyRead() { - var res = t.runCommand('find', {"readConcern": {"level": "local"}}); - assert.commandWorked(res); - return new DBCommandCursor(db.getMongo(), res).toArray()[0].state; + function log(arg) { + jsTest.log(tojson(arg)); + } + + function doRead(coll, readConcern) { + readConcern.maxTimeMS = 3000; + var res = assert.commandWorked(coll.runCommand('find', readConcern)); + return new DBCommandCursor(coll.getMongo(), res).toArray(); + } + + function doDirtyRead(coll) { + log("doing dirty read"); + var ret = doRead(coll, {"readConcern": {"level": "local"}}); + log("done doing dirty read."); + return ret; + } + + function doCommittedRead(coll) { + log("doing committed read"); + var ret = doRead(coll, {"readConcern": {"level": "majority"}}); + log("done doing committed read."); + return ret; } - function doCommittedRead() { - var res = t.runCommand('find', {"readConcern": {"level": "majority"}}); + function readLatestOplogEntry(readConcernLevel) { + var oplog = primary.getDB('local').oplog.rs; + var res = oplog.runCommand('find', + { + "readConcern": {"level": readConcernLevel}, + "maxTimeMS": 3000, + sort: {$natural: -1}, + limit: 1, + }); assert.commandWorked(res); - return new DBCommandCursor(db.getMongo(), res).toArray()[0].state; + return new DBCommandCursor(coll.getMongo(), res).toArray()[0]; } - // Do a write, wait for it to replicate, and ensure it is visible. - assert.writeOK( - t.save({_id: 1, state: 0}, {writeConcern: {w: "majority", wtimeout: 60 * 1000}})); - assert.eq(doDirtyRead(), 0); - assert.eq(doCommittedRead(), 0); + for (var testName in testCases) { + jsTestLog('Running test ' + testName); + var test = testCases[testName]; + + const setUpInitialState = function setUpInitialState() { + assert.writeOK(coll.remove({}, majorityWriteConcern)); + test.prepareCollection(coll); + // Do some sanity checks. + assert.eq(doDirtyRead(coll), test.expectedBefore); + assert.eq(doCommittedRead(coll), test.expectedBefore); + }; - replTest.stop(secondaryId); + // Writes done with majority write concern must be immediately visible to both dirty and + // committed reads. + setUpInitialState(); + test.write(coll, majorityWriteConcern); + assert.eq(doDirtyRead(coll), test.expectedAfter); + assert.eq(doCommittedRead(coll), test.expectedAfter); - // Do a write and ensure it is only visible to dirty reads - assert.writeOK(t.save({_id: 1, state: 1})); - assert.eq(doDirtyRead(), 1); - assert.eq(doCommittedRead(), 0); + // Return to the initial state, then stop the secondary from applying new writes to prevent + // them from becoming committed. + setUpInitialState(); + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "alwaysOn"})); + const initialOplogTs = readLatestOplogEntry('local').ts; - // Try the committed read again after sleeping to ensure it doesn't only work for queries - // immediately after the write. - sleep(1000); - assert.eq(doCommittedRead(), 0); + // Writes done without majority write concern must be immediately visible to dirty read + // and hidden from committed reads until they have been replicated. The rules for seeing + // an oplog entry for a write are the same as for the write itself. + test.write(coll, {}); + assert.eq(doDirtyRead(coll), test.expectedAfter); + assert.neq(readLatestOplogEntry('local').ts, initialOplogTs); + assert.eq(doCommittedRead(coll), test.expectedBefore); + assert.eq(readLatestOplogEntry('majority').ts, initialOplogTs); - // Restart the node and ensure the committed view is updated. - replTest.restart(secondaryId); - db.getLastError("majority", 60 * 1000); - assert.eq(doDirtyRead(), 1); - assert.eq(doCommittedRead(), 1); + // Try the committed read again after sleeping to ensure it doesn't only work for + // queries immediately after the write. + sleep(1000); + assert.eq(doCommittedRead(coll), test.expectedBefore); + assert.eq(readLatestOplogEntry('majority').ts, initialOplogTs); + // Restart oplog application on the secondary and ensure the committed view is updated. + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "off"})); + coll.getDB().getLastError("majority", 60 * 1000); + assert.eq(doCommittedRead(coll), test.expectedAfter); + assert.neq(readLatestOplogEntry('majority').ts, initialOplogTs); + + // The secondary will be able to make the write committed soon after the primary, but there + // is no way to block until it does. + try { + assert.soon(function() { + return friendlyEqual(doCommittedRead(secondaryColl), test.expectedAfter); + }); + } catch (e) { + // generate useful error messages on failures. + assert.eq(doCommittedRead(secondaryColl), test.expectedAfter); + } + } }()); diff --git a/jstests/replsets/read_committed_after_rollback.js b/jstests/replsets/read_committed_after_rollback.js index 68ba334ac9c..4101d9a242c 100644 --- a/jstests/replsets/read_committed_after_rollback.js +++ b/jstests/replsets/read_committed_after_rollback.js @@ -149,8 +149,10 @@ load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. // now be visible as a committed read to both oldPrimary and newPrimary. assert.commandWorked( pureSecondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "off"})); - assert.commandWorked( - newPrimaryColl.runCommand({getLastError: 1, w: 'majority', wtimeout: 30000})); + // Do a write to the new primary so that the old primary can establish a sync source to learn + // about the new commit. + assert.writeOK(newPrimary.getDB(name).unrelatedCollection.insert( + {a: 1}, {writeConcern: {w: 'majority', wtimeout: replTest.kDefaultTimeoutMS}})); assert.eq(doCommittedRead(newPrimaryColl), 'new'); assert.eq(doCommittedRead(oldPrimaryColl), 'new'); }()); diff --git a/jstests/replsets/read_committed_stale_history.js b/jstests/replsets/read_committed_stale_history.js new file mode 100644 index 00000000000..a92595bf439 --- /dev/null +++ b/jstests/replsets/read_committed_stale_history.js @@ -0,0 +1,149 @@ +/* + * Tests that a node on a stale branch of history won't incorrectly mark its ops as committed even + * when hearing about a commit point with a higher optime from a new primary. + */ +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/libs/write_concern_util.js"); + load("jstests/replsets/rslib.js"); + + var name = "readCommittedStaleHistory"; + var dbName = "wMajorityCheck"; + var collName = "stepdown"; + + var rst = new ReplSetTest({ + name: name, + nodes: [{}, {}, {rsConfig: {priority: 0}}, ], + nodeOptions: {enableMajorityReadConcern: ""}, + useBridge: true + }); + + if (!startSetIfSupportsReadMajority(rst)) { + jsTest.log("skipping test since storage engine doesn't support committed reads"); + return; + } + + var nodes = rst.nodes; + rst.initiate(); + + /** + * Waits for the given node to be in state primary *and* have finished drain mode and thus + * be available for writes. + */ + function waitForPrimary(node) { + assert.soon(function() { + return node.adminCommand('ismaster').ismaster; + }); + } + + // Asserts that the given document is not visible in the committed snapshot on the given node. + function checkDocNotCommitted(node, doc) { + var docs = + node.getDB(dbName).getCollection(collName).find(doc).readConcern('majority').toArray(); + assert.eq(0, docs.length, tojson(docs)); + } + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + var primary = rst.getPrimary(); + var secondaries = rst.getSecondaries(); + assert.eq(nodes[0], primary); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + // Stop the secondaries from replicating. + stopServerReplication(secondaries); + // Stop the primary from being able to complete stepping down. + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'alwaysOn'})); + + jsTestLog("Do a write that won't ever reach a majority of nodes"); + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert({a: 2})); + + // Ensure that the write that was just done is not visible in the committed snapshot. + checkDocNotCommitted(nodes[0], {a: 2}); + + // Prevent the primary from rolling back later on. + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); + + jsTest.log("Disconnect primary from all secondaries"); + nodes[0].disconnect(nodes[1]); + nodes[0].disconnect(nodes[2]); + + // Ensure the soon-to-be primary cannot see the write from the old primary. + assert.eq(null, nodes[1].getDB(dbName).getCollection(collName).findOne({a: 2})); + + jsTest.log("Wait for a new primary to be elected"); + // Allow the secondaries to replicate again. + restartServerReplication(secondaries); + + waitForPrimary(nodes[1]); + + jsTest.log("Do a write to the new primary"); + assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( + {a: 3}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + // Ensure the new primary still cannot see the write from the old primary. + assert.eq(null, nodes[1].getDB(dbName).getCollection(collName).findOne({a: 2})); + + // Ensure the stale primary still hasn't committed the write it did that never reached + // the other nodes. + checkDocNotCommitted(nodes[0], {a: 2}); + + jsTest.log("Reconnect the old primary to the rest of the nodes"); + nodes[1].reconnect(nodes[0]); + nodes[2].reconnect(nodes[0]); + + // Sleep 10 seconds to allow some heartbeats to be processed, so we can verify that the + // heartbeats don't cause the stale primary to incorrectly advance the commit point. + sleep(10000); + + // Ensure the new primary still cannot see the write from the old primary. + assert.eq(null, nodes[1].getDB(dbName).getCollection(collName).findOne({a: 2})); + + // Ensure the stale primary still hasn't committed the write it did that never reached + // the other nodes. + checkDocNotCommitted(nodes[0], {a: 2}); + + jsTest.log("Allow the old primary to finish stepping down and become secondary"); + var res = null; + try { + res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'}); + } catch (e) { + // Expected - once we disable the fail point the stepdown will proceed and it's racy whether + // the stepdown closes all connections before or after the configureFailPoint command + // returns + } + if (res) { + assert.commandWorked(res); + } + rst.waitForState(nodes[0], ReplSetTest.State.SECONDARY); + reconnect(nodes[0]); + + // At this point the former primary will attempt to go into rollback, but the + // 'rollbackHangBeforeStart' will prevent it from doing so. + checkDocNotCommitted(nodes[0], {a: 2}); + checkLog.contains(nodes[0], 'rollback - rollbackHangBeforeStart fail point enabled'); + checkDocNotCommitted(nodes[0], {a: 2}); + + jsTest.log("Allow the original primary to roll back its write and catch up to the new primary"); + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); + + assert.soonNoExcept(function() { + return null == nodes[0].getDB(dbName).getCollection(collName).findOne({a: 2}); + }, "Original primary never rolled back its write"); + + rst.awaitReplication(); + + // Ensure that the old primary got the write that the new primary did and sees it as committed. + assert.neq( + null, + nodes[0].getDB(dbName).getCollection(collName).find({a: 3}).readConcern('majority').next()); + + rst.stopSet(); +}()); diff --git a/jstests/replsets/read_committed_with_catalog_changes.js b/jstests/replsets/read_committed_with_catalog_changes.js new file mode 100644 index 00000000000..dae14da31de --- /dev/null +++ b/jstests/replsets/read_committed_with_catalog_changes.js @@ -0,0 +1,343 @@ +/** + * Test read committed functionality when mixed with catalog changes. Since we don't support + * multiple versions of the catalog, operations that modify the catalog may need to lock out + * committed readers until the modification is in the committed snapshot. + * + * The following replicated operations are tested here: + * - creating a collection in an existing db + * - creating a collection in a new db + * - dropping a collection + * - dropping a db + * - dropping a collection and creating one with the same name + * - dropping a db and creating one with the same name + * - renaming a collection to a new, unused name + * - renaming a collection on top of an existing collection + * - creating a foreground index + * - creating a background index + * - dropping an index + * + * The following non-replicated operations are tested here: + * - repair database + * - reindex collection + * - compact collection + */ + +load("jstests/libs/parallelTester.js"); // For ScopedThread. +load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. + +(function() { + "use strict"; + + // Each test case includes a 'prepare' method that sets up the initial state starting with a + // database that has been dropped, a 'performOp' method that does some operation, and two + // arrays, 'blockedCollections' and 'unblockedCollections', that list the collections that + // should be blocked or unblocked between the time the operation is performed until it is + // committed. If the operation is local only and isn't replicated, the test case should include + // a 'localOnly' field set to true. Test cases are not allowed to touch any databases other than + // the one passed in. + const testCases = { + createCollectionInExistingDB: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + }, + performOp: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + createCollectionInNewDB: { + prepare: function(db) {}, + performOp: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['otherDoesNotExist'], // Only existent collections are blocked. + }, + dropCollection: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert(db.coll.drop()); + }, + blockedCollections: [], + unblockedCollections: ['coll', 'other'], + }, + dropDB: { + prepare: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.dropDatabase()); + }, + blockedCollections: [], + unblockedCollections: ['coll'], + }, + dropAndRecreateCollection: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert(db.coll.drop()); + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + dropAndRecreateDB: { + prepare: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.dropDatabase()); + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['otherDoesNotExist'], + }, + renameCollectionToNewName: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.from.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.from.renameCollection('coll')); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other', 'from' /*doesNotExist*/], + }, + renameCollectionToExistingName: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.from.insert({_id: 'from'})); + assert.writeOK(db.coll.insert({_id: 'coll'})); + }, + performOp: function(db) { + assert.commandWorked(db.from.renameCollection('coll', true)); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other', 'from' /*doesNotExist*/], + }, + createIndexForeground: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.ensureIndex({x: 1}, {background: false})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + createIndexBackground: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.ensureIndex({x: 1}, {background: true})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + dropIndex: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + assert.commandWorked(db.coll.ensureIndex({x: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.dropIndex({x: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + + // Remaining cases are local-only operations. + repairDatabase: { + prepare: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.repairDatabase()); + }, + blockedCollections: ['coll'], + unblockedCollections: ['otherDoesNotExist'], + localOnly: true, + }, + reIndex: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + assert.commandWorked(db.coll.ensureIndex({x: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.reIndex()); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + localOnly: true, + }, + compact: { + // At least on WiredTiger, compact is fully inplace so it doesn't need to block readers. + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + assert.commandWorked(db.coll.ensureIndex({x: 1})); + }, + performOp: function(db) { + var res = db.coll.runCommand('compact', {force: true}); + if (res.code != ErrorCodes.CommandNotSupported) { + // It is fine for a storage engine to support snapshots but not compact. Since + // compact doesn't block any collections we are fine with doing a no-op here. + // Other errors should fail the test. + assert.commandWorked(res); + } + + }, + blockedCollections: [], + unblockedCollections: ['coll', 'other'], + localOnly: true, + }, + }; + + // Assertion helpers. These must get all state as arguments rather than through closure since + // they may be passed in to a ScopedThread. + function assertReadsBlock(coll) { + var res = + coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": 1000}); + assert.commandFailedWithCode(res, + ErrorCodes.ExceededTimeLimit, + "Expected read of " + coll.getFullName() + " to block"); + } + + function assertReadsSucceed(coll, timeoutMs = 1000) { + var res = coll.runCommand('find', + {"readConcern": {"level": "majority"}, "maxTimeMS": timeoutMs}); + assert.commandWorked(res, 'reading from ' + coll.getFullName()); + // Exhaust the cursor to avoid leaking cursors on the server. + new DBCommandCursor(coll.getMongo(), res).itcount(); + } + + // Set up a set and grab things for later. + var name = "read_committed_with_catalog_changes"; + var replTest = + new ReplSetTest({name: name, nodes: 3, nodeOptions: {enableMajorityReadConcern: ''}}); + + if (!startSetIfSupportsReadMajority(replTest)) { + jsTest.log("skipping test since storage engine doesn't support committed reads"); + return; + } + + var nodes = replTest.nodeList(); + var config = { + "_id": name, + "members": [ + {"_id": 0, "host": nodes[0]}, + {"_id": 1, "host": nodes[1], priority: 0}, + {"_id": 2, "host": nodes[2], arbiterOnly: true} + ] + }; + replTest.initiate(config); + + // Get connections. + var primary = replTest.getPrimary(); + var secondary = replTest.liveNodes.slaves[0]; + + // This is the DB that all of the tests will use. + var mainDB = primary.getDB('mainDB'); + + // This DB won't be used by any tests so it should always be unblocked. + var otherDB = primary.getDB('otherDB'); + var otherDBCollection = otherDB.collection; + assert.writeOK( + otherDBCollection.insert({}, {writeConcern: {w: "majority", wtimeout: 60 * 1000}})); + assertReadsSucceed(otherDBCollection); + + for (var testName in testCases) { + jsTestLog('Running test ' + testName); + var test = testCases[testName]; + + const setUpInitialState = function setUpInitialState() { + assert.commandWorked(mainDB.dropDatabase()); + test.prepare(mainDB); + mainDB.getLastError('majority', 60 * 1000); + // Do some sanity checks. + assertReadsSucceed(otherDBCollection); + test.blockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + test.unblockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + }; + + // All operations, whether replicated or not, must become visible automatically as long as + // the secondary is keeping up. + setUpInitialState(); + test.performOp(mainDB); + assertReadsSucceed(otherDBCollection); + test.blockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + test.unblockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + + // Return to the initial state, then stop the secondary from applying new writes to prevent + // them from becoming committed. + setUpInitialState(); + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "alwaysOn"})); + + // If the tested operation isn't replicated, do a write to the side collection before + // performing the operation. This will ensure that the operation happens after an + // uncommitted write which prevents it from immediately being marked as committed. + if (test.localOnly) { + assert.writeOK(otherDBCollection.insert({})); + } + + // Perform the op and ensure that blocked collections block and unblocked ones don't. + test.performOp(mainDB); + assertReadsSucceed(otherDBCollection); + test.blockedCollections.forEach((name) => assertReadsBlock(mainDB[name])); + test.unblockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + + // Use background threads to test that reads that start blocked can complete if the + // operation they are waiting on becomes committed while the read is still blocked. + // We don't do this when testing auth because ScopedThread's don't propagate auth + // credentials. + var threads = jsTest.options().auth ? [] : test.blockedCollections.map((name) => { + // This function must get all inputs as arguments and can't use closure because it + // is used in a ScopedThread. + function bgThread(host, collection, assertReadsSucceed) { + // Use a longer timeout since we expect to block for a little while (at least 2 + // seconds). + assertReadsSucceed(new Mongo(host).getCollection(collection), 30 * 1000); + } + var thread = new ScopedThread( + bgThread, primary.host, mainDB[name].getFullName(), assertReadsSucceed); + thread.start(); + return thread; + }); + sleep(1000); // Give the reads a chance to block. + + try { + // Try the committed read again after sleeping to ensure that it still blocks even if it + // isn't immediately after the operation. + test.blockedCollections.forEach((name) => assertReadsBlock(mainDB[name])); + + // Restart oplog application on the secondary and ensure the blocked collections become + // unblocked. + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "off"})); + mainDB.getLastError("majority", 60 * 1000); + test.blockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + + // Wait for the threads to complete and report any errors encountered from running them. + threads.forEach((thread) => { + thread.join(); + thread.join = () => {}; // Make join a no-op for the finally below. + assert(!thread.hasFailed(), "One of the threads failed. See above for details."); + }); + } finally { + // Make sure we wait for all threads to finish. + threads.forEach(thread => thread.join()); + } + } +}()); diff --git a/jstests/replsets/reconfig_without_increased_queues.js b/jstests/replsets/reconfig_without_increased_queues.js index 8b71817166d..39ef7a7e60c 100644 --- a/jstests/replsets/reconfig_without_increased_queues.js +++ b/jstests/replsets/reconfig_without_increased_queues.js @@ -69,6 +69,16 @@ replTest.awaitSecondaryNodes(); + // We cannot reconfig nodes[2] to have priority 0 if it is currently the primary. After the + // first reconfig, it will be unelectable so this only needs to be done once. + if (replTest.getPrimary() === replTest.nodes[2]) { + jsTestLog("Stepping down node 2 before reconfig"); + assert.throws(function() { + replTest.nodes[2].adminCommand({replSetStepDown: 5, force: true}); + }); + replTest.waitForState(replTest.nodes[2], ReplSetTest.State.SECONDARY); + } + // ** Setup different priorities var c = replTest.getReplSetConfigFromNode(); c.members[0].priority = 99; diff --git a/jstests/replsets/replsetfreeze.js b/jstests/replsets/replsetfreeze.js index aff26f4e20a..dcdc738f43c 100644 --- a/jstests/replsets/replsetfreeze.js +++ b/jstests/replsets/replsetfreeze.js @@ -53,6 +53,10 @@ var r = replTest.initiate(config); var master = replTest.getPrimary(); var secondary = replTest.getSecondary(); +assert.commandFailedWithCode(master.getDB("admin").runCommand({replSetFreeze: 30}), + ErrorCodes.NotSecondary, + 'replSetFreeze should return error when run on primary'); + replTest.awaitSecondaryNodes(); print("2: step down m1"); diff --git a/jstests/replsets/request_primary_stepdown.js b/jstests/replsets/request_primary_stepdown.js new file mode 100644 index 00000000000..02050bd55f4 --- /dev/null +++ b/jstests/replsets/request_primary_stepdown.js @@ -0,0 +1,46 @@ +// 3-node replica set - one arbiter and two electable nodes with different priorities. +// Wait for replica set to stabilize with the higher priority node as primary. +// Step down the high priority node. Wait for the lower priority electable node to become primary. +// Eventually the high priority node will run a priority takeover election to become primary. During +// this election that node should make sure that it does not error in _requestRemotePrimaryStepDown. +(function() { + 'use strict'; + load('jstests/replsets/rslib.js'); + + var name = 'request_primary_stepdown'; + var replSet = new ReplSetTest( + {name: name, nodes: [{rsConfig: {priority: 3}}, {}, {rsConfig: {arbiterOnly: true}}]}); + replSet.startSet(); + var conf = replSet.getReplSetConfig(); + conf.protocolVersion = 0; + replSet.initiate(conf); + + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.awaitSecondaryNodes(); + replSet.awaitReplication(); + var primary = replSet.getPrimary(); + + assert.commandWorked( + replSet.nodes[0].adminCommand({setParameter: 1, logComponentVerbosity: {executor: 4}})); + clearRawMongoProgramOutput(); + + // Primary should step down long enough for election to occur on secondary. + var stepDownException = assert.throws(function() { + var result = primary.adminCommand({replSetStepDown: 70, secondaryCatchUpPeriodSecs: 60}); + print('replSetStepDown did not throw exception but returned: ' + tojson(result)); + }); + assert.neq(-1, + tojson(stepDownException).indexOf('error doing query'), + 'replSetStepDown did not disconnect client'); + + // Wait for node 1 to be promoted to primary after node 0 stepped down. + replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000); + + // Eventually node 0 will stand for election again because it has a higher priorty. + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 100 * 1000); + var logContents = rawMongoProgramOutput(); + assert.eq(logContents.indexOf("stepdown period must be longer than secondaryCatchUpPeriodSecs"), + -1, + "_requestRemotePrimaryStepDown sent an invalid replSetStepDown command"); + +})(); diff --git a/jstests/replsets/resync_with_write_load.js b/jstests/replsets/resync_with_write_load.js index 9d8031a1007..1a782ffacbe 100644 --- a/jstests/replsets/resync_with_write_load.js +++ b/jstests/replsets/resync_with_write_load.js @@ -90,7 +90,7 @@ print("waiting for load generation to finish"); loadGen(); // load must stop before we await replication. -replTest.awaitReplication(60 * 1000); +replTest.awaitReplication(240 * 1000); // Make sure oplogs match try { diff --git a/jstests/replsets/rollback_after_sync_source_selection.js b/jstests/replsets/rollback_after_sync_source_selection.js new file mode 100644 index 00000000000..817da77164e --- /dev/null +++ b/jstests/replsets/rollback_after_sync_source_selection.js @@ -0,0 +1,113 @@ +/* + * This tests that nodes get a new sync source if their sync source rolls back between when it is + * chosen as a sync source and when it is first used. The test sets up a five node replicaset + * and creates a simple rollback scenario. Before node 0 goes into rollback, however, we pause + * the oplog fetcher on node 2, which is syncing from node 0. After the rollback occurs we let + * the oplog fetcher continue and after it gets back its first batch from the sync source it + * realizes that its sync source has rolled back and it errors before getting a new sync source. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var name = "rollback_after_sync_source_selection"; + var collName = "test.coll"; + + var rst = new ReplSetTest({ + name: name, + nodes: [ + {}, + {}, + {rsConfig: {priority: 0}}, + {rsConfig: {arbiterOnly: true}}, + {rsConfig: {arbiterOnly: true}} + ], + useBridge: true + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 0}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Create two partitions: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[0]); + nodes[1].disconnect(nodes[2]); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + + jsTestLog("Do a write that replicates to [0,2,3,4]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Pausing node 2's oplog fetcher before first fetch."); + assert.commandWorked(nodes[2].getDB('admin').runCommand( + {configureFailPoint: 'fetcherHangBeforeStart', mode: 'alwaysOn'})); + syncFrom(nodes[2], nodes[0], rst); + checkLog.contains(nodes[2], 'fetcherHangBeforeStart fail point enabled'); + + jsTestLog("Do a write on partition [0,2,3,4]; it won't replicate due to the failpoint."); + assert.writeOK(nodes[0].getCollection(collName).insert({a: 2})); + + jsTestLog("Repartition to: [0,2] and [1,3,4]."); + nodes[1].reconnect(nodes[3]); + nodes[1].reconnect(nodes[4]); + nodes[3].disconnect(nodes[0]); + nodes[3].disconnect(nodes[2]); + nodes[4].disconnect(nodes[0]); + nodes[4].disconnect(nodes[2]); + + jsTestLog("Ensure that 0 steps down and that 1 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + waitForState(nodes[1], ReplSetTest.State.PRIMARY); + assert.eq(nodes[1], rst.getPrimary()); + + jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition."); + assert.writeOK(nodes[1].getCollection(collName).insert({a: 3})); + + jsTestLog("Remove the partition but maintain that node 2 only talks to node 0."); + var node0RBID = nodes[0].adminCommand('replSetGetRBID').rbid; + nodes[0].reconnect(nodes[1]); + nodes[0].reconnect(nodes[3]); + nodes[0].reconnect(nodes[4]); + + jsTestLog("Wait for node 0 to go into ROLLBACK"); + // Wait for a rollback to happen. + assert.soonNoExcept(function() { + var node0RBIDNew = nodes[0].adminCommand('replSetGetRBID').rbid; + return node0RBIDNew !== node0RBID; + }); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + + // At this point nodes 0 and 1 should have the same data. + assert.neq(null, + nodes[0].getCollection(collName).findOne({a: 0}), + "Node " + nodes[0].host + + " did not contain initial op that should be present on all nodes"); + assert.eq(null, + nodes[0].getCollection(collName).findOne({a: 1}), + "Node " + nodes[0].host + " contained op that should have been rolled back"); + assert.eq(null, + nodes[0].getCollection(collName).findOne({a: 2}), + "Node " + nodes[0].host + " contained op that should have been rolled back"); + assert.neq(null, + nodes[0].getCollection(collName).findOne({a: 3}), + "Node " + nodes[0].host + " did not contain op from after rollback"); + + jsTestLog("Let oplog fetcher continue and error that the sync source rolled back."); + // Turn off failpoint on node 2 to allow it to continue fetching. + assert.commandWorked(nodes[2].getDB('admin').runCommand( + {configureFailPoint: 'fetcherHangBeforeStart', mode: 'off'})); + checkLog.contains(nodes[2], + "Upstream node rolled back after verifying that it had our MinValid point."); + + rst.awaitSecondaryNodes(); + rst.stopSet(); +}()); diff --git a/jstests/replsets/rollback_empty_ns.js b/jstests/replsets/rollback_empty_ns.js deleted file mode 100644 index f6a07319eb4..00000000000 --- a/jstests/replsets/rollback_empty_ns.js +++ /dev/null @@ -1,93 +0,0 @@ -// test that a rollback of an op with empty ns causes a message to be logged -// -// If all data-bearing nodes in a replica set are using an ephemeral storage engine, the set will -// not be able to survive a scenario where all data-bearing nodes are down simultaneously. In such a -// scenario, none of the members will have any data, and upon restart will each look for a member to -// inital sync from, so no primary will be elected. This test induces such a scenario, so cannot be -// run on ephemeral storage engines. -// @tags: [requires_persistence] - -// function to check the logs for an entry -doesEntryMatch = function(array, regex) { - var found = false; - for (i = 0; i < array.length; i++) { - if (regex.test(array[i])) { - found = true; - } - } - return found; -}; - -// set up a set and grab things for later -var name = "rollback_empty_ns"; -var replTest = new ReplSetTest({name: name, nodes: 3}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet(); -replTest.initiate({ - "_id": name, - "members": [ - {"_id": 0, "host": nodes[0], priority: 3}, - {"_id": 1, "host": nodes[1]}, - {"_id": 2, "host": nodes[2], arbiterOnly: true} - ] -}); -var a_conn = conns[0]; -var b_conn = conns[1]; -var AID = replTest.getNodeId(a_conn); -var BID = replTest.getNodeId(b_conn); - -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - -// get master and do an initial write -var master = replTest.getPrimary(); -assert(master === conns[0], "conns[0] assumed to be master"); -assert(a_conn.host === master.host, "a_conn assumed to be master"); -var options = { - writeConcern: {w: 2, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 1}, options)); - -// shut down master -replTest.stop(AID); - -// insert a fake oplog entry with an empty ns -master = replTest.getPrimary(); -assert(b_conn.host === master.host, "b_conn assumed to be master"); -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -// another insert to set minvalid ahead -assert.writeOK(b_conn.getDB(name).foo.insert({x: 123})); -var oplog_entry = b_conn.getDB("local").oplog.rs.find().sort({$natural: -1})[0]; -oplog_entry["ts"] = Timestamp(oplog_entry["ts"].t, oplog_entry["ts"].i + 1); -oplog_entry["ns"] = ""; -assert.writeOK(b_conn.getDB("local").oplog.rs.insert(oplog_entry)); - -// shut down B and bring back the original master -replTest.stop(BID); -replTest.restart(AID); -master = replTest.getPrimary(); -assert(a_conn.host === master.host, "a_conn assumed to be master"); - -// do a write so that B will have to roll back -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 2}, options)); - -// restart B, which should rollback and log a message about not rolling back empty ns'd oplog entry -replTest.restart(BID); -var msg = RegExp("ignoring op on rollback no ns TODO : "); -assert.soon(function() { - try { - var log = b_conn.getDB("admin").adminCommand({getLog: "global"}).log; - return doesEntryMatch(log, msg); - } catch (e) { - return false; - } -}, "Did not see a log entry about skipping the empty ns'd oplog entry during rollback"); - -replTest.stopSet(); diff --git a/jstests/replsets/rollback_empty_o.js b/jstests/replsets/rollback_empty_o.js deleted file mode 100644 index f3468fcde5e..00000000000 --- a/jstests/replsets/rollback_empty_o.js +++ /dev/null @@ -1,93 +0,0 @@ -// test that a rollback of an op with empty o causes a message to be logged -// -// If all data-bearing nodes in a replica set are using an ephemeral storage engine, the set will -// not be able to survive a scenario where all data-bearing nodes are down simultaneously. In such a -// scenario, none of the members will have any data, and upon restart will each look for a member to -// inital sync from, so no primary will be elected. This test induces such a scenario, so cannot be -// run on ephemeral storage engines. -// @tags: [requires_persistence] - -// function to check the logs for an entry -doesEntryMatch = function(array, regex) { - var found = false; - for (i = 0; i < array.length; i++) { - if (regex.test(array[i])) { - found = true; - } - } - return found; -}; - -// set up a set and grab things for later -var name = "rollback_empty_o"; -var replTest = new ReplSetTest({name: name, nodes: 3}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet(); -replTest.initiate({ - "_id": name, - "members": [ - {"_id": 0, "host": nodes[0], priority: 3}, - {"_id": 1, "host": nodes[1]}, - {"_id": 2, "host": nodes[2], arbiterOnly: true} - ] -}); -var a_conn = conns[0]; -var b_conn = conns[1]; -var AID = replTest.getNodeId(a_conn); -var BID = replTest.getNodeId(b_conn); - -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - -// get master and do an initial write -var master = replTest.getPrimary(); -assert(master === conns[0], "conns[0] assumed to be master"); -assert(a_conn.host === master.host, "a_conn assumed to be master"); -var options = { - writeConcern: {w: 2, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 1}, options)); - -// shut down master -replTest.stop(AID); - -// insert a fake oplog entry with an empty o -master = replTest.getPrimary(); -assert(b_conn.host === master.host, "b_conn assumed to be master"); -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -// another insert to set minvalid ahead -assert.writeOK(b_conn.getDB(name).foo.insert({x: 123})); -var oplog_entry = b_conn.getDB("local").oplog.rs.find().sort({$natural: -1})[0]; -oplog_entry["ts"] = Timestamp(oplog_entry["ts"].t, oplog_entry["ts"].i + 1); -oplog_entry["o"] = {}; -assert.writeOK(b_conn.getDB("local").oplog.rs.insert(oplog_entry)); - -// shut down B and bring back the original master -replTest.stop(BID); -replTest.restart(AID); -master = replTest.getPrimary(); -assert(a_conn.host === master.host, "a_conn assumed to be master"); - -// do a write so that B will have to roll back -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 2}, options)); - -// restart B, which should rollback and log a message about not rolling back empty o'd oplog entry -replTest.restart(BID); -var msg = RegExp("ignoring op on rollback : "); -assert.soon(function() { - try { - var log = b_conn.getDB("admin").adminCommand({getLog: "global"}).log; - return doesEntryMatch(log, msg); - } catch (e) { - return false; - } -}, "Did not see a log entry about skipping the empty o'd oplog entry during rollback"); - -replTest.stopSet(); diff --git a/jstests/replsets/rollback_empty_o2.js b/jstests/replsets/rollback_empty_o2.js deleted file mode 100644 index 56eb8512575..00000000000 --- a/jstests/replsets/rollback_empty_o2.js +++ /dev/null @@ -1,94 +0,0 @@ -// test that a rollback of an update with empty o2 causes a message to be logged -// -// If all data-bearing nodes in a replica set are using an ephemeral storage engine, the set will -// not be able to survive a scenario where all data-bearing nodes are down simultaneously. In such a -// scenario, none of the members will have any data, and upon restart will each look for a member to -// inital sync from, so no primary will be elected. This test induces such a scenario, so cannot be -// run on ephemeral storage engines. -// @tags: [requires_persistence] - -// function to check the logs for an entry -doesEntryMatch = function(array, regex) { - var found = false; - for (i = 0; i < array.length; i++) { - if (regex.test(array[i])) { - found = true; - } - } - return found; -}; - -// set up a set and grab things for later -var name = "rollback_empty_o2"; -var replTest = new ReplSetTest({name: name, nodes: 3}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet(); -replTest.initiate({ - "_id": name, - "members": [ - {"_id": 0, "host": nodes[0], priority: 3}, - {"_id": 1, "host": nodes[1]}, - {"_id": 2, "host": nodes[2], arbiterOnly: true} - ] -}); -var a_conn = conns[0]; -var b_conn = conns[1]; -var AID = replTest.getNodeId(a_conn); -var BID = replTest.getNodeId(b_conn); - -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - -// get master and do an initial write -var master = replTest.getPrimary(); -assert(master === conns[0], "conns[0] assumed to be master"); -assert(a_conn.host === master.host, "a_conn assumed to be master"); -var options = { - writeConcern: {w: 2, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 1}, options)); - -// shut down master -replTest.stop(AID); - -// insert a fake oplog entry with an empty o2 -master = replTest.getPrimary(); -assert(b_conn.host === master.host, "b_conn assumed to be master"); -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -// another insert to set minvalid ahead -assert.writeOK(b_conn.getDB(name).foo.insert({x: 123})); -var oplog_entry = b_conn.getDB("local").oplog.rs.find().sort({$natural: -1})[0]; -oplog_entry["ts"] = Timestamp(oplog_entry["ts"].t, oplog_entry["ts"].i + 1); -oplog_entry["op"] = "u"; -oplog_entry["o2"] = {}; -assert.writeOK(b_conn.getDB("local").oplog.rs.insert(oplog_entry)); - -// shut down B and bring back the original master -replTest.stop(BID); -replTest.restart(AID); -master = replTest.getPrimary(); -assert(a_conn.host === master.host, "a_conn assumed to be master"); - -// do a write so that B will have to roll back -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 2}, options)); - -// restart B, which should rollback and log a message about not rolling back empty o2'd oplog entry -replTest.restart(BID); -var msg = RegExp("ignoring op on rollback : "); -assert.soon(function() { - try { - var log = b_conn.getDB("admin").adminCommand({getLog: "global"}).log; - return doesEntryMatch(log, msg); - } catch (e) { - return false; - } -}, "Did not see a log entry about skipping the empty o2'd oplog entry during rollback"); - -replTest.stopSet(); diff --git a/jstests/replsets/rollback_too_new.js b/jstests/replsets/rollback_too_new.js index 5619df42e18..c7fe4fa2320 100644 --- a/jstests/replsets/rollback_too_new.js +++ b/jstests/replsets/rollback_too_new.js @@ -9,6 +9,8 @@ (function() { "use strict"; + load("jstests/replsets/rslib.js"); + // set up a set and grab things for later var name = "rollback_too_new"; var replTest = new ReplSetTest({name: name, nodes: 3}); @@ -43,9 +45,22 @@ replTest.stop(CID); - // do one write to master - // in order to trigger a rollback on C - assert.writeOK(master.getDB(name).foo.insert({x: 2}, options)); + // We bump the term to make sure node 0's oplog is ahead of node 2's. + var term = getLatestOp(conns[0]).t; + try { + assert.commandWorked(conns[0].adminCommand({replSetStepDown: 1, force: true})); + } catch (e) { + if (e.message.indexOf("error doing query: failed") < 0) { + throw e; + } + } + + // After stepping down due to the higher term, it will eventually get reelected. + replTest.waitForState(conns[0], ReplSetTest.State.PRIMARY); + // Wait for the node to increase its term. + assert.soon(function() { + return getLatestOp(conns[0]).t > term; + }); // Node C should connect to new master as a sync source because chaining is disallowed. // C is ahead of master but it will still connect to it. diff --git a/jstests/replsets/rollback_with_socket_error_then_steady_state.js b/jstests/replsets/rollback_with_socket_error_then_steady_state.js new file mode 100644 index 00000000000..25a5c863970 --- /dev/null +++ b/jstests/replsets/rollback_with_socket_error_then_steady_state.js @@ -0,0 +1,119 @@ +// This test causes node 2 to enter rollback and then fail with a SocketException before updating +// MinValid or altering durable state in any way. It will then choose a sync source from which it +// is able to stitch the oplog and therefore doesn't need to roll back. Prior to SERVER-27282, the +// node would be "stuck" with state=ROLLBACK while it was doing steady-state replication, with no +// way to reach SECONDARY without restarting the process. +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var collName = "test.coll"; + var counter = 0; + + var rst = new ReplSetTest({ + name: 'rollback_with_socket_error_then_steady_state', + nodes: [ + // Primary flops between nodes 0 and 1. + {}, + {}, + // Node 2 is the node under test. + {rsConfig: {priority: 0}}, + // Arbiters to sway elections. + {rsConfig: {arbiterOnly: true}}, + {rsConfig: {arbiterOnly: true}} + ], + useBridge: true + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 3, wtimeout: 5 * 60 * 1000}})); + + jsTestLog("Create two partitions: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[0]); + nodes[1].disconnect(nodes[2]); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + + jsTestLog("Do a write that is replicated to [0,2,3,4]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: 5 * 60 * 1000}})); + + jsTestLog("Repartition to: [0,2] and [1,3,4]."); + nodes[1].reconnect(nodes[3]); + nodes[1].reconnect(nodes[4]); + nodes[3].disconnect(nodes[0]); + nodes[3].disconnect(nodes[2]); + nodes[4].disconnect(nodes[0]); + nodes[4].disconnect(nodes[2]); + + jsTestLog("Ensure that 0 steps down and that 1 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + waitForState(nodes[1], ReplSetTest.State.PRIMARY); + assert.eq(nodes[1], rst.getPrimary()); + + jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition."); + assert.writeOK(nodes[1].getCollection(collName).insert({a: counter++})); + + // Turn on failpoint on node 2 to pause rollback before doing anything. + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); + + jsTestLog("Repartition to: [0] and [1,2,3,4]."); + nodes[2].disconnect(nodes[0]); + nodes[2].reconnect(nodes[1]); + nodes[2].reconnect(nodes[3]); + nodes[2].reconnect(nodes[4]); + + jsTestLog("Wait for node 2 to decide to go into ROLLBACK and start syncing from node 1."); + // Since nodes 1 and 2 have now diverged, node 2 should go into rollback. The failpoint will + // stop it from actually transitioning to rollback, so the checkLog bellow will ensure that we + // have decided to rollback, but haven't actually started yet. + rst.awaitSyncSource(nodes[2], nodes[1]); + + jsTestLog("Wait for failpoint on node 2 to pause rollback before it starts"); + // Wait for fail point message to be logged. + checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeStart fail point enabled'); + + jsTestLog("Repartition to: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + nodes[2].disconnect(nodes[1]); + nodes[2].reconnect(nodes[0]); + nodes[3].reconnect(nodes[0]); + nodes[3].reconnect(nodes[2]); + nodes[4].reconnect(nodes[0]); + nodes[4].reconnect(nodes[2]); + + // Turn off failpoint on node 2 to allow rollback against node 1 to fail with a network error. + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); + + // Make node 0 ahead of node 2 again so node 2 will pick it as a sync source. + + jsTestLog("waiting for node 0 to be primary"); + waitForState(nodes[1], ReplSetTest.State.SECONDARY); + waitForState(nodes[0], ReplSetTest.State.PRIMARY); + assert.eq(nodes[0], rst.getPrimary()); + + jsTestLog("w:2 write to node 0 (replicated to node 2)"); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: 5 * 60 * 1000}})); + + // At this point node 2 has failed rollback before making any durable changes, including writing + // to minValid. That means that it is free to pick any sync source and will pick node 0 where it + // can pick up where it left off without rolling back. Ensure that it is able to reach SECONDARY + // and doesn't do steady-state replication in ROLLBACK state. + jsTestLog("Wait for node 2 to go into SECONDARY"); + assert.neq(nodes[2].adminCommand('replSetGetStatus').myState, + ReplSetTest.State.ROLLBACK, + "node 2 is doing steady-state replication with state=ROLLBACK!"); + waitForState(nodes[2], ReplSetTest.State.SECONDARY); +}()); diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js index 48e934a88d8..b97f79c00d1 100644 --- a/jstests/replsets/rslib.js +++ b/jstests/replsets/rslib.js @@ -1,11 +1,50 @@ +var syncFrom; var wait, occasionally, reconnect, getLatestOp, waitForAllMembers, reconfig, awaitOpTime; var waitUntilAllNodesCaughtUp; +var waitForState; +var awaitRSClientHosts; +var getLastOpTime; +var startSetIfSupportsReadMajority; (function() { "use strict"; + load("jstests/libs/write_concern_util.js"); + var count = 0; var w = 0; + /** + * A wrapper around `replSetSyncFrom` to ensure that the desired sync source is ahead of the + * syncing node so that the syncing node can choose to sync from the desired sync source. + * It first stops replication on the syncing node so that it can do a write on the desired + * sync source and make sure it's ahead. When replication is restarted, the desired sync + * source will be a valid sync source for the syncing node. + */ + syncFrom = function(syncingNode, desiredSyncSource, rst) { + jsTestLog("Forcing " + syncingNode.name + " to sync from " + desiredSyncSource.name); + + // Ensure that 'desiredSyncSource' doesn't already have the dummy write sitting around from + // a previous syncFrom attempt. + var dummyName = "dummyForSyncFrom"; + rst.getPrimary().getDB(dummyName).getCollection(dummyName).drop(); + assert.soonNoExcept(function() { + return desiredSyncSource.getDB(dummyName).getCollection(dummyName).findOne() == null; + }); + + stopServerReplication(syncingNode); + + assert.writeOK(rst.getPrimary().getDB(dummyName).getCollection(dummyName).insert({a: 1})); + // Wait for 'desiredSyncSource' to get the dummy write we just did so we know it's + // definitely ahead of 'syncingNode' before we call replSetSyncFrom. + assert.soonNoExcept(function() { + return desiredSyncSource.getDB(dummyName).getCollection(dummyName).findOne({a: 1}); + }); + + assert.commandWorked(syncingNode.adminCommand({replSetSyncFrom: desiredSyncSource.name})); + restartServerReplication(syncingNode); + rst.awaitSyncSource(syncingNode, desiredSyncSource); + }; + wait = function(f, msg) { w++; var n = 0; @@ -192,4 +231,141 @@ var waitUntilAllNodesCaughtUp; timeout); }; + /** + * Waits for the given node to reach the given state, ignoring network errors. + */ + waitForState = function(node, state) { + assert.soonNoExcept(function() { + assert.commandWorked(node.adminCommand( + {replSetTest: 1, waitForMemberState: state, timeoutMillis: 60 * 1000 * 5})); + return true; + }); + }; + + /** + * Starts each node in the given replica set if the storage engine supports readConcern + *'majority'. + * Returns true if the replica set was started successfully and false otherwise. + * + * @param replSetTest - The instance of {@link ReplSetTest} to start + * @param options - The options passed to {@link ReplSetTest.startSet} + */ + startSetIfSupportsReadMajority = function(replSetTest, options) { + try { + replSetTest.startSet(options); + } catch (e) { + var conn = MongoRunner.runMongod(); + if (!conn.getDB("admin").serverStatus().storageEngine.supportsCommittedReads) { + MongoRunner.stopMongod(conn); + return false; + } + throw e; + } + return true; + }; + + /** + * Waits for the specified hosts to enter a certain state. + */ + awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { + var hostCount = host.length; + if (hostCount) { + for (var i = 0; i < hostCount; i++) { + awaitRSClientHosts(conn, host[i], hostOk, rs); + } + + return; + } + + timeout = timeout || 5 * 60 * 1000; + + if (hostOk == undefined) + hostOk = { + ok: true + }; + if (host.host) + host = host.host; + if (rs) + rs = rs.name; + + print("Awaiting " + host + " to be " + tojson(hostOk) + " for " + conn + " (rs: " + rs + + ")"); + + var tests = 0; + + assert.soon(function() { + var rsClientHosts = conn.adminCommand('connPoolStats').replicaSets; + if (tests++ % 10 == 0) { + printjson(rsClientHosts); + } + + for (var rsName in rsClientHosts) { + if (rs && rs != rsName) + continue; + + for (var i = 0; i < rsClientHosts[rsName].hosts.length; i++) { + var clientHost = rsClientHosts[rsName].hosts[i]; + if (clientHost.addr != host) + continue; + + // Check that *all* host properties are set correctly + var propOk = true; + for (var prop in hostOk) { + // Use special comparator for tags because isMaster can return the fields in + // different order. The fields of the tags should be treated like a set of + // strings and 2 tags should be considered the same if the set is equal. + if (prop == 'tags') { + if (!clientHost.tags) { + propOk = false; + break; + } + + for (var hostTag in hostOk.tags) { + if (clientHost.tags[hostTag] != hostOk.tags[hostTag]) { + propOk = false; + break; + } + } + + for (var clientTag in clientHost.tags) { + if (clientHost.tags[clientTag] != hostOk.tags[clientTag]) { + propOk = false; + break; + } + } + + continue; + } + + if (isObject(hostOk[prop])) { + if (!friendlyEqual(hostOk[prop], clientHost[prop])) { + propOk = false; + break; + } + } else if (clientHost[prop] != hostOk[prop]) { + propOk = false; + break; + } + } + + if (propOk) { + return true; + } + } + } + + return false; + }, 'timed out waiting for replica set client to recognize hosts', timeout); + }; + + /** + * Returns the last opTime of the connection based from replSetGetStatus. Can only + * be used on replica set nodes. + */ + getLastOpTime = function(conn) { + var replSetStatus = + assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1})); + var connStatus = replSetStatus.members.filter(m => m.self)[0]; + return connStatus.optime; + }; }()); diff --git a/jstests/replsets/server8070.js b/jstests/replsets/server8070.js index e91e95e99a4..5d8435a656e 100644 --- a/jstests/replsets/server8070.js +++ b/jstests/replsets/server8070.js @@ -1,3 +1,5 @@ +load("jstests/replsets/rslib.js"); + // Test for SERVER-8070: Flush buffer before changing sync targets to prevent unnecessary rollbacks // This test writes 50 ops to one secondary's data (member2) and 25 ops to the other secondary's // data (member3), then puts 50 more ops in member3's buffer and makes sure that member3 doesn't try @@ -40,8 +42,9 @@ master.getDB("foo").bar.insert({x: 1}); replSet.awaitReplication(); jsTest.log("Make sure 2 & 3 are syncing from the primary"); -member2.adminCommand({replSetSyncFrom: getHostName() + ":" + replSet.ports[0]}); -member3.adminCommand({replSetSyncFrom: getHostName() + ":" + replSet.ports[0]}); +assert.eq(master, replSet.nodes[0]); +syncFrom(replSet.nodes[1], master, replSet); +syncFrom(replSet.nodes[2], master, replSet); jsTest.log("Stop 2's replication"); member2.runCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'}); @@ -100,8 +103,8 @@ assert(syncingTo !== getHostName() + ":" + replSet.ports[1], "node 3 is syncing jsTest.log("Pause 3's bgsync thread"); var rsBgSyncProduceResult3 = - member3.runCommand({configureFailPoint: 'rsBgSyncProduce', mode: 'alwaysOn'}); -assert.eq(1, rsBgSyncProduceResult3.ok, "member 3 rsBgSyncProduce admin command failed"); + member3.runCommand({configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'}); +assert.eq(1, rsBgSyncProduceResult3.ok, "member 3 stopReplProducer admin command failed"); // count documents in member 3 assert.eq(26, @@ -123,7 +126,7 @@ assert.soon(function() { }, "Replication member 3 did not apply ops 25-75"); jsTest.log("Start 3's bgsync thread"); -member3.runCommand({configureFailPoint: 'rsBgSyncProduce', mode: 'off'}); +member3.runCommand({configureFailPoint: 'stopReplProducer', mode: 'off'}); jsTest.log("Node 3 shouldn't hit rollback"); var end = (new Date()).getTime() + 10000; diff --git a/jstests/replsets/shutdown_primary.js b/jstests/replsets/shutdown_primary.js new file mode 100644 index 00000000000..66d429dec89 --- /dev/null +++ b/jstests/replsets/shutdown_primary.js @@ -0,0 +1,68 @@ +/** + * Test that the shutdown command called on a primary node waits for a majority of secondaries to + * catch up before taking effect, and will fail otherwise. + * + * 1. Initiate a 3-node replica set + * 2. Block replication to secondaries + * 3. Write to primary + * 4. Try to shut down primary and expect failure + * 5. Try to shut down primary in a parallel shell and expect success + * 6. Resume replication on secondaries + * 7. Try to create a new connection to the shut down primary and expect an error + * + */ +(function() { + load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries, + // restartReplicationOnSecondaries + + "use strict"; + + var name = "shutdown_primary"; + + var replTest = new ReplSetTest({name: name, nodes: 3}); + replTest.startSet(); + replTest.initiate(); + + var primary = replTest.getPrimary(); + var testDB = primary.getDB(name); + var timeout = 5 * 60 * 1000; + assert.writeOK(testDB.foo.insert({x: 1}, {writeConcern: {w: 3, wtimeout: timeout}})); + + jsTestLog("Blocking replication to secondaries."); + stopReplicationOnSecondaries(replTest); + + jsTestLog("Executing write to primary."); + assert.writeOK(testDB.foo.insert({x: 2})); + + jsTestLog("Attempting to shut down primary."); + assert.commandFailedWithCode(primary.adminCommand({shutdown: 1}), + ErrorCodes.ExceededTimeLimit, + "shut down did not fail with 'ExceededTimeLimit'"); + + jsTestLog("Verifying primary did not shut down."); + assert.writeOK(testDB.foo.insert({x: 3})); + + jsTestLog("Shutting down primary in a parallel shell"); + var awaitShell = startParallelShell(function() { + db.adminCommand({shutdown: 1, timeoutSecs: 60}); + }, primary.port); + + jsTestLog("Resuming replication."); + restartReplicationOnSecondaries(replTest); + + jsTestLog("Verifying primary shut down and cannot be connected to."); + // Successfully starting shutdown throws a network error. + var exitCode = awaitShell({checkExitSuccess: false}); + assert.neq(0, exitCode, "expected shutdown to close the shell's connection"); + assert.soonNoExcept(function() { + // The parallel shell exits while shutdown is in progress, and if this happens early enough, + // the primary can still accept connections despite successfully starting to shutdown. + // So, retry connecting until connections cannot be established and an error is thrown. + assert.throws(function() { + new Mongo(primary.host); + }); + return true; + }, "expected primary node to shut down and not be connectable"); + + replTest.stopSet(); +})(); diff --git a/jstests/replsets/slavedelay3.js b/jstests/replsets/slavedelay3.js index 2ce6e9b2a80..dc3bb0786d9 100644 --- a/jstests/replsets/slavedelay3.js +++ b/jstests/replsets/slavedelay3.js @@ -29,12 +29,7 @@ nodes[0].disconnect(nodes[2]); master.foo.insert({x: 1}); -assert.commandWorked(nodes[1].getDB("admin").runCommand({"replSetSyncFrom": nodes[0].host})); -var res; -assert.soon(function() { - res = nodes[1].getDB("admin").runCommand({"replSetGetStatus": 1}); - return res.syncingTo === nodes[0].host; -}, "node 4 failed to start chaining: " + tojson(res)); +syncFrom(nodes[1], nodes[0], replTest); // make sure the record still appears in the remote slave assert.soon(function() { diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js index 4b736b3b93a..5a8388da4d4 100644 --- a/jstests/replsets/stepdown.js +++ b/jstests/replsets/stepdown.js @@ -28,40 +28,66 @@ print("\ndo a write"); assert.writeOK(master.getDB("foo").bar.insert({x: 1})); replTest.awaitReplication(); -// lock secondaries -print("\nlock secondaries"); -replTest.liveNodes.slaves.forEach(function(slave) { - printjson(assert.commandWorked(slave.getDB("admin").runCommand({fsync: 1, lock: 1}))); -}); - -print("\nwaiting several seconds before stepdown"); - -sleep(2000); - -for (var i = 0; i < 11; i++) { - // do another write - assert.writeOK(master.getDB("foo").bar.insert({x: i})); - sleep(1000); +// In the event of any error, we have to unlock any nodes that we have fsyncLocked. +function unlockNodes(nodes) { + jsTestLog('Unlocking nodes: ' + tojson(nodes)); + nodes.forEach(function(node) { + try { + jsTestLog('Unlocking node: ' + node); + assert.commandWorked(node.getDB("admin").fsyncUnlock()); + } catch (e) { + jsTestLog('Failed to unlock node: ' + node + ': ' + tojson(e) + + '. Ignoring unlock error and moving on to next node.'); + } + }); } -print("\n do stepdown that should not work"); - -// this should fail, so we don't need to try/catch -printjson(assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 10}))); - -print("\n do stepdown that should work"); -assert.throws(function() { - assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 50, force: true})); -}); - -var r2 = assert.commandWorked(master.getDB("admin").runCommand({ismaster: 1})); -assert.eq(r2.ismaster, false); -assert.eq(r2.secondary, true); +var lockedNodes = []; +try { + // lock secondaries + jsTestLog('Locking nodes: ' + tojson(replTest.liveNodes.slaves)); + replTest.liveNodes.slaves.forEach(function(node) { + jsTestLog('Locking node: ' + node); + jsTestLog( + 'fsync lock ' + node + ' result: ' + + tojson(assert.commandWorked(node.getDB("admin").runCommand({fsync: 1, lock: 1})))); + lockedNodes.push(node); + }); + + jsTestLog('Stepping down primary: ' + master); + + for (var i = 0; i < 11; i++) { + // do another write + assert.writeOK(master.getDB("foo").bar.insert({x: i})); + } -print("\nunlock"); -replTest.liveNodes.slaves.forEach(function(slave) { - printjson(assert.commandWorked(slave.getDB("admin").fsyncUnlock())); -}); + jsTestLog('Do stepdown of primary ' + master + ' that should not work'); + + // this should fail, so we don't need to try/catch + jsTestLog( + 'Step down ' + master + ' expected error: ' + + tojson(assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 10})))); + + // The server will disconnect the client on a successful forced stepdown so we use the + // presence of an exception to confirm the forced stepdown result. + jsTestLog('Do stepdown of primary ' + master + ' that should work'); + var exceptionFromForcedStepDown = assert.throws(function() { + master.getDB("admin") + .runCommand({replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true}); + }); + jsTestLog('Forced stepdown ' + master + ' expected failure: ' + + tojson(exceptionFromForcedStepDown)); + + jsTestLog('Checking isMaster on ' + master); + var r2 = assert.commandWorked(master.getDB("admin").runCommand({ismaster: 1})); + jsTestLog('Result from running isMaster on ' + master + ': ' + tojson(r2)); + assert.eq(r2.ismaster, false); + assert.eq(r2.secondary, true); +} catch (e) { + throw e; +} finally { + unlockNodes(lockedNodes); +} print("\nreset stepped down time"); assert.commandWorked(master.getDB("admin").runCommand({replSetFreeze: 0})); diff --git a/jstests/replsets/stepdown_needs_electable_secondary.js b/jstests/replsets/stepdown_needs_electable_secondary.js new file mode 100644 index 00000000000..9a59ff0fd58 --- /dev/null +++ b/jstests/replsets/stepdown_needs_electable_secondary.js @@ -0,0 +1,143 @@ +/** + * Test to ensure that replSetStepDown called on a primary will only succeed if a majority of nodes + * are caught up to it and that at least one node in this majority is electable. Tests this with a + * 5 node replica set. + * + * 1. Initiate a 5-node replica set + * 2. Disable replication to all secondaries + * 3. Execute some writes on primary + * 4. Try to step down primary and expect to fail + * 5. Enable replication to one unelectable secondary, secondary B + * 6. Await replication to secondary B by executing primary write with writeConcern:2 + * 7. Try to step down primary and expect failure + * 8. Enable replication to a different unelectable secondary, secondary C + * 9. Await replication to secondary C by executing primary write with writeConcern:3 + * 10. Try to step down primary and expect failure + * 11. Enable replication to an electable secondary, secondary A + * 12. Await replication to secondary A by executing primary write with writeConcern:4 + * 13. Try to step down primary and expect success + * 14. Assert that original primary is now a secondary + * + */ +(function() { + load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries, + // restartServerReplication, + // restartReplSetReplication + + 'use strict'; + + var name = 'stepdown_needs_electable_secondary'; + + var replTest = new ReplSetTest({name: name, nodes: 5}); + var nodes = replTest.nodeList(); + + replTest.startSet(); + replTest.initiate({ + "_id": name, + "members": [ + {"_id": 0, "host": nodes[0]}, + {"_id": 1, "host": nodes[1]}, + {"_id": 2, "host": nodes[2]}, + {"_id": 3, "host": nodes[3], "priority": 0}, // unelectable + {"_id": 4, "host": nodes[4], "priority": 0} // unelectable + ], + "settings": {"chainingAllowed": false} + }); + + function assertStepDownFailsWithExceededTimeLimit(node) { + assert.commandFailedWithCode( + node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), + ErrorCodes.ExceededTimeLimit, + "step down did not fail with 'ExceededTimeLimit'"); + } + + function assertStepDownSucceeds(node) { + assert.throws(function() { + node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); + }); + } + + var primary = replTest.getPrimary(); + + jsTestLog("Blocking writes to all secondaries."); + stopReplicationOnSecondaries(replTest); + + jsTestLog("Doing a write to primary."); + var testDB = replTest.getPrimary().getDB('testdb'); + var coll = testDB.stepdown_needs_electable_secondary; + var timeout = 5 * 60 * 1000; + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 1, wtimeout: timeout}})); + + // Try to step down with only the primary caught up (1 node out of 5). + // stepDown should fail. + jsTestLog("Trying to step down primary with only 1 node out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // Get the two unelectable secondaries + var secondaryB_unelectable = replTest.nodes[3]; + var secondaryC_unelectable = replTest.nodes[4]; + + // Get an electable secondary + var secondaryA_electable = replTest.getSecondaries().find(function(s) { + var nodeId = replTest.getNodeId(s); + return (nodeId !== 3 && nodeId !== 4); // nodes 3 and 4 are set to be unelectable + }); + + // Enable writes to Secondary B (unelectable). Await replication. + // (2 out of 5 nodes caught up, 0 electable) + // stepDown should fail due to no caught up majority. + jsTestLog("Re-enabling writes to unelectable secondary: node #" + + replTest.getNodeId(secondaryB_unelectable) + ", " + secondaryB_unelectable); + restartServerReplication(secondaryB_unelectable); + + // Wait for this secondary to catch up by issuing a write that must be replicated to 2 nodes + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 2, wtimeout: timeout}})); + + // Try to step down and fail + jsTestLog("Trying to step down primary with only 2 nodes out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // Enable writes to Secondary C (unelectable). Await replication. + // (3 out of 5 nodes caught up, 0 electable) + // stepDown should fail due to caught up majority without electable node. + jsTestLog("Re-enabling writes to unelectable secondary: node #" + + replTest.getNodeId(secondaryC_unelectable) + ", " + secondaryC_unelectable); + restartServerReplication(secondaryC_unelectable); + + // Wait for this secondary to catch up by issuing a write that must be replicated to 3 nodes + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 3, wtimeout: timeout}})); + + // Try to step down and fail + jsTestLog("Trying to step down primary with a caught up majority that " + + "doesn't contain an electable node."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // Enable writes to Secondary A (electable). Await replication. + // (4 out of 5 nodes caught up, 1 electable) + // stepDown should succeed due to caught up majority containing an electable node. + jsTestLog("Re-enabling writes to electable secondary: node #" + + replTest.getNodeId(secondaryA_electable) + ", " + secondaryA_electable); + restartServerReplication(secondaryA_electable); + + // Wait for this secondary to catch up by issuing a write that must be replicated to 4 nodes + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 4, wtimeout: timeout}})); + + // Try to step down. We expect success, so catch the exception thrown by 'replSetStepDown'. + jsTestLog("Trying to step down primary with a caught up majority that " + + "does contain an electable node."); + + assertStepDownSucceeds(primary); + + // Make sure that original primary has transitioned to SECONDARY state + jsTestLog("Wait for PRIMARY " + primary.host + " to completely step down."); + replTest.waitForState(primary, ReplSetTest.State.SECONDARY); + + // Disable all fail points for clean shutdown + restartReplSetReplication(replTest); + replTest.stopSet(); + +}()); diff --git a/jstests/replsets/stepdown_needs_majority.js b/jstests/replsets/stepdown_needs_majority.js new file mode 100644 index 00000000000..51c6dac4def --- /dev/null +++ b/jstests/replsets/stepdown_needs_majority.js @@ -0,0 +1,110 @@ +/** + * Test to ensure that 'replSetStepDown' called on a primary will fail if an electable node is + * caught up to it but less than a majority of nodes are caught up to it. Additionally tests that + * step down will then succeed once a majority has caught up. Tests this with a 5 node replica set. + * + * 1. Initiate a 5-node replica set + * 2. Disable replication to all secondaries + * 3. Execute a write on primary with writeConcern:1 + * 4. Try to step down primary and expect to fail + * 5. Enable replication to one secondary (Secondary A) + * 6. Await replication to Secondary A by executing primary write with writeConcern:2 + * 7. Try to step down primary and expect failure + * 8. Enable replication to a different secondary (Secondary B) + * 9. Await replication to Secondary B by executing primary write with writeConcern:3 + * 10. Try to step down primary and expect success + * + */ +(function() { + load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries, // + // restartServerReplication, + // restartReplSetReplication + + 'use strict'; + + function assertStepDownFailsWithExceededTimeLimit(node) { + assert.commandFailedWithCode( + node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), + ErrorCodes.ExceededTimeLimit, + "step down did not fail with 'ExceededTimeLimit'"); + } + + function assertStepDownSucceeds(node) { + assert.throws(function() { + node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); + }); + } + + function nodeIdStr(repltest, node) { + return "node #" + repltest.getNodeId(node) + ", " + node.host; + } + + // + // Test setup + // + var name = 'stepdown_needs_majority'; + var replTest = new ReplSetTest({name: name, nodes: 5, settings: {chainingAllowed: false}}); + + replTest.startSet(); + replTest.initiate(); + + var primary = replTest.getPrimary(); + var testDB = primary.getDB('testdb'); + var coll = testDB[name]; + var dummy_doc = { + "dummy_key": "dummy_val" + }; + var timeout = 5 * 60 * 1000; + + // + // Block writes to all secondaries + // + jsTestLog("Blocking writes to all secondaries."); + stopReplicationOnSecondaries(replTest); + + // + // Write to the primary and attempt stepdown + // + jsTestLog("Issuing a write to the primary(" + primary.host + ") with write_concern:1"); + assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 1, wtimeout: timeout}})); + + jsTestLog("Trying to step down primary with only 1 node out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // + // Re-enable writes to Secondary A and attempt stepdown + // + var secondaryA = replTest.getSecondaries()[0]; + jsTestLog("Reenabling writes to one secondary (" + nodeIdStr(replTest, secondaryA) + ")"); + restartServerReplication(secondaryA); + + jsTestLog("Issuing a write to the primary with write_concern:2"); + assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 2, wtimeout: timeout}})); + + jsTestLog("Trying to step down primary with only 2 nodes out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // + // Re-enable writes to Secondary B and attempt stepdown + // + var secondaryB = replTest.getSecondaries()[1]; + jsTestLog("Reenabling writes to another secondary (" + nodeIdStr(replTest, secondaryB) + ")"); + restartServerReplication(secondaryB); + + jsTestLog("Issuing a write to the primary with write_concern:3"); + assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 3, wtimeout: timeout}})); + + jsTestLog("Trying to step down primary with 3 nodes out of 5 caught up."); + assertStepDownSucceeds(primary); + + jsTestLog("Waiting for PRIMARY(" + primary.host + ") to step down & become SECONDARY."); + replTest.waitForState(primary, ReplSetTest.State.SECONDARY); + + // + // Disable failpoints and stop replica set + // + jsTestLog("Disabling all fail points to allow for clean shutdown"); + restartReplSetReplication(replTest); + replTest.stopSet(); + +}()); diff --git a/jstests/replsets/stepdown_wrt_electable.js b/jstests/replsets/stepdown_wrt_electable.js deleted file mode 100644 index c929f2a2c56..00000000000 --- a/jstests/replsets/stepdown_wrt_electable.js +++ /dev/null @@ -1,40 +0,0 @@ -// Test that replSetStepDown filters out non-electable nodes -var replTest = new ReplSetTest({name: 'testSet', nodes: 2}); -var nodes = replTest.startSet(); - -// setup config -var c = replTest.getReplSetConfig(); -c.members[1].priority = 0; // not electable -replTest.initiate(c); - -var master = replTest.getPrimary(); -var testDB = master.getDB('test'); -var firstPrimary = testDB.isMaster().primary; - -// do a write to allow stepping down of the primary; -// otherwise, the primary will refuse to step down -testDB.foo.insert({x: 1}); -replTest.awaitReplication(); - -// stepdown should fail since there is no-one to elect within 10 secs -testDB.adminCommand({replSetStepDown: 5}); -assert(master.getDB("a").isMaster().ismaster, "not master"); - -// step down the primary asyncronously so it doesn't kill this test -var wait = startParallelShell("db.adminCommand({replSetStepDown:1000, force:true})", master.port); -var exitCode = wait({checkExitSuccess: false}); -assert.neq(0, exitCode, "expected replSetStepDown to close the shell's connection"); - -// check that the old primary is no longer master -assert.soon(function() { - try { - var isMaster = master.getDB("a").isMaster(); - printjson(isMaster); - return !(isMaster.ismaster); - } catch (e) { - return false; - } -}, "they shouldn't be master, but are"); - -// stop -replTest.stopSet(); diff --git a/jstests/replsets/sync2.js b/jstests/replsets/sync2.js index 481f59a13d9..0948ed70c4d 100644 --- a/jstests/replsets/sync2.js +++ b/jstests/replsets/sync2.js @@ -1,56 +1,53 @@ -var replTest = new ReplSetTest({name: 'sync2', nodes: 5, useBridge: true}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet({oplogSize: "2"}); -replTest.initiate({ - "_id": "sync2", - "members": [ - {"_id": 0, host: nodes[0], priority: 2}, - {"_id": 1, host: nodes[1]}, - {"_id": 2, host: nodes[2]}, - {"_id": 3, host: nodes[3]}, - {"_id": 4, host: nodes[4]} - ] -}); - -var master = replTest.getPrimary(); -jsTestLog("Replica set test initialized"); - -// initial sync -master.getDB("foo").bar.insert({x: 1}); -replTest.awaitReplication(); - -conns[0].disconnect(conns[4]); -conns[1].disconnect(conns[2]); -conns[2].disconnect(conns[3]); -conns[3].disconnect(conns[1]); - -// 4 is connected to 2 -conns[4].disconnect(conns[1]); -conns[4].disconnect(conns[3]); - -assert.soon(function() { - master = replTest.getPrimary(); - return master === conns[0]; -}, 60 * 1000, "node 0 did not become primary quickly enough"); - -replTest.awaitReplication(); -jsTestLog("Checking that ops still replicate correctly"); -var option = { - writeConcern: {w: 5, wtimeout: 30000} -}; -// In PV0, this write can fail as a result of a bad spanning tree. If 2 was syncing from 4 prior to -// bridging, it will not change sync sources and receive the write in time. This was not a problem -// in 3.0 because the old version of mongobridge caused all the nodes to restart during -// partitioning, forcing the set to rebuild the spanning tree. -assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); - -// 4 is connected to 3 -conns[4].disconnect(conns[2]); -conns[4].reconnect(conns[3]); - -option = { - writeConcern: {w: 5, wtimeout: 30000} -}; -assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); - -replTest.stopSet(); +// Tests that nodes sync from each other properly and that nodes find new sync sources when they +// are disconnected from their current sync source. + +(function() { + 'use strict'; + + var replTest = new ReplSetTest({ + name: 'sync2', + nodes: [{rsConfig: {priority: 5}}, {arbiter: true}, {}, {}, {}], + useBridge: true + }); + var conns = replTest.startSet(); + replTest.initiate(); + + var master = replTest.getPrimary(); + jsTestLog("Replica set test initialized"); + + master.getDB("foo").bar.insert({x: 1}); + replTest.awaitReplication(); + + conns[0].disconnect(conns[4]); + conns[1].disconnect(conns[2]); + conns[2].disconnect(conns[3]); + conns[3].disconnect(conns[1]); + + // 4 is connected to 2 + conns[4].disconnect(conns[1]); + conns[4].disconnect(conns[3]); + + assert.soon(function() { + master = replTest.getPrimary(); + return master === conns[0]; + }, replTest.kDefaultTimeoutMS, "node 0 did not become primary quickly enough"); + + replTest.awaitReplication(); + jsTestLog("Checking that ops still replicate correctly"); + var option = { + writeConcern: {w: conns.length - 1, wtimeout: replTest.kDefaultTimeoutMS} + }; + // In PV0, this write can fail as a result of a bad spanning tree. If 2 was syncing from 4 prior + // to bridging, it will not change sync sources and receive the write in time. This was not a + // problem in 3.0 because the old version of mongobridge caused all the nodes to restart during + // partitioning, forcing the set to rebuild the spanning tree. + assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); + + // 4 is connected to 3 + conns[4].disconnect(conns[2]); + conns[4].reconnect(conns[3]); + + assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); + + replTest.stopSet(); +}()); diff --git a/jstests/replsets/tags.js b/jstests/replsets/tags.js index 55a0c4e2927..2de03f22d4f 100644 --- a/jstests/replsets/tags.js +++ b/jstests/replsets/tags.js @@ -1,260 +1,8 @@ (function() { 'use strict'; - var num = 5; - var host = getHostName(); - var name = 'tags'; + load("jstests/replsets/libs/tags.js"); - var replTest = new ReplSetTest({name: name, nodes: num, useBridge: true}); - var nodes = replTest.nodeList(); - var conns = replTest.startSet(); - var port = replTest.ports; - replTest.initiate({ - _id: name, - members: [ - { - _id: 0, - host: nodes[0], - tags: { - server: '0', - dc: 'ny', - ny: '1', - rack: 'ny.rk1', - }, - }, - { - _id: 1, - host: nodes[1], - priority: 2, - tags: { - server: '1', - dc: 'ny', - ny: '2', - rack: 'ny.rk1', - }, - }, - { - _id: 2, - host: nodes[2], - priority: 3, - tags: { - server: '2', - dc: 'ny', - ny: '3', - rack: 'ny.rk2', 2: 'this', - }, - }, - { - _id: 3, - host: nodes[3], - tags: { - server: '3', - dc: 'sf', - sf: '1', - rack: 'sf.rk1', - }, - }, - { - _id: 4, - host: nodes[4], - tags: { - server: '4', - dc: 'sf', - sf: '2', - rack: 'sf.rk2', - }, - }, - ], - settings: { - getLastErrorModes: { - '2 dc and 3 server': { - dc: 2, - server: 3, - }, - '1 and 2': { - 2: 1, - server: 1, - }, - '2': { - 2: 1, - }, - '3 and 4': { - sf: 2, - }, - '3 or 4': { - sf: 1, - }, - }, - }, - }); - - replTest.awaitReplication(); - - // Create collection to guard against timeouts due to file allocation. - assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); - replTest.awaitReplication(); - - var ensurePrimary = function(nodeId, expectedWritableNodes) { - jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.'); - replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000); - primary = replTest.getPrimary(); - var writeConcern = { - writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000} - }; - assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); - return primary; - }; - - // 2 should eventually stage a priority takeover from the primary. - var primary = ensurePrimary(2, 3); - - jsTestLog('primary is now 2'); - var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - jsTestLog('test configuration = ' + tojson(config)); - - jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); - conns[0].disconnect(conns[3]); - conns[0].disconnect(conns[4]); - conns[1].disconnect(conns[3]); - conns[1].disconnect(conns[4]); - conns[2].disconnect(conns[3]); - conns[2].disconnect(conns[4]); - conns[3].disconnect(conns[4]); - jsTestLog('Done setting up partitions'); - - jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); - jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); - - var doc = { - x: 1 - }; - - // This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs. - // Some writes are expected to block for this 'timeout' duration before failing. - // Depending on the order of heartbeats (containing last committed op time) received - // by a node, it might hang up on its sync source. This may cause some of the write concern - // tests to fail. - var timeout = 20 * 1000; - - jsTestLog('test1'); - primary = ensurePrimary(2, 3); - - jsTestLog('Non-existent write concern should be rejected.'); - options = { - writeConcern: {w: 'blahblah', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert.eq(ErrorCodes.UnknownReplWriteConcern, - result.getWriteConcernError().code, - tojson(result.getWriteConcernError())); - - jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = primary.getDB('foo').bar.insert(doc, options); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - conns[1].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3] ' + - '(all nodes besides node 3 can replicate from primary node 2)'); - primary = ensurePrimary(2, 4); - - jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + - primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError())); - - conns[3].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + - '(all secondaries can replicate from primary node 2)'); - primary = ensurePrimary(2, 5); - - jsTestLog('31004 should sync from 31001 (31026)'); - jsTestLog('31003 should sync from 31004 (31024)'); - jsTestLog('Write concern "3 and 4" should work - ' + - 'nodes 3 and 4 are connected to primary via node 1.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" - writes to primary only.'); - options = { - writeConcern: {w: '2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "1 and 2"'); - options = { - writeConcern: {w: '1 and 2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2 dc and 3 server"'); - primary = ensurePrimary(2, 5); - options = { - writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Bringing down current primary node 2 ' + primary.host + - ' to allow next higher priority node 1 ' + replTest.nodes[1].host + - ' to become primary.'); - - // Is this necessary since 3 will be connected to the new primary via node 4? - conns[1].reconnect(conns[3]); - - conns[2].disconnect(conns[0]); - conns[2].disconnect(conns[1]); - - // Is this necessary when we partition node 2 off from the rest of the nodes? - replTest.stop(2); - jsTestLog('partitions: [0-1] [2] [1-3-4] ' + - '(all secondaries except down node 2 can replicate from new primary node 1)'); - - // Node 1 with slightly higher priority will take over. - jsTestLog('1 must become primary here because otherwise the other members will take too ' + - 'long timing out their old sync threads'); - primary = ensurePrimary(1, 4); - - jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + - ' is down.'); - options = { - writeConcern: {w: '2', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - replTest.stopSet(); - jsTestLog('tags.js SUCCESS'); + var nodes = [{}, {}, {}, {}, {}]; + new TagsTest({nodes: nodes}).run(); }()); diff --git a/jstests/replsets/temp_namespace_restart_as_standalone.js b/jstests/replsets/temp_namespace_restart_as_standalone.js new file mode 100644 index 00000000000..063b1f3bbf4 --- /dev/null +++ b/jstests/replsets/temp_namespace_restart_as_standalone.js @@ -0,0 +1,87 @@ +/** + * Tests that temporary collections are not dropped when a member of a replica set is started up as + * a stand-alone mongod, i.e. without the --replSet parameter. + * + * @tags: [requires_persistence] + */ +(function() { + var rst = new ReplSetTest({nodes: 2}); + rst.startSet(); + + // Rig the election so that the first node becomes the primary and remains primary despite the + // secondary being terminated during this test. + var replSetConfig = rst.getReplSetConfig(); + replSetConfig.members[1].priority = 0; + replSetConfig.members[1].votes = 0; + rst.initiate(replSetConfig); + + var primaryConn = rst.getPrimary(); + var secondaryConn = rst.getSecondary(); + + var primaryDB = primaryConn.getDB("test"); + var secondaryDB = secondaryConn.getDB("test"); + + // Create a temporary collection and wait until the operation has replicated to the secondary. + assert.commandWorked(primaryDB.runCommand({ + create: "temp_collection", + temp: true, + })); + + // Verify that the temporary collection exists on the primary and has temp=true. + var primaryCollectionInfos = primaryDB.getCollectionInfos({name: "temp_collection"}); + assert.eq(1, primaryCollectionInfos.length, "'temp_collection' wasn't created on the primary"); + assert.eq("temp_collection", + primaryCollectionInfos[0].name, + "'temp_collection' wasn't created on the primary"); + assert.eq(true, + primaryCollectionInfos[0].options.temp, + "'temp_collection' wasn't created as temporary on the primary: " + + tojson(primaryCollectionInfos[0].options)); + + rst.awaitReplication(); + + // Verify that the temporary collection exists on the secondary and has temp=true. + var secondaryCollectionInfos = secondaryDB.getCollectionInfos({name: "temp_collection"}); + assert.eq( + 1, secondaryCollectionInfos.length, "'temp_collection' wasn't created on the secondary"); + assert.eq("temp_collection", + secondaryCollectionInfos[0].name, + "'temp_collection' wasn't created on the secondary"); + assert.eq(true, + secondaryCollectionInfos[0].options.temp, + "'temp_collection' wasn't created as temporary on the secondary: " + + tojson(secondaryCollectionInfos[0].options)); + + // Shut down the secondary and restart it as a stand-alone mongod. + var secondaryNodeId = rst.getNodeId(secondaryDB.getMongo()); + rst.stop(secondaryNodeId); + + secondaryConn = MongoRunner.runMongod({dbpath: secondaryConn.dbpath, noCleanData: true}); + assert.neq(null, secondaryConn, "secondary failed to start up as a stand-alone mongod"); + secondaryDB = secondaryConn.getDB("test"); + + // Verify that the temporary collection still exists on the secondary and has temp=true. + secondaryCollectionInfos = secondaryDB.getCollectionInfos({name: "temp_collection"}); + assert.eq(1, + secondaryCollectionInfos.length, + "'temp_collection' was dropped after restarting the secondary as a stand-alone"); + assert.eq("temp_collection", + secondaryCollectionInfos[0].name, + "'temp_collection' was dropped after restarting the secondary as a stand-alone"); + assert.eq(true, + secondaryCollectionInfos[0].options.temp, + "'temp_collection' is no longer temporary after restarting the secondary as a" + + " stand-alone: " + tojson(secondaryCollectionInfos[0].options)); + + // Shut down the secondary and restart it as a member of the replica set. + MongoRunner.stopMongod(secondaryConn); + + var restart = true; + rst.start(secondaryNodeId, {}, restart); + + // Verify that writes are replicated to the temporary collection and can successfully be applied + // by the secondary after having restarted it. + assert.writeOK(primaryDB.temp_collection.insert({}, {writeConcern: {w: 2, wtimeout: 60000}})); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/write_concern_after_stepdown.js b/jstests/replsets/write_concern_after_stepdown.js new file mode 100644 index 00000000000..57cc44fda79 --- /dev/null +++ b/jstests/replsets/write_concern_after_stepdown.js @@ -0,0 +1,91 @@ +/* + * Tests that heartbeats containing writes from a different branch of history can't cause a stale + * primary to incorrectly acknowledge a w:majority write that's about to be rolled back. + */ +(function() { + 'use strict'; + + load("jstests/replsets/rslib.js"); + load("jstests/libs/write_concern_util.js"); + + var name = "writeConcernStepDownAndBackUp"; + var dbName = "wMajorityCheck"; + var collName = "stepdownAndBackUp"; + + var rst = new ReplSetTest( + {name: name, nodes: [{}, {}, {rsConfig: {priority: 0}}, ], useBridge: true}); + var nodes = rst.startSet(); + rst.initiate(); + + function waitForPrimary(node) { + assert.soon(function() { + return node.adminCommand('ismaster').ismaster; + }); + } + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + var primary = rst.getPrimary(); + var secondaries = rst.getSecondaries(); + assert.eq(nodes[0], primary); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + // Stop the secondaries from replicating. + stopServerReplication(secondaries); + // Stop the primary from being able to complete stepping down. + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'alwaysOn'})); + + jsTestLog("Do w:majority write that will block waiting for replication."); + var doMajorityWrite = function() { + var res = db.getSiblingDB('wMajorityCheck') + .stepdownAndBackUp.insert({a: 2}, {writeConcern: {w: 'majority'}}); + assert.writeErrorWithCode(res, ErrorCodes.PrimarySteppedDown); + }; + + var joinMajorityWriter = startParallelShell(doMajorityWrite, nodes[0].port); + + jsTest.log("Disconnect primary from all secondaries"); + nodes[0].disconnect(nodes[1]); + nodes[0].disconnect(nodes[2]); + + jsTest.log("Wait for a new primary to be elected"); + // Allow the secondaries to replicate again. + restartServerReplication(secondaries); + + waitForPrimary(nodes[1]); + + jsTest.log("Do a write to the new primary"); + assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( + {a: 3}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTest.log("Reconnect the old primary to the rest of the nodes"); + // Only allow the old primary to connect to the other nodes, not the other way around. + // This is so that the old priamry will detect that it needs to step down and step itself down, + // rather than one of the other nodes detecting this and sending it a replSetStepDown command, + // which would cause the old primary to kill all operations and close all connections, making + // the way that the insert in the parallel shell fails be nondeterministic. Rather than + // handling all possible failure modes in the parallel shell, allowing heartbeat connectivity in + // only one direction makes it easier for the test to fail deterministically. + nodes[1].acceptConnectionsFrom(nodes[0]); + nodes[2].acceptConnectionsFrom(nodes[0]); + + joinMajorityWriter(); + + // Allow the old primary to finish stepping down so that shutdown can finish. + var res = null; + try { + res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'}); + } catch (e) { + // Expected - once we disable the fail point the stepdown will proceed and it's racy whether + // the stepdown closes all connections before or after the configureFailPoint command + // returns + } + if (res) { + assert.commandWorked(res); + } + + rst.stopSet(); +}()); diff --git a/jstests/replsets/write_concern_after_stepdown_and_stepup.js b/jstests/replsets/write_concern_after_stepdown_and_stepup.js new file mode 100644 index 00000000000..22167ee356c --- /dev/null +++ b/jstests/replsets/write_concern_after_stepdown_and_stepup.js @@ -0,0 +1,116 @@ +/* + * Tests that heartbeats containing writes from a different branch of history can't cause a stale + * primary to incorrectly acknowledge a w:majority write that's about to be rolled back, even if the + * stale primary is re-elected primary before waiting for the write concern acknowledgement. + */ +(function() { + 'use strict'; + + var name = "writeConcernStepDownAndBackUp"; + var dbName = "wMajorityCheck"; + var collName = "stepdownAndBackUp"; + + var rst = new ReplSetTest( + {name: name, nodes: [{}, {}, {rsConfig: {priority: 0}}, ], useBridge: true}); + var nodes = rst.startSet(); + rst.initiate(); + + var timeout = 5 * 60 * 1000; + + function waitForState(node, state) { + assert.soonNoExcept(function() { + assert.commandWorked(node.adminCommand( + {replSetTest: 1, waitForMemberState: state, timeoutMillis: timeout})); + return true; + }); + } + + function waitForPrimary(node) { + assert.soon(function() { + return node.adminCommand('ismaster').ismaster; + }); + } + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + var primary = rst.getPrimary(); + var secondaries = rst.getSecondaries(); + assert.eq(nodes[0], primary); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 3, wtimeout: timeout}})); + + // Stop the secondaries from replicating. + secondaries.forEach(function(node) { + assert.commandWorked( + node.adminCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'})); + }); + // Stop the primary from calling into awaitReplication() + assert.commandWorked(nodes[0].adminCommand( + {configureFailPoint: 'hangBeforeWaitingForWriteConcern', mode: 'alwaysOn'})); + + jsTestLog("Do w:majority write that won't enter awaitReplication() until after the primary " + + "has stepped down and back up"); + var doMajorityWrite = function() { + assert.commandWorked(db.adminCommand({ismaster: 1})); + + assert.throws(function() { + db.getSiblingDB('wMajorityCheck') + .stepdownAndBackUp.insert({a: 2}, {writeConcern: {w: 'majority'}}); + }); + }; + + var joinMajorityWriter = startParallelShell(doMajorityWrite, nodes[0].port); + + jsTest.log("Disconnect primary from all secondaries"); + nodes[0].disconnect(nodes[1]); + nodes[0].disconnect(nodes[2]); + + jsTest.log("Wait for a new primary to be elected"); + // Allow the secondaries to replicate again. + secondaries.forEach(function(node) { + assert.commandWorked( + node.adminCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'off'})); + }); + + waitForPrimary(nodes[1]); + + jsTest.log("Do a write to the new primary"); + assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( + {a: 3}, {writeConcern: {w: 2, wtimeout: timeout}})); + + jsTest.log("Reconnect the old primary to the rest of the nodes"); + nodes[0].reconnect(nodes[1]); + nodes[0].reconnect(nodes[2]); + + jsTest.log("Wait for the old primary to step down, roll back its write, and apply the " + + "new writes from the new primary"); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + rst.awaitReplication(); + + // At this point all 3 nodes should have the same data + assert.soonNoExcept(function() { + nodes.forEach(function(node) { + assert.eq(null, + node.getDB(dbName).getCollection(collName).findOne({a: 2}), + "Node " + node.host + " contained op that should have been rolled back"); + assert.neq(null, + node.getDB(dbName).getCollection(collName).findOne({a: 3}), + "Node " + node.host + + " was missing op from branch of history that should have persisted"); + }); + return true; + }); + + jsTest.log("Make the original primary become primary once again"); + rst.stepUp(nodes[0]); + + jsTest.log("Unblock the thread waiting for replication of the now rolled-back write, ensure " + + "that the write concern failed"); + assert.commandWorked(nodes[0].adminCommand( + {configureFailPoint: 'hangBeforeWaitingForWriteConcern', mode: 'off'})); + + joinMajorityWriter(); + + rst.stopSet(); +}()); diff --git a/jstests/sharding/authCommands.js b/jstests/sharding/authCommands.js index 2fb42801a40..c22e7aee6a8 100644 --- a/jstests/sharding/authCommands.js +++ b/jstests/sharding/authCommands.js @@ -71,6 +71,8 @@ var doTest = function() { assert.eq(expectedDocs, testDB.foo.count()); // Wait for the balancer to start back up + assert.writeOK( + configDB.settings.update({_id: 'balancer'}, {$set: {_waitForDelete: true}}, true)); st.startBalancer(); // Make sure we've done at least some splitting, so the balancer will work @@ -83,11 +85,6 @@ var doTest = function() { return x < 2 && configDB.locks.findOne({_id: 'test.foo'}).state == 0; }, "no balance happened", 5 * 60 * 1000); - assert.soon(function() { - print("Waiting for migration cleanup to occur..."); - return testDB.foo.find().itcount() == testDB.foo.count(); - }); - var map = function() { emit(this.i, this.j); }; diff --git a/jstests/sharding/cleanup_orphaned_basic.js b/jstests/sharding/cleanup_orphaned_basic.js index 3ed9015941b..5a119ec3fff 100644 --- a/jstests/sharding/cleanup_orphaned_basic.js +++ b/jstests/sharding/cleanup_orphaned_basic.js @@ -63,10 +63,10 @@ // Ping shard[1] so it will be aware that it is sharded. Otherwise cleanupOrphaned // may fail. assert.commandWorked(mongosAdmin.runCommand( - {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[1]._id})); + {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[1]._id, _waitForDelete: true})); assert.commandWorked(mongosAdmin.runCommand( - {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[0]._id})); + {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[0]._id, _waitForDelete: true})); // Collection's home is shard0, there are no chunks assigned to shard1. st.shard1.getCollection(ns).insert({}); diff --git a/jstests/sharding/conn_pool_stats.js b/jstests/sharding/conn_pool_stats.js index 872c20602a5..04952a273b6 100644 --- a/jstests/sharding/conn_pool_stats.js +++ b/jstests/sharding/conn_pool_stats.js @@ -1,19 +1,32 @@ // Tests for the connPoolStats command. +(function() { + "use strict"; + // Create a cluster with 2 shards. + var cluster = new ShardingTest({shards: 2}); -// Create a cluster with 2 shards. -var cluster = new ShardingTest({shards: 2}); + // Needed because the command was expanded post 3.2 + var version = cluster.s.getDB("admin").runCommand({buildinfo: 1}).versionArray; + var post32 = (version[0] > 4) || ((version[0] == 3) && (version[1] > 2)); -// Run the connPoolStats command -stats = cluster.s.getDB("admin").runCommand({connPoolStats: 1}); + // Run the connPoolStats command + var stats = cluster.s.getDB("admin").runCommand({connPoolStats: 1}); -// Validate output -printjson(stats); -assert.commandWorked(stats); -assert("replicaSets" in stats); -assert("hosts" in stats); -assert("numClientConnections" in stats); -assert("numAScopedConnections" in stats); -assert("totalInUse" in stats); -assert("totalAvailable" in stats); -assert("totalCreated" in stats); -assert.lte(stats["totalInUse"] + stats["totalAvailable"], stats["totalCreated"], tojson(stats)); + // Validate output + printjson(stats); + assert.commandWorked(stats); + assert("replicaSets" in stats); + assert("hosts" in stats); + assert("numClientConnections" in stats); + assert("numAScopedConnections" in stats); + assert("totalInUse" in stats); + assert("totalAvailable" in stats); + assert("totalCreated" in stats); + assert.lte(stats["totalInUse"] + stats["totalAvailable"], stats["totalCreated"], tojson(stats)); + if (post32) { + assert("pools" in stats); + assert("totalRefreshing" in stats); + assert.lte(stats["totalInUse"] + stats["totalAvailable"] + stats["totalRefreshing"], + stats["totalCreated"], + tojson(stats)); + } +})(); diff --git a/jstests/sharding/cursor1.js b/jstests/sharding/cursor1.js index e7d54ac85e4..eccc9a70c31 100644 --- a/jstests/sharding/cursor1.js +++ b/jstests/sharding/cursor1.js @@ -44,8 +44,6 @@ assert.eq(numObjs, cursor3.itcount(), "c3"); // Test that a cursor with a 1 second timeout eventually times out. - gc(); - gc(); var cur = db.foo.find().batchSize(2); assert(cur.next(), "T1"); assert(cur.next(), "T2"); @@ -65,9 +63,6 @@ } }, "cursor failed to time out", /*timeout*/ 30000, /*interval*/ 5000); - gc(); - gc(); - s.stop(); })(); diff --git a/jstests/sharding/migration_critical_section_concurrency.js b/jstests/sharding/migration_critical_section_concurrency.js new file mode 100644 index 00000000000..059d6a0e32e --- /dev/null +++ b/jstests/sharding/migration_critical_section_concurrency.js @@ -0,0 +1,63 @@ +// This test ensures that if one collection is its migration critical section, this won't stall +// operations for other sharded or unsharded collections + +load('./jstests/libs/chunk_manipulation_util.js'); + +(function() { + 'use strict'; + + var staticMongod = MongoRunner.runMongod({}); // For startParallelOps. + + var st = new ShardingTest({mongos: 1, shards: 2}); + assert.commandWorked(st.s0.adminCommand({enableSharding: 'TestDB'})); + st.ensurePrimaryShard('TestDB', st.shard0.shardName); + + var testDB = st.s0.getDB('TestDB'); + + assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.Coll0', key: {Key: 1}})); + assert.commandWorked(st.s0.adminCommand({split: 'TestDB.Coll0', middle: {Key: 0}})); + + var coll0 = testDB.Coll0; + assert.writeOK(coll0.insert({Key: -1, Value: '-1'})); + assert.writeOK(coll0.insert({Key: 1, Value: '1'})); + + assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.Coll1', key: {Key: 1}})); + assert.commandWorked(st.s0.adminCommand({split: 'TestDB.Coll1', middle: {Key: 0}})); + + var coll1 = testDB.Coll1; + assert.writeOK(coll1.insert({Key: -1, Value: '-1'})); + assert.writeOK(coll1.insert({Key: 1, Value: '1'})); + + // Ensure that coll0 has chunks on both shards so we can test queries against both donor and + // recipient for Coll1's migration below + assert.commandWorked( + st.s0.adminCommand({moveChunk: 'TestDB.Coll0', find: {Key: 1}, to: st.shard1.shardName})); + + // Pause the move chunk operation in the critical section + pauseMigrateAtStep(st.shard1, migrateStepNames.done); + var joinMoveChunk = moveChunkParallel( + staticMongod, st.s0.host, {Key: 1}, null, 'TestDB.Coll1', st.shard1.shardName); + + // Wait till the donor reaches the critical section + waitForMoveChunkStep(st.shard0, moveChunkStepNames.reachedSteadyState); + + // Ensure that operations for 'Coll0' are not stalled + assert.eq(1, coll0.find({Key: {$lte: -1}}).maxTimeMS(5000).itcount()); + assert.eq(1, coll0.find({Key: {$gte: 1}}).maxTimeMS(5000).itcount()); + assert.writeOK(coll0.insert({Key: -2, Value: '-2'}, {writeConcern: {wtimeout: 5000}})); + assert.writeOK(coll0.insert({Key: 2, Value: '2'}, {writeConcern: {wtimeout: 5000}})); + assert.eq(2, coll0.find({Key: {$lte: -1}}).maxTimeMS(5000).itcount()); + assert.eq(2, coll0.find({Key: {$gte: 1}}).maxTimeMS(5000).itcount()); + + // Ensure that operations for non-sharded collections are not stalled + var collUnsharded = testDB.CollUnsharded; + assert.eq(0, collUnsharded.find({}).maxTimeMS(5000).itcount()); + assert.writeOK( + collUnsharded.insert({TestKey: 0, Value: 'Zero'}, {writeConcern: {wtimeout: 5000}})); + assert.eq(1, collUnsharded.find({}).maxTimeMS(5000).itcount()); + + unpauseMigrateAtStep(st.shard1, migrateStepNames.done); + joinMoveChunk(); + + st.stop(); +})(); diff --git a/jstests/sharding/move_stale_mongos.js b/jstests/sharding/move_stale_mongos.js index 44dd7a7649f..9df8c80af03 100644 --- a/jstests/sharding/move_stale_mongos.js +++ b/jstests/sharding/move_stale_mongos.js @@ -15,10 +15,20 @@ var curShardIndex = 0; for (var i = 0; i < 100; i += 10) { assert.commandWorked(st.s0.getDB('admin').runCommand({split: testNs, middle: {_id: i}})); + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the split + } var nextShardIndex = (curShardIndex + 1) % shards.length; - assert.commandWorked(st.s1.getDB('admin').runCommand( - {moveChunk: testNs, find: {_id: i + 5}, to: shards[nextShardIndex]})); + assert.commandWorked(st.s1.getDB('admin').runCommand({ + moveChunk: testNs, + find: {_id: i + 5}, + to: shards[nextShardIndex], + _waitForDelete: true + })); curShardIndex = nextShardIndex; + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the move + } } st.stop(); diff --git a/jstests/sharding/shard6.js b/jstests/sharding/shard6.js index 3b4f3e416eb..7d3736c5c21 100644 --- a/jstests/sharding/shard6.js +++ b/jstests/sharding/shard6.js @@ -1,8 +1,8 @@ // shard6.js -summary = ""; +var summary = ""; -s = new ShardingTest({name: "shard6", shards: 2}); +var s = new ShardingTest({name: "shard6", shards: 2}); s.config.settings.update({_id: "balancer"}, {$set: {stopped: true}}, true); @@ -10,29 +10,33 @@ s.adminCommand({enablesharding: "test"}); s.ensurePrimaryShard('test', 'shard0001'); s.adminCommand({shardcollection: "test.data", key: {num: 1}}); -db = s.getDB("test"); +var db = s.getDB("test"); function poolStats(where) { var total = 0; var msg = "poolStats " + where + " "; - var x = db.runCommand("connPoolStats").hosts; - for (var h in x) { - var z = x[h]; - msg += z.created + " "; - total += z.created; + var stats = db.runCommand("connPoolStats"); + for (var h in stats.hosts) { + if (!stats.hosts.hasOwnProperty(h)) { + continue; + } + var host = stats.hosts[h]; + msg += host.created + " "; + total += host.created; } - printjson(x); + printjson(stats.hosts); print("****\n" + msg + "\n*****"); summary += msg + "\n"; + + assert.eq(total, stats.totalCreated, "mismatched number of total connections created"); + return total; } poolStats("at start"); // we want a lot of data, so lets make a 50k string to cheat :) -bigString = ""; -while (bigString.length < 50000) - bigString += "this is a big string. "; +var bigString = "this is a big string. ".repeat(50000); // ok, now lets insert a some data var num = 0; @@ -42,7 +46,7 @@ for (; num < 100; num++) { assert.eq(100, db.data.find().toArray().length, "basic find after setup"); -connBefore = poolStats("setup done"); +poolStats("setup done"); // limit @@ -54,7 +58,7 @@ for (var i = 1; i < 10; i++) { poolStats("after loop : " + i); } -assert.eq(connBefore, poolStats("limit test done"), "limit test conns"); +poolStats("limit test done"); function assertOrder(start, num) { var a = db.data.find().skip(start).limit(num).sort({num: 1}).map(function(z) { @@ -99,7 +103,7 @@ poolStats("after checking itcount"); // --- test save support --- -o = db.data.findOne(); +var o = db.data.findOne(); o.x = 16; db.data.save(o); o = db.data.findOne({_id: o._id}); diff --git a/jstests/sharding/shard_kill_and_pooling.js b/jstests/sharding/shard_kill_and_pooling.js index 6b8397f9e37..369ebe6c3ee 100644 --- a/jstests/sharding/shard_kill_and_pooling.js +++ b/jstests/sharding/shard_kill_and_pooling.js @@ -44,10 +44,8 @@ for (var test = 0; test < 2; test++) { jsTest.log("Returning the connections back to the pool."); for (var i = 0; i < conns.length; i++) { - conns[i] = null; + conns[i].close(); } - // Make sure we return connections back to the pool - gc(); // Don't make test fragile by linking to format of shardConnPoolStats, but this is useful if // something goes wrong. diff --git a/jstests/sharding/sharding_rs2.js b/jstests/sharding/sharding_rs2.js index ac186e6478b..c8663ee8dc8 100644 --- a/jstests/sharding/sharding_rs2.js +++ b/jstests/sharding/sharding_rs2.js @@ -136,10 +136,9 @@ assert.eq(100, ts.find().itcount(), "B5"); assert.eq(100, ts.find().batchSize(5).itcount(), "B6"); - t.find().batchSize(3).next(); - gc(); - gc(); - gc(); + var cursor = t.find().batchSize(3); + cursor.next(); + cursor.close(); // --- sharded ---- diff --git a/jstests/sharding/split_stale_mongos.js b/jstests/sharding/split_stale_mongos.js index e40112f0cd0..2a2a9b50cc8 100644 --- a/jstests/sharding/split_stale_mongos.js +++ b/jstests/sharding/split_stale_mongos.js @@ -12,7 +12,13 @@ assert.commandWorked(admin.runCommand({shardCollection: testNs, key: {_id: 1}})) for (var i = 0; i < 100; i += 10) { assert.commandWorked(st.s0.getDB('admin').runCommand({split: testNs, middle: {_id: i}})); + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the previous split + } assert.commandWorked(st.s1.getDB('admin').runCommand({split: testNs, middle: {_id: i + 5}})); + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the previous split + } } -st.stop(); \ No newline at end of file +st.stop(); diff --git a/jstests/sharding/sync_cluster_config/sccc_read_pref.js b/jstests/sharding/sync_cluster_config/sccc_read_pref.js new file mode 100644 index 00000000000..24e606677c0 --- /dev/null +++ b/jstests/sharding/sync_cluster_config/sccc_read_pref.js @@ -0,0 +1,18 @@ +/** + * Simple test that makes sure that running a command with read preference settings on a SCCC setup + * works. + */ +(function() { + 'use strict'; + + // Start a sharding cluster with a single shard, which has one node + var st = new ShardingTest({shards: 1, config: 3, other: {sync: true}}); + st.stopBalancer(); + + st.s.setReadPref('secondaryPreferred'); + var count = st.s.getDB('config').mongos.count(); + + assert.eq(1, count); + + st.stop(); +})(); diff --git a/jstests/slow1/replsets_priority1.js b/jstests/slow1/replsets_priority1.js index 7e5d562214b..4bbb856a4b6 100644 --- a/jstests/slow1/replsets_priority1.js +++ b/jstests/slow1/replsets_priority1.js @@ -73,7 +73,7 @@ var checkPrimaryIs = function(node) { }, 15); return ok; - }, node.host + '==1', 60000, 1000); + }, node.host + '==1', 240000, 1000); everyoneOkSoon(); }; diff --git a/jstests/ssl/ssl_fips.js b/jstests/ssl/ssl_fips.js index 47f148340d9..9b3e4b94c96 100644 --- a/jstests/ssl/ssl_fips.js +++ b/jstests/ssl/ssl_fips.js @@ -24,10 +24,7 @@ if (mongo != 0) { print("mongod failed to start, checking for FIPS support"); mongoOutput = rawMongoProgramOutput(); assert(mongoOutput.match(/this version of mongodb was not compiled with FIPS support/) || - mongoOutput.match(/FIPS_mode_set:fips mode not supported/) || - // Ubuntu 16.04's OpenSSL produces an unexpected error message, remove this check when - // SERVER-24350 is resolved - mongoOutput.match(/error:00000000:lib\(0\):func\(0\):reason\(0\)/)); + mongoOutput.match(/FIPS_mode_set:fips mode not supported/)); } else { // verify that auth works, SERVER-18051 md.getDB("admin").createUser({user: "root", pwd: "root", roles: ["root"]}); diff --git a/rpm/mongodb-enterprise-unstable.spec b/rpm/mongodb-enterprise-unstable.spec index cd4c66607b8..e5556b9a1cd 100644 --- a/rpm/mongodb-enterprise-unstable.spec +++ b/rpm/mongodb-enterprise-unstable.spec @@ -34,7 +34,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server (enterprise) Group: Applications/Databases -Requires: openssl, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools %description server @@ -58,7 +58,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools %description shell @@ -105,7 +105,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools %description tools diff --git a/rpm/mongodb-enterprise.spec b/rpm/mongodb-enterprise.spec index 95fd1410306..3373b1bb19d 100644 --- a/rpm/mongodb-enterprise.spec +++ b/rpm/mongodb-enterprise.spec @@ -35,7 +35,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server (enterprise) Group: Applications/Databases -Requires: openssl, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongo-10gen-enterprise-server Provides: mongo-10gen-enterprise-server @@ -61,7 +61,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongo-10gen-enterprise-shell Provides: mongo-10gen-enterprise-shell @@ -112,7 +112,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongo-10gen-enterprise-tools Provides: mongo-10gen-enterprise-tools diff --git a/rpm/mongodb-org-unstable.spec b/rpm/mongodb-org-unstable.spec index 27b2266bd3e..39e305c3afc 100644 --- a/rpm/mongodb-org-unstable.spec +++ b/rpm/mongodb-org-unstable.spec @@ -33,7 +33,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools %description server @@ -57,7 +57,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools %description shell @@ -104,7 +104,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools %description tools diff --git a/rpm/mongodb-org.spec b/rpm/mongodb-org.spec index 00e30576b5c..40d2808060b 100644 --- a/rpm/mongodb-org.spec +++ b/rpm/mongodb-org.spec @@ -35,7 +35,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen-server Provides: mongo-10gen-server @@ -61,7 +61,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen-shell Provides: mongo-10gen-shell @@ -112,7 +112,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen-tools Provides: mongo-10gen-tools diff --git a/site_scons/site_tools/compilation_db.py b/site_scons/site_tools/compilation_db.py index a4c82eebd53..5c0d90e5f41 100644 --- a/site_scons/site_tools/compilation_db.py +++ b/site_scons/site_tools/compilation_db.py @@ -30,6 +30,16 @@ import itertools # communicate more gracefully? __COMPILATION_DB_ENTRIES=[] +# Cribbed from Tool/cc.py and Tool/c++.py. It would be better if +# we could obtain this from SCons. +_CSuffixes = ['.c'] +if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') + +_CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] +if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') + # We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even # integrate with the cache, but there doesn't seem to be much call for it. class __CompilationDbNode(SCons.Node.Node): @@ -99,20 +109,15 @@ def generate(env, **kwargs): static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - # TODO: Is there a way to obtain the configured suffixes for C and C++ - # from the existing obj builders? Seems unfortunate to re-iterate them. - CSuffixes = ['.c'] - CXXSuffixes = ['.cc', '.cxx', '.cpp'] - env['COMPILATIONDB_COMSTR'] = kwargs.get( 'COMPILATIONDB_COMSTR', 'Building compilation database $TARGET') components_by_suffix = itertools.chain( - itertools.product(CSuffixes, [ + itertools.product(_CSuffixes, [ (static_obj, SCons.Defaults.StaticObjectEmitter, '$CCCOM'), (shared_obj, SCons.Defaults.SharedObjectEmitter, '$SHCCCOM'), ]), - itertools.product(CXXSuffixes, [ + itertools.product(_CXXSuffixes, [ (static_obj, SCons.Defaults.StaticObjectEmitter, '$CXXCOM'), (shared_obj, SCons.Defaults.SharedObjectEmitter, '$SHCXXCOM'), ]), @@ -122,13 +127,14 @@ def generate(env, **kwargs): suffix = entry[0] builder, base_emitter, command = entry[1] - builder.add_emitter( - suffix, SCons.Builder.ListEmitter( - [ - makeEmitCompilationDbEntry(command), - base_emitter, - ] - )) + # Assumes a dictionary emitter + emitter = builder.emitter[suffix] + builder.emitter[suffix] = SCons.Builder.ListEmitter( + [ + emitter, + makeEmitCompilationDbEntry(command), + ] + ) env['BUILDERS']['__COMPILATIONDB_Entry'] = SCons.Builder.Builder( action=SCons.Action.Action(CompilationDbEntryAction, None), diff --git a/site_scons/site_tools/incremental_link.py b/site_scons/site_tools/incremental_link.py new file mode 100644 index 00000000000..cf74ef9674c --- /dev/null +++ b/site_scons/site_tools/incremental_link.py @@ -0,0 +1,45 @@ +# Copyright 2017 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import SCons + +def _tag_as_precious(target, source, env): + env.Precious(target) + return target, source + +def generate(env): + builders = env['BUILDERS'] + for builder in ('Program', 'SharedLibrary', 'LoadableModule'): + emitter = builders[builder].emitter + builders[builder].emitter = SCons.Builder.ListEmitter([ + emitter, + _tag_as_precious, + ]) + +def exists(env): + # By default, the windows linker is incremental, so unless + # overridden in the environment with /INCREMENTAL:NO, the tool is + # in play. + if env.TargetOSIs('windows') and not "/INCREMENTAL:NO" in env['LINKFLAGS']: + return True + + # On posix platforms, excluding darwin, we may have enabled + # incremental linking. Check for the relevant flags. + if env.TargetOSIs('posix') and \ + not env.TargetOSIs('darwin') and \ + "-fuse-ld=gold" in env['LINKFLAGS'] and \ + "-Wl,--incremental" in env['LINKFLAGS']: + return True + + return False diff --git a/site_scons/site_tools/split_dwarf.py b/site_scons/site_tools/split_dwarf.py new file mode 100644 index 00000000000..95130c9e9a3 --- /dev/null +++ b/site_scons/site_tools/split_dwarf.py @@ -0,0 +1,65 @@ +# Copyright 2017 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import SCons + +_splitDwarfFlag = '-gsplit-dwarf' + +# Cribbed from Tool/cc.py and Tool/c++.py. It would be better if +# we could obtain this from SCons. +_CSuffixes = ['.c'] +if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') + +_CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] +if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') + +def _dwo_emitter(target, source, env): + new_targets = [] + for t in target: + base, ext = SCons.Util.splitext(str(t)) + if not any(ext == env[osuffix] for osuffix in ['OBJSUFFIX', 'SHOBJSUFFIX']): + continue + # TODO: Move 'dwo' into DWOSUFFIX so it can be customized? For + # now, GCC doesn't let you control the output filename, so it + # doesn't matter. + dwotarget = (t.builder.target_factory or env.File)(base + ".dwo") + new_targets.append(dwotarget) + targets = target + new_targets + return (targets, source) + +def generate(env): + suffixes = [] + if _splitDwarfFlag in env['CCFLAGS']: + suffixes = _CSuffixes + _CXXSuffixes + else: + if _splitDwarfFlag in env['CFLAGS']: + suffixes.extend(_CSuffixes) + if _splitDwarfFlag in env['CXXFLAGS']: + suffixes.extend(_CXXSuffixes) + + for object_builder in SCons.Tool.createObjBuilders(env): + emitterdict = object_builder.builder.emitter + for suffix in emitterdict.iterkeys(): + if not suffix in suffixes: + continue + base = emitterdict[suffix] + emitterdict[suffix] = SCons.Builder.ListEmitter([ + base, + _dwo_emitter, + ]) + +def exists(env): + return any(_splitDwarfFlag in env[f] for f in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS']) diff --git a/src/mongo/SConscript b/src/mongo/SConscript index e4b6c1dc770..18f75acdaeb 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -218,6 +218,7 @@ config_header_substs = ( ('@mongo_config_have_memset_s@', 'MONGO_CONFIG_HAVE_MEMSET_S'), ('@mongo_config_have_posix_monotonic_clock@', 'MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK'), ('@mongo_config_have_std_is_trivially_copyable@', 'MONGO_CONFIG_HAVE_STD_IS_TRIVIALLY_COPYABLE'), + ('@mongo_config_have_std_enable_if_t@', 'MONGO_CONFIG_HAVE_STD_ENABLE_IF_T'), ('@mongo_config_have_std_make_unique@', 'MONGO_CONFIG_HAVE_STD_MAKE_UNIQUE'), ('@mongo_config_have_std_align@', 'MONGO_CONFIG_HAVE_STD_ALIGN'), ('@mongo_config_have_strnlen@', 'MONGO_CONFIG_HAVE_STRNLEN'), diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 7657112cac6..692fae39172 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -155,6 +155,11 @@ error_code("OplogOutOfOrder", 152) error_code("CanRepairToDowngrade", 157) error_code("MustUpgrade", 158) error_code("QueryPlanKilled", 173) +error_code("FileOpenFailed", 174) +error_code("ZoneNotFound", 175) +error_code("RangeOverlapConflict", 176) +error_code("WindowsPdhError", 177) +error_code("BadPerfCounterPath", 178) error_code("ReceivedOpReplyMessage", 198); # Non-sequential error codes (for compatibility only) diff --git a/src/mongo/base/string_data.h b/src/mongo/base/string_data.h index e7a3d677594..ef09cbe827f 100644 --- a/src/mongo/base/string_data.h +++ b/src/mongo/base/string_data.h @@ -35,6 +35,7 @@ #include #include +#include "mongo/stdx/type_traits.h" #define MONGO_INCLUDE_INVARIANT_H_WHITELISTED #include "mongo/util/invariant.h" #undef MONGO_INCLUDE_INVARIANT_H_WHITELISTED @@ -59,6 +60,9 @@ class StringData { StringData(const char* c, size_t len, TrustedInitTag) : _data(c), _size(len) {} public: + // Iterator type + using const_iterator = const char*; + /** Constructs an empty StringData. */ StringData() = default; @@ -98,6 +102,23 @@ public: invariant(_data || (_size == 0)); } + /** + * Constructs a StringData with begin and end iterators. begin points to the beginning of the + * string. end points to the position past the end of the string. In a null-terminated string, + * end points to the null-terminator. + * + * We template the second parameter to ensure if StringData is called with 0 in the second + * parameter, the (ptr,len) constructor is chosen instead. + */ + template < + typename InputIt, + typename = stdx::enable_if_t::value>> + StringData(InputIt begin, InputIt end) { + invariant(begin && end); + _data = begin; + _size = std::distance(begin, end); + } + /** * Returns -1, 0, or 1 if 'this' is less, equal, or greater than 'other' in * lexicographical order. @@ -171,9 +192,6 @@ public: // // iterators // - - typedef const char* const_iterator; - const_iterator begin() const { return rawData(); } diff --git a/src/mongo/bson/timestamp.cpp b/src/mongo/bson/timestamp.cpp index 2abd3a8990a..232aff04128 100644 --- a/src/mongo/bson/timestamp.cpp +++ b/src/mongo/bson/timestamp.cpp @@ -34,6 +34,7 @@ #include #include +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/util/time_support.h" namespace mongo { @@ -70,4 +71,11 @@ std::string Timestamp::toString() const { ss << std::hex << secs << ':' << i; return ss.str(); } + + +BSONObj Timestamp::toBSON() const { + BSONObjBuilder bldr; + bldr.append("", *this); + return bldr.obj(); +} } diff --git a/src/mongo/bson/timestamp.h b/src/mongo/bson/timestamp.h index 1dc04973efb..39c0eb572a8 100644 --- a/src/mongo/bson/timestamp.h +++ b/src/mongo/bson/timestamp.h @@ -34,6 +34,8 @@ namespace mongo { +class BSONObj; + /** * Timestamp: A combination of a count of seconds since the POSIX epoch plus an ordinal value. */ @@ -112,6 +114,7 @@ public: // Append the BSON representation of this Timestamp to the given BufBuilder with the given // name. This lives here because Timestamp manages its own serialization format. void append(BufBuilder& builder, const StringData& fieldName) const; + BSONObj toBSON() const; private: std::tuple tie() const { diff --git a/src/mongo/client/SConscript b/src/mongo/client/SConscript index d112f34fe14..a8a5367e2fe 100644 --- a/src/mongo/client/SConscript +++ b/src/mongo/client/SConscript @@ -60,6 +60,7 @@ saslClientSource = [ 'sasl_client_session.cpp', 'sasl_plain_client_conversation.cpp', 'sasl_scramsha1_client_conversation.cpp', + 'scram_sha1_client_cache.cpp', ] # Add in actual sasl dependencies if sasl is enabled, otherwise diff --git a/src/mongo/client/authenticate.cpp b/src/mongo/client/authenticate.cpp index 58002af5607..6db235f552b 100644 --- a/src/mongo/client/authenticate.cpp +++ b/src/mongo/client/authenticate.cpp @@ -237,7 +237,7 @@ void authX509(RunCommandHook runCommand, void auth(RunCommandHook runCommand, const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientName, AuthCompletionHandler handler) { std::string mechanism; @@ -273,7 +273,7 @@ bool needsFallback(const AuthResponse& response) { void asyncAuth(RunCommandHook runCommand, const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientName, AuthCompletionHandler handler) { auth(runCommand, @@ -297,7 +297,7 @@ void asyncAuth(RunCommandHook runCommand, } // namespace void authenticateClient(const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientName, RunCommandHook runCommand, AuthCompletionHandler handler) { diff --git a/src/mongo/client/authenticate.h b/src/mongo/client/authenticate.h index 65c649c49c2..f8d7434f30a 100644 --- a/src/mongo/client/authenticate.h +++ b/src/mongo/client/authenticate.h @@ -92,7 +92,7 @@ extern const char* const kMechanismScramSha1; * tantamount to authentication failure, but may also indicate more serious problems. */ void authenticateClient(const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientSubjectName, RunCommandHook runCommand, AuthCompletionHandler handler = AuthCompletionHandler()); diff --git a/src/mongo/client/authenticate_test.cpp b/src/mongo/client/authenticate_test.cpp index 0de2a1e9670..6a6596d52d8 100644 --- a/src/mongo/client/authenticate_test.cpp +++ b/src/mongo/client/authenticate_test.cpp @@ -173,13 +173,13 @@ public: TEST_F(AuthClientTest, MongoCR) { auto params = loadMongoCRConversation(); - auth::authenticateClient(std::move(params), "", "", _runCommandCallback); + auth::authenticateClient(std::move(params), HostAndPort(), "", _runCommandCallback); } TEST_F(AuthClientTest, asyncMongoCR) { auto params = loadMongoCRConversation(); auth::authenticateClient(std::move(params), - "", + HostAndPort(), "", _runCommandCallback, [this](auth::AuthResponse response) { ASSERT(response.isOK()); }); @@ -188,13 +188,13 @@ TEST_F(AuthClientTest, asyncMongoCR) { #ifdef MONGO_CONFIG_SSL TEST_F(AuthClientTest, X509) { auto params = loadX509Conversation(); - auth::authenticateClient(std::move(params), "", _username, _runCommandCallback); + auth::authenticateClient(std::move(params), HostAndPort(), _username, _runCommandCallback); } TEST_F(AuthClientTest, asyncX509) { auto params = loadX509Conversation(); auth::authenticateClient(std::move(params), - "", + HostAndPort(), _username, _runCommandCallback, [this](auth::AuthResponse response) { ASSERT(response.isOK()); }); diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp index 2a019db9787..598d58afcd1 100644 --- a/src/mongo/client/connpool.cpp +++ b/src/mongo/client/connpool.cpp @@ -62,6 +62,11 @@ PoolForHost::~PoolForHost() { } void PoolForHost::clear() { + if (!_parentDestroyed) { + logNoCache() << "Dropping all pooled connections to " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)"; + } + while (!_pool.empty()) { StoredConnection sc = _pool.top(); delete sc.conn; @@ -80,9 +85,19 @@ void PoolForHost::done(DBConnectionPool* pool, DBClientBase* c) { if (isFailed || // Another (later) connection was reported as broken to this host - (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) || + (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec)) { + logNoCache() << "Ending connection to host " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)" + << " due to bad connection status; " << openConnections() + << " connections to that host remain open"; + pool->onDestroy(c); + delete c; + } else if (_maxPoolSize >= 0 && static_cast(_pool.size()) >= _maxPoolSize) { // We have a pool size that we need to enforce - (_maxPoolSize >= 0 && static_cast(_pool.size()) >= _maxPoolSize)) { + logNoCache() << "Ending idle connection to host " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)" + << " because the pool meets constraints; " << openConnections() + << " connections to that host remain open"; pool->onDestroy(c); delete c; } else { @@ -95,9 +110,11 @@ void PoolForHost::reportBadConnectionAt(uint64_t microSec) { if (microSec != DBClientBase::INVALID_SOCK_CREATION_TIME && microSec > _minValidCreationTimeMicroSec) { _minValidCreationTimeMicroSec = microSec; - log() << "Detected bad connection created at " << _minValidCreationTimeMicroSec - << " microSec, clearing pool for " << _hostName << " of " << _pool.size() - << " connections" << endl; + + logNoCache() << "Detected bad connection created at " << _minValidCreationTimeMicroSec + << " microSec, clearing pool for " << _hostName << " of " << openConnections() + << " connections" << endl; + clear(); } } @@ -130,6 +147,9 @@ DBClientBase* PoolForHost::get(DBConnectionPool* pool, double socketTimeout) { } void PoolForHost::flush() { + log() << "Dropping all pooled connections to " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)"; + while (!_pool.empty()) { StoredConnection c = _pool.top(); _pool.pop(); @@ -196,10 +216,17 @@ DBClientBase* DBConnectionPool::_get(const string& ident, double socketTimeout) stdx::lock_guard L(_mutex); PoolForHost& p = _pools[PoolKey(ident, socketTimeout)]; p.setMaxPoolSize(_maxPoolSize); + p.setSocketTimeout(socketTimeout); p.initializeHostName(ident); return p.get(this, socketTimeout); } +int DBConnectionPool::openConnections(const string& ident, double socketTimeout) { + stdx::lock_guard L(_mutex); + PoolForHost& p = _pools[PoolKey(ident, socketTimeout)]; + return p.openConnections(); +} + DBClientBase* DBConnectionPool::_finishCreate(const string& host, double socketTimeout, DBClientBase* conn) { @@ -219,11 +246,17 @@ DBClientBase* DBConnectionPool::_finishCreate(const string& host, throw; } + log() << "Successfully connected to " << host << " (" << openConnections(host, socketTimeout) + << " connections now open to " << host << " with a " << socketTimeout + << " second timeout)"; + return conn; } DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTimeout) { - DBClientBase* c = _get(url.toString(), socketTimeout); + auto host = url.toString(); + + DBClientBase* c = _get(host, socketTimeout); if (c) { try { onHandedOut(c); @@ -236,7 +269,7 @@ DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTi string errmsg; c = url.connect(errmsg, socketTimeout); - uassert(13328, _name + ": connect failed " + url.toString() + " : " + errmsg, c); + uassert(13328, _name + ": connect failed " + host + " : " + errmsg, c); return _finishCreate(url.toString(), socketTimeout, c); } @@ -262,6 +295,7 @@ DBClientBase* DBConnectionPool::get(const string& host, double socketTimeout) { host, 11002, str::stream() << _name << " error: " << errmsg); + return _finishCreate(host, socketTimeout, c); } @@ -284,7 +318,13 @@ void DBConnectionPool::release(const string& host, DBClientBase* c) { DBConnectionPool::~DBConnectionPool() { - // connection closing is handled by ~PoolForHost + // Do not log in destruction, because global connection pools get + // destroyed after the logging framework. + stdx::lock_guard L(_mutex); + for (PoolMap::iterator i = _pools.begin(); i != _pools.end(); i++) { + PoolForHost& p = i->second; + p._parentDestroyed = true; + } } void DBConnectionPool::flush() { @@ -361,11 +401,11 @@ void DBConnectionPool::appendConnectionStats(executor::ConnectionPoolStats* stat invariant(uri.isOK()); HostAndPort host = uri.getValue().getServers().front(); - executor::ConnectionStatsPerHost hostStats{ - static_cast(i->second.numInUse()), - static_cast(i->second.numAvailable()), - static_cast(i->second.numCreated())}; - stats->updateStatsForHost(host, hostStats); + executor::ConnectionStatsPer hostStats{static_cast(i->second.numInUse()), + static_cast(i->second.numAvailable()), + static_cast(i->second.numCreated()), + 0}; + stats->updateStatsForHost("global", host, hostStats); } } } diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h index 509965971ff..969a905df5f 100644 --- a/src/mongo/client/connpool.h +++ b/src/mongo/client/connpool.h @@ -57,12 +57,15 @@ public: // Sentinel value indicating pool has no cleanup limit static const int kPoolSizeUnlimited; + friend class DBConnectionPool; + PoolForHost() : _created(0), _minValidCreationTimeMicroSec(0), _type(ConnectionString::INVALID), _maxPoolSize(kPoolSizeUnlimited), - _checkedOut(0) {} + _checkedOut(0), + _parentDestroyed(false) {} ~PoolForHost(); @@ -80,6 +83,13 @@ public: _maxPoolSize = maxPoolSize; } + /** + * Sets the socket timeout on this host, for reporting purposes only. + */ + void setSocketTimeout(double socketTimeout) { + _socketTimeout = socketTimeout; + } + int numAvailable() const { return (int)_pool.size(); } @@ -88,6 +98,13 @@ public: return _checkedOut; } + /** + * Returns the number of open connections in this pool. + */ + int openConnections() const { + return _checkedOut + (int)_pool.size(); + } + void createdOne(DBClientBase* base); long long numCreated() const { return _created; @@ -140,6 +157,7 @@ private: }; std::string _hostName; + double _socketTimeout; std::stack _pool; int64_t _created; @@ -151,6 +169,9 @@ private: // The number of currently active connections from this pool int _checkedOut; + + // Whether our parent DBConnectionPool object is in destruction + bool _parentDestroyed; }; class DBConnectionHook { @@ -197,6 +218,11 @@ public: return _maxPoolSize; } + /** + * Returns the number of connections to the given host pool. + */ + int openConnections(const std::string& ident, double socketTimeout); + /** * Sets the maximum number of connections pooled per-host. * diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp index 706c5ed8838..6fd1c849c8d 100644 --- a/src/mongo/client/dbclient.cpp +++ b/src/mongo/client/dbclient.cpp @@ -536,7 +536,7 @@ void DBClientWithCommands::_auth(const BSONObj& params) { auth::authenticateClient( params, - HostAndPort(getServerAddress()).host(), + HostAndPort(getServerAddress()), clientName, [this](RemoteCommandRequest request, auth::AuthCompletionHandler handler) { BSONObj info; @@ -678,9 +678,11 @@ bool DBClientWithCommands::eval(const string& dbname, const string& jscode) { list DBClientWithCommands::getDatabaseNames() { BSONObj info; - uassert(10005, - "listdatabases failed", - runCommand("admin", BSON("listDatabases" << 1), info, QueryOption_SlaveOk)); + uassert( + 10005, + "listdatabases failed", + runCommand( + "admin", BSON("listDatabases" << 1 << "nameOnly" << true), info, QueryOption_SlaveOk)); uassert(10006, "listDatabases.databases not array", info["databases"].type() == Array); list names; @@ -922,6 +924,12 @@ Status DBClientConnection::connect(const HostAndPort& serverAddress) { return swIsMasterReply.getStatus(); } + // Ensure that the isMaster response is "ok:1". + auto isMasterStatus = getStatusFromCommandResult(swIsMasterReply.getValue().data); + if (!isMasterStatus.isOK()) { + return isMasterStatus; + } + auto swProtocolSet = rpc::parseProtocolSetFromIsMasterReply(swIsMasterReply.getValue().data); if (!swProtocolSet.isOK()) { return swProtocolSet.getStatus(); diff --git a/src/mongo/client/dbclientcursor.cpp b/src/mongo/client/dbclientcursor.cpp index 2e0a5bdade3..d76277d49ca 100644 --- a/src/mongo/client/dbclientcursor.cpp +++ b/src/mongo/client/dbclientcursor.cpp @@ -324,8 +324,6 @@ void DBClientCursor::dataReceived(bool& retry, string& host) { /** If true, safe to call next(). Requests more from server if necessary. */ bool DBClientCursor::more() { - _assertIfNull(); - if (!_putBack.empty()) return true; @@ -343,7 +341,6 @@ bool DBClientCursor::more() { } BSONObj DBClientCursor::next() { - DEV _assertIfNull(); if (!_putBack.empty()) { BSONObj ret = _putBack.top(); _putBack.pop(); diff --git a/src/mongo/client/dbclientcursor.h b/src/mongo/client/dbclientcursor.h index a085d90dbd9..a0d9d5e00b5 100644 --- a/src/mongo/client/dbclientcursor.h +++ b/src/mongo/client/dbclientcursor.h @@ -70,7 +70,6 @@ public: then perhaps stop. */ int objsLeftInBatch() const { - _assertIfNull(); return _putBack.size() + batch.nReturned - batch.pos; } bool moreInCurrentBatch() { @@ -143,7 +142,6 @@ public: ResultFlag_ErrSet is the possible exception to that */ bool hasResultFlag(int flag) { - _assertIfNull(); return (resultFlags & flag) != 0; } @@ -290,11 +288,6 @@ private: void requestMore(); - // Don't call from a virtual function - void _assertIfNull() const { - uassert(13348, "connection died", this); - } - // init pieces void _assembleInit(Message& toSend); }; diff --git a/src/mongo/client/mongo_uri.cpp b/src/mongo/client/mongo_uri.cpp index 443a35f9858..cbf438441f0 100644 --- a/src/mongo/client/mongo_uri.cpp +++ b/src/mongo/client/mongo_uri.cpp @@ -56,14 +56,21 @@ const char kMongoDBURL[] = "(?:([^:]+)(?::([^@]+))?@)?" // servers: grabs all host:port or UNIX socket names - "((?:(?:[^\\/]+|/.+.sock?),?)+)" + "((?:[^\\/]+|/.+\\.sock)(?:(?:[^\\/]+|/.+\\.sock),)*)" - // database: matches anything but the chars that cannot - // be part of a MongoDB database name. - "(?:/([^/\\.\\ \"*<>:\\|\\?]*))?" + // database and options are grouped together + "(?:/" + + // database: matches anything but the chars that cannot be part of a MongoDB database name which + // are (in order) - forward slash, back slash, dot, space, double-quote, dollar sign, asterisk, + // less than, greater than, colon, pipe, question mark. + "([^/\\\\\\.\\ \"\\$*<>:\\|\\?]*)?" // options - "(?:\\?(?:(.+=.+)&?)+)*"; + "(?:\\?([^&=?]+=[^&=?]+(?:&[^&=?]+=[^&=?]+)*))?" + + // close db/options group + ")?"; } // namespace @@ -86,7 +93,7 @@ StatusWith MongoURI::parse(const std::string& url) { str::stream() << "Failed to parse mongodb:// URL: " << url); } - // We have 5 top level captures, plus the whole input. + // We have the whole input plus 5 top level captures (user, password, host, db, options). invariant(matches.size() == 6); if (!matches[3].matched) { diff --git a/src/mongo/client/mongo_uri_connect.cpp b/src/mongo/client/mongo_uri_connect.cpp index ff9c048b610..cdb4766fb69 100644 --- a/src/mongo/client/mongo_uri_connect.cpp +++ b/src/mongo/client/mongo_uri_connect.cpp @@ -162,15 +162,15 @@ BSONObj MongoURI::_makeAuthObjFromOptions(int maxWireVersion) const { } DBClientBase* MongoURI::connect(std::string& errmsg, double socketTimeout) const { - auto ret = _connectString.connect(errmsg, socketTimeout); + auto ret = std::unique_ptr(_connectString.connect(errmsg, socketTimeout)); if (!ret) { - return ret; + return nullptr; } if (!_user.empty()) { ret->auth(_makeAuthObjFromOptions(ret->getMaxWireVersion())); } - return ret; + return ret.release(); } } // namespace mongo diff --git a/src/mongo/client/mongo_uri_test.cpp b/src/mongo/client/mongo_uri_test.cpp index 89cf855ffed..7a3d5f9c812 100644 --- a/src/mongo/client/mongo_uri_test.cpp +++ b/src/mongo/client/mongo_uri_test.cpp @@ -61,6 +61,8 @@ const URITestCase validCases[] = { {"mongodb://127.0.0.1/dbName?foo=a&c=b", "", "", kMaster, "", 1, 2, "dbName"}, + {"mongodb://localhost/?foo=bar", "", "", kMaster, "", 1, 1, ""}, + {"mongodb://user:pwd@127.0.0.1:1234", "user", "pwd", kMaster, "", 1, 0, ""}, {"mongodb://user@127.0.0.1:1234", "user", "", kMaster, "", 1, 0, ""}, @@ -269,9 +271,20 @@ const URITestCase validCases[] = { const InvalidURITestCase invalidCases[] = { + // No host. {"mongodb://"}, + // Needs a "/" after the hosts and before the options. {"mongodb://localhost:27017,localhost:27018?replicaSet=missingSlash"}, + + // Host list must actually be comma separated. + {"mongodb://localhost:27017localhost:27018"}, + + // Domain sockets have to end in ".sock". + {"mongodb:///notareal/domainsock"}, + + // Options can't have multiple question marks. Only one. + {"mongodb://localhost:27017/?foo=a?c=b&d=e?asdf=foo"}, }; TEST(MongoURI, GoodTrickyURIs) { diff --git a/src/mongo/client/native_sasl_client_session.cpp b/src/mongo/client/native_sasl_client_session.cpp index 7976c09e413..8f8d4fe91af 100644 --- a/src/mongo/client/native_sasl_client_session.cpp +++ b/src/mongo/client/native_sasl_client_session.cpp @@ -33,6 +33,7 @@ #include "mongo/client/sasl_client_conversation.h" #include "mongo/client/sasl_plain_client_conversation.h" #include "mongo/client/sasl_scramsha1_client_conversation.h" +#include "mongo/client/scram_sha1_client_cache.h" #include "mongo/util/mongoutils/str.h" namespace mongo { @@ -47,6 +48,9 @@ MONGO_INITIALIZER(NativeSaslClientContext)(InitializerContext* context) { return Status::OK(); } +// Global cache for SCRAM-SHA-1 credentials +SCRAMSHA1ClientCache* scramsha1ClientCache = new SCRAMSHA1ClientCache; + } // namespace NativeSaslClientSession::NativeSaslClientSession() @@ -63,7 +67,7 @@ Status NativeSaslClientSession::initialize() { if (mechanism == "PLAIN") { _saslConversation.reset(new SaslPLAINClientConversation(this)); } else if (mechanism == "SCRAM-SHA-1") { - _saslConversation.reset(new SaslSCRAMSHA1ClientConversation(this)); + _saslConversation.reset(new SaslSCRAMSHA1ClientConversation(this, scramsha1ClientCache)); } else { return Status(ErrorCodes::BadValue, mongoutils::str::stream() << "SASL mechanism " << mechanism diff --git a/src/mongo/client/sasl_client_authenticate.cpp b/src/mongo/client/sasl_client_authenticate.cpp index ecb7dfd7d1e..1d305d44d05 100644 --- a/src/mongo/client/sasl_client_authenticate.cpp +++ b/src/mongo/client/sasl_client_authenticate.cpp @@ -39,7 +39,7 @@ namespace mongo { using namespace mongoutils; void (*saslClientAuthenticate)(auth::RunCommandHook runCommand, - StringData hostname, + const HostAndPort& hostname, const BSONObj& saslParameters, auth::AuthCompletionHandler handler) = nullptr; diff --git a/src/mongo/client/sasl_client_authenticate.h b/src/mongo/client/sasl_client_authenticate.h index 8e52bee407d..cde16cd3afe 100644 --- a/src/mongo/client/sasl_client_authenticate.h +++ b/src/mongo/client/sasl_client_authenticate.h @@ -68,7 +68,7 @@ class BSONObj; * returned. */ extern void (*saslClientAuthenticate)(auth::RunCommandHook runCommand, - StringData hostname, + const HostAndPort& hostname, const BSONObj& saslParameters, auth::AuthCompletionHandler handler); diff --git a/src/mongo/client/sasl_client_authenticate_impl.cpp b/src/mongo/client/sasl_client_authenticate_impl.cpp index aa4d7e64838..35b575e545a 100644 --- a/src/mongo/client/sasl_client_authenticate_impl.cpp +++ b/src/mongo/client/sasl_client_authenticate_impl.cpp @@ -116,7 +116,7 @@ Status extractPassword(const BSONObj& saslParameters, * Returns Status::OK() on success. */ Status configureSession(SaslClientSession* session, - StringData hostname, + const HostAndPort& hostname, StringData targetDatabase, const BSONObj& saslParameters) { std::string mechanism; @@ -134,10 +134,11 @@ Status configureSession(SaslClientSession* session, session->setParameter(SaslClientSession::parameterServiceName, value); status = bsonExtractStringFieldWithDefault( - saslParameters, saslCommandServiceHostnameFieldName, hostname, &value); + saslParameters, saslCommandServiceHostnameFieldName, hostname.host(), &value); if (!status.isOK()) return status; session->setParameter(SaslClientSession::parameterServiceHostname, value); + session->setParameter(SaslClientSession::parameterServiceHostAndPort, hostname.toString()); status = bsonExtractStringField(saslParameters, saslCommandUserFieldName, &value); if (!status.isOK()) @@ -247,7 +248,7 @@ void asyncSaslConversation(auth::RunCommandHook runCommand, * "client". */ void saslClientAuthenticateImpl(auth::RunCommandHook runCommand, - StringData hostname, + const HostAndPort& hostname, const BSONObj& saslParameters, auth::AuthCompletionHandler handler) { int saslLogLevel = getSaslClientLogLevel(saslParameters); diff --git a/src/mongo/client/sasl_client_session.h b/src/mongo/client/sasl_client_session.h index c93a843c9d7..3cda8fba89c 100644 --- a/src/mongo/client/sasl_client_session.h +++ b/src/mongo/client/sasl_client_session.h @@ -63,6 +63,7 @@ public: enum Parameter { parameterServiceName = 0, parameterServiceHostname, + parameterServiceHostAndPort, parameterMechanism, parameterUser, parameterPassword, diff --git a/src/mongo/client/sasl_scramsha1_client_conversation.cpp b/src/mongo/client/sasl_scramsha1_client_conversation.cpp index e7d8643668a..fb6fb1702ea 100644 --- a/src/mongo/client/sasl_scramsha1_client_conversation.cpp +++ b/src/mongo/client/sasl_scramsha1_client_conversation.cpp @@ -34,6 +34,7 @@ #include "mongo/base/parse_number.h" #include "mongo/client/sasl_client_session.h" +#include "mongo/client/scram_sha1_client_cache.h" #include "mongo/platform/random.h" #include "mongo/util/base64.h" #include "mongo/util/mongoutils/str.h" @@ -46,13 +47,12 @@ using std::unique_ptr; using std::string; SaslSCRAMSHA1ClientConversation::SaslSCRAMSHA1ClientConversation( - SaslClientSession* saslClientSession) - : SaslClientConversation(saslClientSession), _step(0), _authMessage(""), _clientNonce("") {} - -SaslSCRAMSHA1ClientConversation::~SaslSCRAMSHA1ClientConversation() { - // clear the _saltedPassword memory - memset(_saltedPassword, 0, scram::hashSize); -} + SaslClientSession* saslClientSession, SCRAMSHA1ClientCache* clientCache) + : SaslClientConversation(saslClientSession), + _step(0), + _authMessage(), + _clientCache(clientCache), + _clientNonce() {} StatusWith SaslSCRAMSHA1ClientConversation::step(StringData inputData, std::string* outputData) { @@ -159,7 +159,7 @@ StatusWith SaslSCRAMSHA1ClientConversation::_secondStep(const std::vector< } std::string salt = input[1].substr(2); - int iterationCount; + size_t iterationCount; Status status = parseNumberFromStringWithBase(input[2].substr(2), 10, &iterationCount); if (status != Status::OK()) { @@ -178,14 +178,28 @@ StatusWith SaslSCRAMSHA1ClientConversation::_secondStep(const std::vector< return StatusWith(ex.toStatus()); } - scram::generateSaltedPassword( - _saslClientSession->getParameter(SaslClientSession::parameterPassword), - reinterpret_cast(decodedSalt.c_str()), - decodedSalt.size(), - iterationCount, - _saltedPassword); + scram::SCRAMPresecrets presecrets( + _saslClientSession->getParameter(SaslClientSession::parameterPassword).toString(), + std::vector(decodedSalt.begin(), decodedSalt.end()), + iterationCount); + + StatusWith targetHost = HostAndPort::parse( + _saslClientSession->getParameter(SaslClientSession::parameterServiceHostAndPort)); + + if (targetHost.isOK()) { + _credentials = _clientCache->getCachedSecrets(targetHost.getValue(), presecrets); + + if (!_credentials) { + _credentials = scram::generateSecrets(presecrets); + + _clientCache->setCachedSecrets( + std::move(targetHost.getValue()), std::move(presecrets), _credentials); + } + } else { + _credentials = scram::generateSecrets(presecrets); + } - std::string clientProof = scram::generateClientProof(_saltedPassword, _authMessage); + std::string clientProof = scram::generateClientProof(_credentials, _authMessage); StringBuilder sb; sb << "c=biws,r=" << nonce << ",p=" << clientProof; @@ -224,7 +238,7 @@ StatusWith SaslSCRAMSHA1ClientConversation::_thirdStep(const std::vector. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/client/scram_sha1_client_cache.h" + +namespace mongo { + +scram::SCRAMSecrets SCRAMSHA1ClientCache::getCachedSecrets( + const HostAndPort& target, const scram::SCRAMPresecrets& presecrets) const { + const stdx::lock_guard lock(_hostToSecretsMutex); + + // Search the cache for a record associated with the host we're trying to connect to. + auto foundSecret = _hostToSecrets.find(target); + if (foundSecret != _hostToSecrets.end()) { + // Presecrets contain parameters provided by the server, which may change. If the + // cached presecrets don't match the presecrets we have on hand, we must not return the + // stale cached secrets. We'll need to rerun the SCRAM computation. + const scram::SCRAMPresecrets& foundPresecrets = foundSecret->second.first; + if (foundPresecrets == presecrets) { + return foundSecret->second.second; + } + } + return {}; +} + +void SCRAMSHA1ClientCache::setCachedSecrets(HostAndPort target, + scram::SCRAMPresecrets presecrets, + scram::SCRAMSecrets secrets) { + const stdx::lock_guard lock(_hostToSecretsMutex); + + decltype(_hostToSecrets)::iterator it; + bool insertionSuccessful; + auto cacheRecord = std::make_pair(std::move(presecrets), std::move(secrets)); + // Insert the presecrets, and the secrets we computed for them into the cache + std::tie(it, insertionSuccessful) = _hostToSecrets.emplace(std::move(target), cacheRecord); + // If there was already a cache entry for the target HostAndPort, we should overwrite it. + // We have fresher presecrets and secrets. + if (!insertionSuccessful) { + it->second = std::move(cacheRecord); + } +} + +} // namespace mongo diff --git a/src/mongo/client/scram_sha1_client_cache.h b/src/mongo/client/scram_sha1_client_cache.h new file mode 100644 index 00000000000..ee595094ec5 --- /dev/null +++ b/src/mongo/client/scram_sha1_client_cache.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ +#pragma once + +#include +#include + +#include "mongo/crypto/mechanism_scram.h" +#include "mongo/platform/unordered_map.h" +#include "mongo/stdx/mutex.h" +#include "mongo/util/net/hostandport.h" + +namespace mongo { + +/** + * A cache for the intermediate steps of the SCRAM-SHA-1 computation. + * + * Clients wishing to authenticate to a server using SCRAM-SHA-1 + * must produce a set of credential objects from their password, + * a salt, and an iteration count. The computation to generate these + * is very expensive, proportional to the iteration count. The high + * cost of this computation prevents brute force attacks on + * intercepted SCRAM authentication data, or a stolen password + * database. The inputs to the function are unlikely to frequently + * change. Caching the relationship between the inputs and the + * resulting output should make repeated authentication attempts + * to a single server much faster. + * + * This is explicitly permitted by RFC5802, section 5.1: + * + * "Note that a client implementation MAY cache + * ClientKey&ServerKey (or just SaltedPassword) for later + * reauthentication to the same service, as it is likely that the + * server is going to advertise the same salt value upon + * reauthentication. This might be useful for mobile clients where + * CPU usage is a concern." + */ +class SCRAMSHA1ClientCache { +public: + /** + * Returns precomputed SCRAMSecrets, if one has already been + * stored for the specified hostname and the provided presecrets + * match those recorded for the hostname. Otherwise, no secrets + * are returned. + */ + scram::SCRAMSecrets getCachedSecrets(const HostAndPort& target, + const scram::SCRAMPresecrets& presecrets) const; + + /** + * Records a set of precomputed SCRAMSecrets for the specified + * host, along with the presecrets used to generate them. + */ + void setCachedSecrets(HostAndPort target, + scram::SCRAMPresecrets presecrets, + scram::SCRAMSecrets secrets); + +private: + mutable stdx::mutex _hostToSecretsMutex; + std::unordered_map> + _hostToSecrets; +}; + +} // namespace mongo diff --git a/src/mongo/client/syncclusterconnection.cpp b/src/mongo/client/syncclusterconnection.cpp index c15cf186267..a6b1662d4f0 100644 --- a/src/mongo/client/syncclusterconnection.cpp +++ b/src/mongo/client/syncclusterconnection.cpp @@ -276,9 +276,7 @@ BSONObj SyncClusterConnection::findOne(const string& ns, const BSONObj* fieldsToReturn, int queryOptions) { if (ns.find(".$cmd") != string::npos) { - string cmdName = query.obj.firstElementFieldName(); - - int lockType = _lockType(cmdName); + int lockType = _lockType(query); if (lockType > 0) { // write $cmd string errmsg; @@ -381,10 +379,10 @@ unique_ptr SyncClusterConnection::query(const string& ns, int batchSize) { _lastErrors.clear(); if (ns.find(".$cmd") != string::npos) { - string cmdName = query.obj.firstElementFieldName(); - int lockType = _lockType(cmdName); + int lockType = _lockType(query); uassert(13054, - (string) "write $cmd not supported in SyncClusterConnection::query for:" + cmdName, + str::stream() << "write $cmd not supported in SyncClusterConnection::query for: " + << query.toString(), lockType <= 0); } @@ -622,7 +620,14 @@ void SyncClusterConnection::say(Message& toSend, bool isRetry, string* actualSer _checkLast(); } -int SyncClusterConnection::_lockType(const string& name) { +int SyncClusterConnection::_lockType(const Query& query) { + string name = query.obj.firstElementFieldName(); + + if (name == "query") { + // actual command is embedded in the query object + name = query.obj["query"].Obj().firstElementFieldName(); + } + { stdx::lock_guard lk(_mutex); map::iterator i = _lockTypes.find(name); diff --git a/src/mongo/client/syncclusterconnection.h b/src/mongo/client/syncclusterconnection.h index 40373d4ae19..06fba3d0be2 100644 --- a/src/mongo/client/syncclusterconnection.h +++ b/src/mongo/client/syncclusterconnection.h @@ -198,7 +198,7 @@ private: const BSONObj* fieldsToReturn, int queryOptions, int batchSize); - int _lockType(const std::string& name); + int _lockType(const Query& query); void _checkLast(); void _connect(const std::string& host); diff --git a/src/mongo/crypto/SConscript b/src/mongo/crypto/SConscript index f66e88be2b3..5b8ed85a684 100644 --- a/src/mongo/crypto/SConscript +++ b/src/mongo/crypto/SConscript @@ -8,18 +8,41 @@ env.SConscript( ], ) -env.Library('crypto_tom', - ['crypto_tom.cpp'], - LIBDEPS=['tom/tomcrypt']) +env.Library('sha1_block', + source=[ + 'sha1_block.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + ]) + +env.Library( + target='sha1_block_tom', + source=[ + 'sha1_block_tom.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + 'sha1_block', + 'tom/tomcrypt' + ]) -env.Library('crypto_openssl', - ['crypto_openssl.cpp']) +env.Library('sha1_block_openssl', + source=[ + 'sha1_block_openssl.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + 'sha1_block', + ]) env.Library('scramauth', ['mechanism_scram.cpp'], LIBDEPS=['$BUILD_DIR/mongo/base', - 'crypto_${MONGO_CRYPTO}']) + '$BUILD_DIR/mongo/base/secure_allocator', + '$BUILD_DIR/mongo/util/secure_zero_memory', + 'sha1_block_${MONGO_CRYPTO}']) -env.CppUnitTest('crypto_test', - ['crypto_test.cpp'], - LIBDEPS=['crypto_${MONGO_CRYPTO}']) +env.CppUnitTest('sha1_block_test', + ['sha1_block_test.cpp'], + LIBDEPS=['sha1_block_${MONGO_CRYPTO}']) diff --git a/src/mongo/crypto/crypto.h b/src/mongo/crypto/crypto.h deleted file mode 100644 index 7b08e371a84..00000000000 --- a/src/mongo/crypto/crypto.h +++ /dev/null @@ -1,51 +0,0 @@ -/** -* Copyright (C) 2014 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects for -* all of the code used other than as permitted herein. If you modify file(s) -* with this exception, you may extend this exception to your version of the -* file(s), but you are not obligated to do so. If you do not wish to do so, -* delete this exception statement from your version. If you delete this -* exception statement from all source files in the program, then also delete -* it in the license file. -*/ - -#pragma once - -#include - -namespace mongo { -namespace crypto { -/* - * Computes a SHA-1 hash of 'input'. - */ -bool sha1(const unsigned char* input, const size_t inputLen, unsigned char* output); - -/* - * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' - */ -bool hmacSha1(const unsigned char* key, - const size_t keyLen, - const unsigned char* input, - const size_t inputLen, - unsigned char* output, - unsigned int* outputLen); - -} // namespace crypto -} // namespace mongo diff --git a/src/mongo/crypto/crypto_openssl.cpp b/src/mongo/crypto/crypto_openssl.cpp deleted file mode 100644 index 4dc1e5d02c4..00000000000 --- a/src/mongo/crypto/crypto_openssl.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2014 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/config.h" -#include "mongo/util/scopeguard.h" - -#ifndef MONGO_CONFIG_SSL -#error This file should only be included in SSL-enabled builds -#endif - -#include -#include -#include - -namespace mongo { -namespace crypto { -/* - * Computes a SHA-1 hash of 'input'. - */ -bool sha1(const unsigned char* input, const size_t inputLen, unsigned char* output) { - EVP_MD_CTX digestCtx; - EVP_MD_CTX_init(&digestCtx); - ON_BLOCK_EXIT(EVP_MD_CTX_cleanup, &digestCtx); - - if (1 != EVP_DigestInit_ex(&digestCtx, EVP_sha1(), NULL)) { - return false; - } - - if (1 != EVP_DigestUpdate(&digestCtx, input, inputLen)) { - return false; - } - - return (1 == EVP_DigestFinal_ex(&digestCtx, output, NULL)); -} - -/* - * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' - */ -bool hmacSha1(const unsigned char* key, - const size_t keyLen, - const unsigned char* input, - const size_t inputLen, - unsigned char* output, - unsigned int* outputLen) { - return HMAC(EVP_sha1(), key, keyLen, input, inputLen, output, outputLen); -} - -} // namespace crypto -} // namespace mongo diff --git a/src/mongo/crypto/crypto_test.cpp b/src/mongo/crypto/crypto_test.cpp deleted file mode 100644 index 52d488b92cf..00000000000 --- a/src/mongo/crypto/crypto_test.cpp +++ /dev/null @@ -1,521 +0,0 @@ -/** - * Copyright (C) 2014 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/crypto/crypto.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { -const int digestLen = 20; - -// SHA-1 test vectors from http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA_All.pdf -const struct { - const char* msg; - unsigned char hash[digestLen]; -} sha1Tests[] = {{"abc", - {0xa9, - 0x99, - 0x3e, - 0x36, - 0x47, - 0x06, - 0x81, - 0x6a, - 0xba, - 0x3e, - 0x25, - 0x71, - 0x78, - 0x50, - 0xc2, - 0x6c, - 0x9c, - 0xd0, - 0xd8, - 0x9d}}, - - {"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", - {0x84, - 0x98, - 0x3E, - 0x44, - 0x1C, - 0x3B, - 0xD2, - 0x6E, - 0xBA, - 0xAE, - 0x4A, - 0xA1, - 0xF9, - 0x51, - 0x29, - 0xE5, - 0xE5, - 0x46, - 0x70, - 0xF1}}}; - -TEST(CryptoVectors, SHA1) { - unsigned char sha1Result[digestLen]; - size_t numTests = sizeof(sha1Tests) / sizeof(sha1Tests[0]); - for (size_t i = 0; i < numTests; i++) { - ASSERT(crypto::sha1(reinterpret_cast(sha1Tests[i].msg), - strlen(sha1Tests[i].msg), - sha1Result)) - << "Failed SHA1 iteration " << i; - ASSERT(0 == memcmp(sha1Tests[i].hash, sha1Result, digestLen)) << "Failed SHA1 iteration " - << i; - } -} - -const int maxKeySize = 80; -const int maxDataSize = 54; -// HMAC-SHA-1 test vectors from http://tools.ietf.org/html/rfc2202.html -const struct { - unsigned char key[maxKeySize]; - int keyLen; - unsigned char data[maxDataSize]; - int dataLen; - unsigned char hash[digestLen]; -} hmacSha1Tests[] = { - // RFC test case 1 - {{0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b}, - 20, - {0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - 8, - {0xb6, - 0x17, - 0x31, - 0x86, - 0x55, - 0x05, - 0x72, - 0x64, - 0xe2, - 0x8b, - 0xc0, - 0xb6, - 0xfb, - 0x37, - 0x8c, - 0x8e, - 0xf1, - 0x46, - 0xbe, - 0x00}}, - - // RFC test case 3 - {{0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa}, - 20, - {0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd}, - 50, - {0x12, - 0x5d, - 0x73, - 0x42, - 0xb9, - 0xac, - 0x11, - 0xcd, - 0x91, - 0xa3, - 0x9a, - 0xf4, - 0x8a, - 0xa1, - 0x7b, - 0x4f, - 0x63, - 0xf1, - 0x75, - 0xd3}}, - - // RFC test case 4 - {{0x01, - 0x02, - 0x03, - 0x04, - 0x05, - 0x06, - 0x07, - 0x08, - 0x09, - 0x0a, - 0x0b, - 0x0c, - 0x0d, - 0x0e, - 0x0f, - 0x10, - 0x11, - 0x12, - 0x13, - 0x14, - 0x15, - 0x16, - 0x17, - 0x18, - 0x19}, - 25, - {0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd}, - 50, - {0x4c, - 0x90, - 0x07, - 0xf4, - 0x02, - 0x62, - 0x50, - 0xc6, - 0xbc, - 0x84, - 0x14, - 0xf9, - 0xbf, - 0x50, - 0xc8, - 0x6c, - 0x2d, - 0x72, - 0x35, - 0xda}}, - - // RFC test case 6 - {{0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa}, - 80, - {0x54, - 0x65, - 0x73, - 0x74, - 0x20, - 0x55, - 0x73, - 0x69, - 0x6e, - 0x67, - 0x20, - 0x4c, - 0x61, - 0x72, - 0x67, - 0x65, - 0x72, - 0x20, - 0x54, - 0x68, - 0x61, - 0x6e, - 0x20, - 0x42, - 0x6c, - 0x6f, - 0x63, - 0x6b, - 0x2d, - 0x53, - 0x69, - 0x7a, - 0x65, - 0x20, - 0x4b, - 0x65, - 0x79, - 0x20, - 0x2d, - 0x20, - 0x48, - 0x61, - 0x73, - 0x68, - 0x20, - 0x4b, - 0x65, - 0x79, - 0x20, - 0x46, - 0x69, - 0x72, - 0x73, - 0x74}, - 54, - {0xaa, - 0x4a, - 0xe5, - 0xe1, - 0x52, - 0x72, - 0xd0, - 0x0e, - 0x95, - 0x70, - 0x56, - 0x37, - 0xce, - 0x8a, - 0x3b, - 0x55, - 0xed, - 0x40, - 0x21, - 0x12}}}; - -TEST(CryptoVectors, HMACSHA1) { - unsigned char hmacSha1Result[digestLen]; - unsigned int hashLen = digestLen; - - size_t numTests = sizeof(hmacSha1Tests) / sizeof(hmacSha1Tests[0]); - for (size_t i = 0; i < numTests; i++) { - ASSERT(crypto::hmacSha1(hmacSha1Tests[i].key, - hmacSha1Tests[i].keyLen, - hmacSha1Tests[i].data, - hmacSha1Tests[i].dataLen, - hmacSha1Result, - &hashLen)) - << "Failed HMAC-SHA1 iteration " << i; - ASSERT(0 == memcmp(hmacSha1Tests[i].hash, hmacSha1Result, digestLen)) - << "Failed HMAC-SHA1 iteration " << i; - } -} -} // namespace -} // namespace mongo diff --git a/src/mongo/crypto/crypto_tom.cpp b/src/mongo/crypto/crypto_tom.cpp deleted file mode 100644 index a629dff945d..00000000000 --- a/src/mongo/crypto/crypto_tom.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2014 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/config.h" - -#ifdef MONGO_CONFIG_SSL -#error This file should not be included if compiling with SSL support -#endif - -#include "mongo/crypto/tom/tomcrypt.h" - -namespace mongo { -namespace crypto { -/* - * Computes a SHA-1 hash of 'input'. - */ -bool sha1(const unsigned char* input, const size_t inputLen, unsigned char* output) { - hash_state hashState; - if (sha1_init(&hashState) != CRYPT_OK) { - return false; - } - if (sha1_process(&hashState, input, inputLen) != CRYPT_OK) { - return false; - } - if (sha1_done(&hashState, output) != CRYPT_OK) { - return false; - } - - return true; -} - -/* - * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' - */ -bool hmacSha1(const unsigned char* key, - const size_t keyLen, - const unsigned char* input, - const size_t inputLen, - unsigned char* output, - unsigned int* outputLen) { - if (!key || !input || !output) { - return false; - } - - static int hashId = -1; - if (hashId == -1) { - register_hash(&sha1_desc); - hashId = find_hash("sha1"); - } - - unsigned long sha1HashLen = 20; - if (hmac_memory(hashId, key, keyLen, input, inputLen, output, &sha1HashLen) != CRYPT_OK) { - return false; - } - - *outputLen = sha1HashLen; - return true; -} - -} // namespace crypto -} // namespace mongo diff --git a/src/mongo/crypto/mechanism_scram.cpp b/src/mongo/crypto/mechanism_scram.cpp index 086e9943c4c..c18bc39cc51 100644 --- a/src/mongo/crypto/mechanism_scram.cpp +++ b/src/mongo/crypto/mechanism_scram.cpp @@ -32,29 +32,54 @@ #include -#include "mongo/crypto/crypto.h" #include "mongo/platform/random.h" #include "mongo/util/base64.h" +#include "mongo/util/secure_zero_memory.h" namespace mongo { namespace scram { using std::unique_ptr; +namespace { +/** + * Compare two arrays of bytes for equality in constant time. + * + * This means that the function runs for the same amount of time even if they differ. Unlike memcmp, + * this function does not exit on the first difference. + * + * Returns true if the two arrays are equal. + * + * TODO: evaluate if LTO inlines or changes the code flow of this function. + */ +NOINLINE_DECL +bool consttimeMemEqual(volatile const unsigned char* s1, // NOLINT - using volatile to + volatile const unsigned char* s2, // NOLINT - disable compiler optimizations + size_t length) { + unsigned int ret = 0; + + for (size_t i = 0; i < length; ++i) { + ret |= s1[i] ^ s2[i]; + } + + return (1 & ((ret - 1) >> 8)); +} +} // namespace + // Compute the SCRAM step Hi() as defined in RFC5802 -static void HMACIteration(const unsigned char input[], - size_t inputLen, - const unsigned char salt[], - size_t saltLen, - unsigned int iterationCount, - unsigned char output[]) { - unsigned char intermediateDigest[hashSize]; - unsigned char startKey[hashSize]; - // Placeholder for HMAC return size, will always be scram::hashSize for HMAC SHA-1 - unsigned int hashLen = 0; - - uassert(17450, "invalid salt length provided", saltLen + 4 == hashSize); - memcpy(startKey, salt, saltLen); +static SHA1Block HMACIteration(const unsigned char input[], + size_t inputLen, + const unsigned char salt[], + size_t saltLen, + unsigned int iterationCount) { + SHA1Block output; + SHA1Block intermediateDigest; + // Reserve a 20 byte block for the initial key. We use 16 byte salts, and must reserve an extra + // 4 bytes for a suffix mandated by RFC5802. + std::array startKey; + + uassert(17450, "invalid salt length provided", saltLen + 4 == startKey.size()); + std::copy(salt, salt + saltLen, startKey.begin()); startKey[saltLen] = 0; startKey[saltLen + 1] = 0; @@ -62,72 +87,62 @@ static void HMACIteration(const unsigned char input[], startKey[saltLen + 3] = 1; // U1 = HMAC(input, salt + 0001) - fassert(17494, crypto::hmacSha1(input, inputLen, startKey, saltLen + 4, output, &hashLen)); - - memcpy(intermediateDigest, output, hashSize); + output = SHA1Block::computeHmac(input, inputLen, startKey.data(), startKey.size()); + intermediateDigest = output; // intermediateDigest contains Ui and output contains the accumulated XOR:ed result for (size_t i = 2; i <= iterationCount; i++) { - unsigned char intermediateOutput[hashSize]; - fassert(17495, - crypto::hmacSha1( - input, inputLen, intermediateDigest, hashSize, intermediateOutput, &hashLen)); - memcpy(intermediateDigest, intermediateOutput, hashSize); - for (size_t k = 0; k < hashSize; k++) { - output[k] ^= intermediateDigest[k]; - } + intermediateDigest = SHA1Block::computeHmac( + input, inputLen, intermediateDigest.data(), intermediateDigest.size()); + output.xorInline(intermediateDigest); } + + return output; } // Iterate the hash function to generate SaltedPassword -void generateSaltedPassword(StringData hashedPassword, - const unsigned char* salt, - const int saltLen, - const int iterationCount, - unsigned char saltedPassword[hashSize]) { +SHA1Block generateSaltedPassword(const SCRAMPresecrets& presecrets) { // saltedPassword = Hi(hashedPassword, salt) - HMACIteration(reinterpret_cast(hashedPassword.rawData()), - hashedPassword.size(), - salt, - saltLen, - iterationCount, - saltedPassword); + SHA1Block saltedPassword = + HMACIteration(reinterpret_cast(presecrets.hashedPassword.c_str()), + presecrets.hashedPassword.size(), + presecrets.salt.data(), + presecrets.salt.size(), + presecrets.iterationCount); + + return saltedPassword; +} + +SCRAMSecrets generateSecrets(const SCRAMPresecrets& presecrets) { + SHA1Block saltedPassword = generateSaltedPassword(presecrets); + return generateSecrets(saltedPassword); } -void generateSecrets(const std::string& hashedPassword, - const unsigned char salt[], - size_t saltLen, - size_t iterationCount, - unsigned char storedKey[hashSize], - unsigned char serverKey[hashSize]) { - unsigned char saltedPassword[hashSize]; - unsigned char clientKey[hashSize]; - unsigned int hashLen = 0; - - generateSaltedPassword(hashedPassword, salt, saltLen, iterationCount, saltedPassword); - - // clientKey = HMAC(saltedPassword, "Client Key") - fassert(17498, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(clientKeyConst.data()), - clientKeyConst.size(), - clientKey, - &hashLen)); - - // storedKey = H(clientKey) - fassert(17499, crypto::sha1(clientKey, hashSize, storedKey)); - - // serverKey = HMAC(saltedPassword, "Server Key") - fassert(17500, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(serverKeyConst.data()), - serverKeyConst.size(), - serverKey, - &hashLen)); +SCRAMSecrets generateSecrets(const SHA1Block& saltedPassword) { + auto generateAndStoreSecrets = [&saltedPassword]( + SHA1Block& clientKey, SHA1Block& storedKey, SHA1Block& serverKey) { + + // ClientKey := HMAC(saltedPassword, "Client Key") + clientKey = + SHA1Block::computeHmac(saltedPassword.data(), + saltedPassword.size(), + reinterpret_cast(clientKeyConst.data()), + clientKeyConst.size()); + + // StoredKey := H(clientKey) + storedKey = SHA1Block::computeHash(clientKey.data(), clientKey.size()); + + // ServerKey := HMAC(SaltedPassword, "Server Key") + serverKey = + SHA1Block::computeHmac(saltedPassword.data(), + saltedPassword.size(), + reinterpret_cast(serverKeyConst.data()), + serverKeyConst.size()); + }; + return SCRAMSecrets(std::move(generateAndStoreSecrets)); } + BSONObj generateCredentials(const std::string& hashedPassword, int iterationCount) { const int saltLenQWords = 2; @@ -142,115 +157,76 @@ BSONObj generateCredentials(const std::string& hashedPassword, int iterationCoun base64::encode(reinterpret_cast(userSalt), sizeof(userSalt)); // Compute SCRAM secrets serverKey and storedKey - unsigned char storedKey[hashSize]; - unsigned char serverKey[hashSize]; - - generateSecrets(hashedPassword, - reinterpret_cast(userSalt), - saltLenQWords * sizeof(uint64_t), - iterationCount, - storedKey, - serverKey); + auto secrets = generateSecrets( + SCRAMPresecrets(hashedPassword, + std::vector(reinterpret_cast(userSalt), + reinterpret_cast(userSalt) + + saltLenQWords * sizeof(uint64_t)), + iterationCount)); - std::string encodedStoredKey = base64::encode(reinterpret_cast(storedKey), hashSize); - std::string encodedServerKey = base64::encode(reinterpret_cast(serverKey), hashSize); + std::string encodedStoredKey = secrets->storedKey.toString(); + std::string encodedServerKey = secrets->serverKey.toString(); return BSON(iterationCountFieldName << iterationCount << saltFieldName << encodedUserSalt << storedKeyFieldName << encodedStoredKey << serverKeyFieldName << encodedServerKey); } -std::string generateClientProof(const unsigned char saltedPassword[hashSize], +std::string generateClientProof(const SCRAMSecrets& clientCredentials, const std::string& authMessage) { - // ClientKey := HMAC(saltedPassword, "Client Key") - unsigned char clientKey[hashSize]; - unsigned int hashLen = 0; - fassert(18689, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(clientKeyConst.data()), - clientKeyConst.size(), - clientKey, - &hashLen)); - - // StoredKey := H(clientKey) - unsigned char storedKey[hashSize]; - fassert(18701, crypto::sha1(clientKey, hashSize, storedKey)); - // ClientSignature := HMAC(StoredKey, AuthMessage) - unsigned char clientSignature[hashSize]; - fassert(18702, - crypto::hmacSha1(storedKey, - hashSize, - reinterpret_cast(authMessage.c_str()), - authMessage.size(), - clientSignature, - &hashLen)); - - // ClientProof := ClientKey XOR ClientSignature - unsigned char clientProof[hashSize]; - for (size_t i = 0; i < hashSize; i++) { - clientProof[i] = clientKey[i] ^ clientSignature[i]; - } - - return base64::encode(reinterpret_cast(clientProof), hashSize); + SHA1Block clientSignature = + SHA1Block::computeHmac(clientCredentials->storedKey.data(), + clientCredentials->storedKey.size(), + reinterpret_cast(authMessage.c_str()), + authMessage.size()); + + clientSignature.xorInline(clientCredentials->clientKey); + return clientSignature.toString(); } -/** - * Compare two arrays of bytes for equality in constant time. - * - * This means that the function runs for the same amount of time even if they differ. Unlike memcmp, - * this function does not exit on the first difference. - * - * Returns true if the two arrays are equal. - * - * TODO: evaluate if LTO inlines or changes the code flow of this function. - */ -NOINLINE_DECL -bool memequal(volatile const unsigned char* s1, volatile const unsigned char* s2, size_t length) { - unsigned char ret = 0; - - for (size_t i = 0; i < length; ++i) { - ret |= s1[i] ^ s2[i]; - } - - return ret == 0; -} - -bool verifyServerSignature(const unsigned char saltedPassword[hashSize], +bool verifyServerSignature(const SCRAMSecrets& clientCredentials, const std::string& authMessage, const std::string& receivedServerSignature) { - // ServerKey := HMAC(SaltedPassword, "Server Key") - unsigned int hashLen; - unsigned char serverKey[hashSize]; - fassert(18703, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(serverKeyConst.data()), - serverKeyConst.size(), - serverKey, - &hashLen)); - // ServerSignature := HMAC(ServerKey, AuthMessage) - unsigned char serverSignature[hashSize]; - fassert(18704, - crypto::hmacSha1(serverKey, - hashSize, - reinterpret_cast(authMessage.c_str()), - authMessage.size(), - serverSignature, - &hashLen)); - - std::string encodedServerSignature = - base64::encode(reinterpret_cast(serverSignature), sizeof(serverSignature)); + SHA1Block serverSignature = + SHA1Block::computeHmac(clientCredentials->serverKey.data(), + clientCredentials->serverKey.size(), + reinterpret_cast(authMessage.c_str()), + authMessage.size()); + + std::string encodedServerSignature = serverSignature.toString(); if (encodedServerSignature.size() != receivedServerSignature.size()) { return false; } - return memequal(reinterpret_cast(encodedServerSignature.c_str()), - reinterpret_cast(receivedServerSignature.c_str()), - encodedServerSignature.size()); + return consttimeMemEqual( + reinterpret_cast(encodedServerSignature.c_str()), + reinterpret_cast(receivedServerSignature.c_str()), + encodedServerSignature.size()); +} + +bool verifyClientProof(StringData clientProof, StringData storedKey, StringData authMessage) { + // ClientSignature := HMAC(StoredKey, AuthMessage) + SHA1Block clientSignature = + SHA1Block::computeHmac(reinterpret_cast(storedKey.rawData()), + storedKey.size(), + reinterpret_cast(authMessage.rawData()), + authMessage.size()); + + auto clientProofSHA1Status = SHA1Block::fromBuffer( + reinterpret_cast(clientProof.rawData()), clientProof.size()); + uassertStatusOK(clientProofSHA1Status); + clientSignature.xorInline(clientProofSHA1Status.getValue()); + + // StoredKey := H(clientSignature) + SHA1Block computedStoredKey = + SHA1Block::computeHash(clientSignature.data(), clientSignature.size()); + + return consttimeMemEqual(reinterpret_cast(storedKey.rawData()), + computedStoredKey.data(), + computedStoredKey.size()); } } // namespace scram diff --git a/src/mongo/crypto/mechanism_scram.h b/src/mongo/crypto/mechanism_scram.h index 65e77b38f74..92b79efffb9 100644 --- a/src/mongo/crypto/mechanism_scram.h +++ b/src/mongo/crypto/mechanism_scram.h @@ -30,12 +30,13 @@ #include +#include "mongo/base/secure_allocator.h" #include "mongo/base/status.h" +#include "mongo/crypto/sha1_block.h" #include "mongo/db/jsobj.h" namespace mongo { namespace scram { -const unsigned int hashSize = 20; const std::string serverKeyConst = "Server Key"; const std::string clientKeyConst = "Client Key"; @@ -45,25 +46,90 @@ const std::string saltFieldName = "salt"; const std::string storedKeyFieldName = "storedKey"; const std::string serverKeyFieldName = "serverKey"; +/* + * The precursors necessary to perform the computation which produces SCRAMSecrets. + * These are the original password, its salt, and the number of times it must be + * hashed to produce the SaltedPassword used to generate the rest of the SCRAMSecrets. + */ +struct SCRAMPresecrets { + SCRAMPresecrets(std::string hashedPassword, + std::vector salt, + size_t iterationCount) + : hashedPassword(std::move(hashedPassword)), + salt(std::move(salt)), + iterationCount(iterationCount) {} + + std::string hashedPassword; + std::vector salt; + size_t iterationCount; +}; + +inline bool operator==(const SCRAMPresecrets& lhs, const SCRAMPresecrets& rhs) { + return lhs.hashedPassword == rhs.hashedPassword && lhs.salt == rhs.salt && + lhs.iterationCount == rhs.iterationCount; +} + /* * Computes the SaltedPassword from password, salt and iterationCount. */ -void generateSaltedPassword(StringData hashedPassword, - const unsigned char* salt, - const int saltLen, - const int iterationCount, - unsigned char saltedPassword[hashSize]); +SHA1Block generateSaltedPassword(const SCRAMPresecrets& presecrets); /* - * Computes the SCRAM secrets storedKey and serverKey using the salt 'salt' + * Stores all of the keys, generated from a password, needed for a client or server to perform a + * SCRAM handshake. + * These keys are reference counted, and allocated using the SecureAllocator. + * May be unpopulated. SCRAMSecrets created via the default constructor are unpopulated. + * The behavior is undefined if the accessors are called when unpopulated. + */ +class SCRAMSecrets { +private: + struct SCRAMSecretsHolder { + SHA1Block clientKey; + SHA1Block storedKey; + SHA1Block serverKey; + }; + +public: + // Creates an unpopulated SCRAMSecrets object. + SCRAMSecrets() = default; + + // Creates a populated SCRAMSecrets object. First, allocates secure storage, then provides it + // to a callback, which fills the memory. + template + explicit SCRAMSecrets(T initializationFun) + : _ptr(std::make_shared>()) { + initializationFun((*this)->clientKey, (*this)->storedKey, (*this)->serverKey); + } + + // Returns true if the underlying shared_pointer is populated. + explicit operator bool() const { + return static_cast(_ptr); + } + + const SecureHandle& operator*() const { + invariant(_ptr); + return *_ptr; + } + + const SecureHandle& operator->() const { + invariant(_ptr); + return *_ptr; + } + +private: + std::shared_ptr> _ptr; +}; + +/* + * Computes the SCRAM secrets clientKey, storedKey, and serverKey using the salt 'salt' * and iteration count 'iterationCount' as defined in RFC5802 (server side). */ -void generateSecrets(const std::string& hashedPassword, - const unsigned char salt[], - size_t saltLen, - size_t iterationCount, - unsigned char storedKey[hashSize], - unsigned char serverKey[hashSize]); +SCRAMSecrets generateSecrets(const SCRAMPresecrets& presecrets); + +/* + * Computes the ClientKey and StoredKey from SaltedPassword (client side). + */ +SCRAMSecrets generateSecrets(const SHA1Block& saltedPassword); /* * Generates the user salt and the SCRAM secrets storedKey and serverKey as @@ -72,9 +138,9 @@ void generateSecrets(const std::string& hashedPassword, BSONObj generateCredentials(const std::string& hashedPassword, int iterationCount); /* - * Computes the ClientProof from SaltedPassword and authMessage (client side). + * Computes the ClientProof from ClientKey, StoredKey, and authMessage (client side). */ -std::string generateClientProof(const unsigned char saltedPassword[hashSize], +std::string generateClientProof(const SCRAMSecrets& clientCredentials, const std::string& authMessage); /* @@ -89,8 +155,14 @@ bool validatePassword(const std::string& hashedPassword, /* * Verifies ServerSignature (client side). */ -bool verifyServerSignature(const unsigned char saltedPassword[hashSize], +bool verifyServerSignature(const SCRAMSecrets& clientCredentials, const std::string& authMessage, const std::string& serverSignature); + +/* + * Verifies ClientProof (server side). + */ +bool verifyClientProof(StringData clientProof, StringData storedKey, StringData authMessage); + } // namespace scram } // namespace mongo diff --git a/src/mongo/crypto/sha1_block.cpp b/src/mongo/crypto/sha1_block.cpp new file mode 100644 index 00000000000..43565419a7d --- /dev/null +++ b/src/mongo/crypto/sha1_block.cpp @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2017 MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/crypto/sha1_block.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/base64.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +const size_t SHA1Block::kHashLength = 20; + +SHA1Block::SHA1Block(HashType hash) : _hash(std::move(hash)) {} + +StatusWith SHA1Block::fromBuffer(const uint8_t* input, size_t inputLen) { + if (inputLen != kHashLength) { + return {ErrorCodes::InvalidLength, + str::stream() << "Unsupported SHA1Hash hash length: " << inputLen}; + } + + HashType newHash; + memcpy(newHash.data(), input, inputLen); + return SHA1Block(newHash); +} + +StatusWith SHA1Block::fromBinData(const BSONBinData& binData) { + if (binData.type != BinDataGeneral) { + return {ErrorCodes::UnsupportedFormat, "SHA1Block only accepts BinDataGeneral type"}; + } + + if (binData.length != kHashLength) { + return {ErrorCodes::UnsupportedFormat, + str::stream() << "Unsupported SHA1Block hash length: " << binData.length}; + } + + HashType newHash; + memcpy(newHash.data(), binData.data, binData.length); + return SHA1Block(newHash); +} + +std::string SHA1Block::toString() const { + return base64::encode(reinterpret_cast(_hash.data()), _hash.size()); +} + +void SHA1Block::appendAsBinData(BSONObjBuilder& builder, StringData fieldName) { + builder.appendBinData(fieldName, _hash.size(), BinDataGeneral, _hash.data()); +} + +void SHA1Block::xorInline(const SHA1Block& other) { + for (size_t x = 0; x < _hash.size(); x++) { + _hash[x] ^= other._hash[x]; + } +} + +bool SHA1Block::operator==(const SHA1Block& rhs) const { + return rhs._hash == this->_hash; +} + +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block.h b/src/mongo/crypto/sha1_block.h new file mode 100644 index 00000000000..266f6602e6c --- /dev/null +++ b/src/mongo/crypto/sha1_block.h @@ -0,0 +1,113 @@ +/** +* Copyright (C) 2017 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status_with.h" + +namespace mongo { + +struct BSONBinData; +class BSONObjBuilder; + +/** + * Data structure with fixed sized byte array that can be used as HMAC key or result of a SHA1 + * computation. + */ +class SHA1Block { +public: + static const size_t kHashLength; // 20 + using HashType = std::array; + + SHA1Block() = default; + SHA1Block(HashType rawHash); + static StatusWith fromBuffer(const uint8_t* input, size_t inputLen); + + /** + * Computes a SHA-1 hash of 'input'. + */ + static SHA1Block computeHash(const uint8_t* input, size_t inputLen); + + /** + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' + */ + static SHA1Block computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen) { + SHA1Block output; + SHA1Block::computeHmac(key, keyLen, input, inputLen, &output); + return output; + } + + /** + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key'. Writes the results into + * a pre-allocated SHA1Block. This lets us allocate SHA1Blocks with the SecureAllocator. + */ + static void computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen, + SHA1Block* const output); + + const uint8_t* data() const { + return _hash.data(); + } + + size_t size() const { + return _hash.size(); + } + + /** + * Make a new SHA1Block from a BSON BinData value. + */ + static StatusWith fromBinData(const BSONBinData& binData); + + /** + * Append this to a builder using the given name as a BSON BinData type value. + */ + void appendAsBinData(BSONObjBuilder& builder, StringData fieldName); + + /** + * Do a bitwise xor against another SHA1Block and replace the current contents of this block + * with the result. + */ + void xorInline(const SHA1Block& other); + + std::string toString() const; + bool operator==(const SHA1Block& rhs) const; + +private: + HashType _hash; +}; + +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block_openssl.cpp b/src/mongo/crypto/sha1_block_openssl.cpp new file mode 100644 index 00000000000..ce0cab8e9e6 --- /dev/null +++ b/src/mongo/crypto/sha1_block_openssl.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/crypto/sha1_block.h" + +#include "mongo/config.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/scopeguard.h" + +#ifndef MONGO_CONFIG_SSL +#error This file should only be included in SSL-enabled builds +#endif + +#include +#include +#include +#include +#if OPENSSL_VERSION_NUMBER < 0x10100000L +namespace { +// Copies of OpenSSL after 1.1.0 define new EVP digest routines. We must +// polyfill used definitions to interact with older OpenSSL versions. +EVP_MD_CTX* EVP_MD_CTX_new() { + void* ret = OPENSSL_malloc(sizeof(EVP_MD_CTX)); + + if (ret != NULL) { + memset(ret, 0, sizeof(EVP_MD_CTX)); + } + return static_cast(ret); +} + +void EVP_MD_CTX_free(EVP_MD_CTX* ctx) { + EVP_MD_CTX_cleanup(ctx); + OPENSSL_free(ctx); +} +} // namespace +#endif + +namespace mongo { +/* + * Computes a SHA-1 hash of 'input'. + */ +SHA1Block SHA1Block::computeHash(const uint8_t* input, size_t inputLen) { + HashType output; + + EVP_MD_CTX digestCtx; + EVP_MD_CTX_init(&digestCtx); + ON_BLOCK_EXIT(EVP_MD_CTX_cleanup, &digestCtx); + + fassert(40379, + EVP_DigestInit_ex(&digestCtx, EVP_sha1(), NULL) == 1 && + EVP_DigestUpdate(&digestCtx, input, inputLen) == 1 && + EVP_DigestFinal_ex(&digestCtx, output.data(), NULL) == 1); + return SHA1Block(output); +} + +/* + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key', writes output into 'output'. + */ +void SHA1Block::computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen, + SHA1Block* const output) { + fassert(40380, + HMAC(EVP_sha1(), key, keyLen, input, inputLen, output->_hash.data(), NULL) != NULL); +} + +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block_test.cpp b/src/mongo/crypto/sha1_block_test.cpp new file mode 100644 index 00000000000..651b2e52bfe --- /dev/null +++ b/src/mongo/crypto/sha1_block_test.cpp @@ -0,0 +1,565 @@ +/** + * Copyright (C) 2014 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/crypto/sha1_block.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +// SHA-1 test vectors from http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA_All.pdf +const struct { + const char* msg; + SHA1Block hash; +} sha1Tests[] = {{"abc", + SHA1Block::HashType{0xa9, + 0x99, + 0x3e, + 0x36, + 0x47, + 0x06, + 0x81, + 0x6a, + 0xba, + 0x3e, + 0x25, + 0x71, + 0x78, + 0x50, + 0xc2, + 0x6c, + 0x9c, + 0xd0, + 0xd8, + 0x9d}}, + + {"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + SHA1Block::HashType{0x84, + 0x98, + 0x3E, + 0x44, + 0x1C, + 0x3B, + 0xD2, + 0x6E, + 0xBA, + 0xAE, + 0x4A, + 0xA1, + 0xF9, + 0x51, + 0x29, + 0xE5, + 0xE5, + 0x46, + 0x70, + 0xF1}}}; + +TEST(CryptoVectors, SHA1) { + size_t numTests = sizeof(sha1Tests) / sizeof(sha1Tests[0]); + for (size_t i = 0; i < numTests; i++) { + SHA1Block result = SHA1Block::computeHash( + reinterpret_cast(sha1Tests[i].msg), strlen(sha1Tests[i].msg)); + ASSERT(sha1Tests[i].hash == result) << "Failed SHA1 iteration " << i; + } +} + +const int maxKeySize = 80; +const int maxDataSize = 54; +// HMAC-SHA-1 test vectors from http://tools.ietf.org/html/rfc2202.html +const struct { + unsigned char key[maxKeySize]; + int keyLen; + unsigned char data[maxDataSize]; + int dataLen; + SHA1Block hash; +} hmacSha1Tests[] = { + // RFC test case 1 + {{0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b}, + 20, + {0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, + 8, + SHA1Block::HashType{0xb6, + 0x17, + 0x31, + 0x86, + 0x55, + 0x05, + 0x72, + 0x64, + 0xe2, + 0x8b, + 0xc0, + 0xb6, + 0xfb, + 0x37, + 0x8c, + 0x8e, + 0xf1, + 0x46, + 0xbe, + 0x00}}, + + // RFC test case 3 + {{0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa}, + 20, + {0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd}, + 50, + SHA1Block::HashType{0x12, + 0x5d, + 0x73, + 0x42, + 0xb9, + 0xac, + 0x11, + 0xcd, + 0x91, + 0xa3, + 0x9a, + 0xf4, + 0x8a, + 0xa1, + 0x7b, + 0x4f, + 0x63, + 0xf1, + 0x75, + 0xd3}}, + + // RFC test case 4 + {{0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0a, + 0x0b, + 0x0c, + 0x0d, + 0x0e, + 0x0f, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19}, + 25, + {0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd}, + 50, + SHA1Block::HashType{0x4c, + 0x90, + 0x07, + 0xf4, + 0x02, + 0x62, + 0x50, + 0xc6, + 0xbc, + 0x84, + 0x14, + 0xf9, + 0xbf, + 0x50, + 0xc8, + 0x6c, + 0x2d, + 0x72, + 0x35, + 0xda}}, + + // RFC test case 6 + {{0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa}, + 80, + {0x54, + 0x65, + 0x73, + 0x74, + 0x20, + 0x55, + 0x73, + 0x69, + 0x6e, + 0x67, + 0x20, + 0x4c, + 0x61, + 0x72, + 0x67, + 0x65, + 0x72, + 0x20, + 0x54, + 0x68, + 0x61, + 0x6e, + 0x20, + 0x42, + 0x6c, + 0x6f, + 0x63, + 0x6b, + 0x2d, + 0x53, + 0x69, + 0x7a, + 0x65, + 0x20, + 0x4b, + 0x65, + 0x79, + 0x20, + 0x2d, + 0x20, + 0x48, + 0x61, + 0x73, + 0x68, + 0x20, + 0x4b, + 0x65, + 0x79, + 0x20, + 0x46, + 0x69, + 0x72, + 0x73, + 0x74}, + 54, + SHA1Block::HashType{0xaa, + 0x4a, + 0xe5, + 0xe1, + 0x52, + 0x72, + 0xd0, + 0x0e, + 0x95, + 0x70, + 0x56, + 0x37, + 0xce, + 0x8a, + 0x3b, + 0x55, + 0xed, + 0x40, + 0x21, + 0x12}}}; + +TEST(CryptoVectors, HMACSHA1) { + size_t numTests = sizeof(hmacSha1Tests) / sizeof(hmacSha1Tests[0]); + for (size_t i = 0; i < numTests; i++) { + SHA1Block result = SHA1Block::computeHmac(hmacSha1Tests[i].key, + hmacSha1Tests[i].keyLen, + hmacSha1Tests[i].data, + hmacSha1Tests[i].dataLen); + ASSERT(hmacSha1Tests[i].hash == result) << "Failed HMAC-SHA1 iteration " << i; + } +} + +TEST(SHA1Block, BinDataRoundTrip) { + SHA1Block::HashType rawHash; + rawHash.fill(0); + for (size_t i = 0; i < rawHash.size(); i++) { + rawHash[i] = i; + } + + SHA1Block testHash(rawHash); + + BSONObjBuilder builder; + testHash.appendAsBinData(builder, "hash"); + auto newObj = builder.done(); + + auto hashElem = newObj["hash"]; + ASSERT_EQ(BinData, hashElem.type()); + ASSERT_EQ(BinDataGeneral, hashElem.binDataType()); + + int binLen = 0; + auto rawBinData = hashElem.binData(binLen); + ASSERT_EQ(SHA1Block::kHashLength, static_cast(binLen)); + + auto newHashStatus = + SHA1Block::fromBinData(BSONBinData(rawBinData, binLen, hashElem.binDataType())); + ASSERT_OK(newHashStatus.getStatus()); + ASSERT_TRUE(testHash == newHashStatus.getValue()); +} + +TEST(SHA1Block, CanOnlyConstructFromBinGeneral) { + std::string dummy(SHA1Block::kHashLength, 'x'); + + auto newHashStatus = SHA1Block::fromBinData(BSONBinData(dummy.c_str(), dummy.size(), newUUID)); + ASSERT_EQ(ErrorCodes::UnsupportedFormat, newHashStatus.getStatus()); +} + +TEST(SHA1Block, FromBinDataShouldRegectWrongSize) { + std::string dummy(SHA1Block::kHashLength - 1, 'x'); + + auto newHashStatus = + SHA1Block::fromBinData(BSONBinData(dummy.c_str(), dummy.size(), BinDataGeneral)); + ASSERT_EQ(ErrorCodes::UnsupportedFormat, newHashStatus.getStatus()); +} + +TEST(SHA1Block, FromBufferShouldRejectWrongLength) { + std::string dummy(SHA1Block::kHashLength - 1, 'x'); + + auto newHashStatus = + SHA1Block::fromBuffer(reinterpret_cast(dummy.c_str()), dummy.size()); + ASSERT_EQ(ErrorCodes::InvalidLength, newHashStatus.getStatus()); +} + + +} // namespace +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block_tom.cpp b/src/mongo/crypto/sha1_block_tom.cpp new file mode 100644 index 00000000000..7d32dbc2f53 --- /dev/null +++ b/src/mongo/crypto/sha1_block_tom.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2014 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/crypto/sha1_block.h" + +#include "mongo/config.h" +#include "mongo/util/assert_util.h" + +#ifdef MONGO_CONFIG_SSL +#error This file should not be included if compiling with SSL support +#endif + +#include "mongo/crypto/tom/tomcrypt.h" + +namespace mongo { + +/* + * Computes a SHA-1 hash of 'input'. + */ +SHA1Block SHA1Block::computeHash(const uint8_t* input, size_t inputLen) { + HashType output; + + hash_state hashState; + fassert(40381, + sha1_init(&hashState) == CRYPT_OK && + sha1_process(&hashState, input, inputLen) == CRYPT_OK && + sha1_done(&hashState, output.data()) == CRYPT_OK); + return SHA1Block(output); +} + +/* + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' + */ +void SHA1Block::computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen, + SHA1Block* const output) { + invariant(key && input); + + static int hashId = -1; + if (hashId == -1) { + register_hash(&sha1_desc); + hashId = find_hash("sha1"); + } + + unsigned long sha1HashLen = 20; + fassert(40382, + hmac_memory(hashId, key, keyLen, input, inputLen, output->_hash.data(), &sha1HashLen) == + CRYPT_OK); +} + +} // namespace mongo diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 628bb239813..a8ce1b54416 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -618,7 +618,6 @@ serverOnlyFiles = [ "repl/storage_interface_impl.cpp", "repl/sync_source_feedback.cpp", "service_context_d.cpp", - "stats/fill_locker_info.cpp", "stats/lock_server_status_section.cpp", "stats/range_deleter_server_status.cpp", "stats/snapshots.cpp", @@ -689,6 +688,7 @@ serveronlyLibdeps = [ "s/sharding", "startup_warnings_mongod", "stats/counters", + "stats/fill_locker_info", "stats/top", "storage/devnull/storage_devnull", "storage/ephemeral_for_test/storage_ephemeral_for_test", diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index 90c7a01e72c..1201d5e8d0e 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -178,3 +178,20 @@ env.CppUnitTest('authorization_manager_test', 'authorization_manager_test.cpp', LIBDEPS=['authcore', 'authmocks', 'saslauth']) env.CppUnitTest('authorization_session_test', 'authorization_session_test.cpp', LIBDEPS=['authcore', 'authmocks', 'saslauth']) + +env.Library( + target='authorization_session_for_test', + source=[ + 'authorization_session_for_test.cpp', + ], + LIBDEPS=[ + 'authcore', + ] +) + +env.CppUnitTest('sasl_scramsha1_test', + 'sasl_scramsha1_test.cpp', + LIBDEPS=[ + 'saslauth', + '$BUILD_DIR/mongo/client/sasl_client', + ]) diff --git a/src/mongo/db/auth/action_set.cpp b/src/mongo/db/auth/action_set.cpp index 6f89222679b..b9b11981aad 100644 --- a/src/mongo/db/auth/action_set.cpp +++ b/src/mongo/db/auth/action_set.cpp @@ -41,6 +41,12 @@ namespace mongo { +ActionSet::ActionSet(std::initializer_list actions) { + for (auto& action : actions) { + addAction(action); + } +} + void ActionSet::addAction(const ActionType& action) { if (action == ActionType::anyAction) { addAllActions(); diff --git a/src/mongo/db/auth/action_set.h b/src/mongo/db/auth/action_set.h index 339a68f019c..3bea5d9fdc1 100644 --- a/src/mongo/db/auth/action_set.h +++ b/src/mongo/db/auth/action_set.h @@ -44,6 +44,7 @@ namespace mongo { class ActionSet { public: ActionSet() : _actions(0) {} + ActionSet(std::initializer_list actions); void addAction(const ActionType& action); void addAllActionsFromSet(const ActionSet& actionSet); diff --git a/src/mongo/db/auth/authorization_session.cpp b/src/mongo/db/auth/authorization_session.cpp index 4f80f19439e..676fccd070d 100644 --- a/src/mongo/db/auth/authorization_session.cpp +++ b/src/mongo/db/auth/authorization_session.cpp @@ -467,7 +467,12 @@ static int buildResourceSearchList(const ResourcePattern& target, resourceSearchList[size++] = ResourcePattern::forAnyResource(); if (target.isExactNamespacePattern()) { if (!target.ns().isSystem()) { - resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); + // Some databases should not be matchable with ResourcePattern::forAnyNormalResource. + // 'local' and 'config' are used to store special system collections, which user level + // administrators should not be able to manipulate. + if (target.ns().db() != "local" && target.ns().db() != "config") { + resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); + } resourceSearchList[size++] = ResourcePattern::forDatabaseName(target.ns().db()); } resourceSearchList[size++] = ResourcePattern::forCollectionName(target.ns().coll()); diff --git a/src/mongo/db/auth/authorization_session.h b/src/mongo/db/auth/authorization_session.h index 7b5ce4799fa..61c6093bcf7 100644 --- a/src/mongo/db/auth/authorization_session.h +++ b/src/mongo/db/auth/authorization_session.h @@ -93,7 +93,7 @@ public: // Takes ownership of the externalState. explicit AuthorizationSession(std::unique_ptr externalState); - ~AuthorizationSession(); + virtual ~AuthorizationSession(); AuthorizationManager& getAuthorizationManager(); @@ -256,16 +256,24 @@ public: // called. bool isImpersonating() const; -private: - // If any users authenticated on this session are marked as invalid this updates them with - // up-to-date information. May require a read lock on the "admin" db to read the user data. - void _refreshUserInfoAsNeeded(OperationContext* txn); - +protected: // Builds a vector of all roles held by users who are authenticated on this connection. The // vector is stored in _authenticatedRoleNames. This function is called when users are // logged in or logged out, as well as when the user cache is determined to be out of date. void _buildAuthenticatedRolesVector(); + // All Users who have been authenticated on this connection. + UserSet _authenticatedUsers; + + // The roles of the authenticated users. This vector is generated when the authenticated + // users set is changed. + std::vector _authenticatedRoleNames; + +private: + // If any users authenticated on this session are marked as invalid this updates them with + // up-to-date information. May require a read lock on the "admin" db to read the user data. + void _refreshUserInfoAsNeeded(OperationContext* txn); + // Checks if this connection is authorized for the given Privilege, ignoring whether or not // we should even be doing authorization checks in general. Note: this may acquire a read // lock on the admin database (to update out-of-date user privilege information). @@ -273,12 +281,6 @@ private: std::unique_ptr _externalState; - // All Users who have been authenticated on this connection. - UserSet _authenticatedUsers; - // The roles of the authenticated users. This vector is generated when the authenticated - // users set is changed. - std::vector _authenticatedRoleNames; - // A vector of impersonated UserNames and a vector of those users' RoleNames. // These are used in the auditing system. They are not used for authz checks. std::vector _impersonatedUserNames; diff --git a/src/mongo/db/auth/authorization_session_for_test.cpp b/src/mongo/db/auth/authorization_session_for_test.cpp new file mode 100644 index 00000000000..a38891a3f23 --- /dev/null +++ b/src/mongo/db/auth/authorization_session_for_test.cpp @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/authorization_session_for_test.h" + +#include +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user_set.h" +#include "mongo/stdx/memory.h" + +namespace mongo { +const StringData AuthorizationSessionForTest::kTestDBName = + StringData("authorizationSessionForTestDB"); + +AuthorizationSessionForTest::~AuthorizationSessionForTest() { + revokeAllPrivileges(); +} + +void AuthorizationSessionForTest::assumePrivilegesForDB(Privilege privilege, StringData dbName) { + assumePrivilegesForDB(std::vector{privilege}, dbName); +} + +void AuthorizationSessionForTest::assumePrivilegesForDB(PrivilegeVector privileges, + StringData dbName) { + auto user = stdx::make_unique(UserName("authorizationSessionForTestUser", dbName)); + user->addPrivileges(privileges); + + _authenticatedUsers.add(user.get()); + _testUsers.emplace_back(std::move(user)); + _buildAuthenticatedRolesVector(); +} + +void AuthorizationSessionForTest::revokePrivilegesForDB(StringData dbName) { + _authenticatedUsers.removeByDBName(dbName); + _testUsers.erase(std::remove_if(_testUsers.begin(), + _testUsers.end(), + [&](const std::unique_ptr& user) { + return dbName == user->getName().getDB(); + }), + _testUsers.end()); +} + +void AuthorizationSessionForTest::revokeAllPrivileges() { + _testUsers.erase(std::remove_if(_testUsers.begin(), + _testUsers.end(), + [&](const std::unique_ptr& user) { + _authenticatedUsers.removeByDBName(user->getName().getDB()); + return true; + }), + _testUsers.end()); +} +} // namespace mongo diff --git a/src/mongo/db/auth/authorization_session_for_test.h b/src/mongo/db/auth/authorization_session_for_test.h new file mode 100644 index 00000000000..84ec29df5ea --- /dev/null +++ b/src/mongo/db/auth/authorization_session_for_test.h @@ -0,0 +1,85 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/authz_session_external_state.h" +#include "mongo/db/auth/user.h" + +namespace mongo { + +class AuthorizationSessionForTest : public AuthorizationSession { + MONGO_DISALLOW_COPYING(AuthorizationSessionForTest); + +public: + // A database name used for testing purposes, deliberately named to minimize collisions with + // other test users. + static const StringData kTestDBName; + + /** + * Construct a new AuthorizationSessionForTest, taking ownership of 'externalState'. + */ + explicit AuthorizationSessionForTest(std::unique_ptr externalState) + : AuthorizationSession(std::move(externalState)) {} + + /** + * Cleans up any privileges granted via assumePrivilegesForDB(). + */ + ~AuthorizationSessionForTest(); + + /** + * Grants this session all privileges in 'privileges' for the database named 'dbName'. Any prior + * privileges granted on 'dbName' via a call to this method are erased. + * + * Do not use this method if also adding users via addAndAuthorizeUser() in the same database. + */ + void assumePrivilegesForDB(PrivilegeVector privilege, StringData dbName = kTestDBName); + void assumePrivilegesForDB(Privilege privilege, StringData dbName = kTestDBName); + + /** + * Revoke all privileges granted via assumePrivilegesForDB() on the database named 'dbName'. + * + * Do not use this method if also adding users via addAndAuthorizeUser() in the same database. + */ + void revokePrivilegesForDB(StringData dbName); + + /** + * Revokes all privileges granted via assumePrivilegesForDB() on every database. + * + * Do not use this method if also adding users via addAndAuthorizeUser() in the same database. + */ + void revokeAllPrivileges(); + +private: + std::vector> _testUsers; +}; +} // namespace mongo diff --git a/src/mongo/db/auth/privilege_parser.cpp b/src/mongo/db/auth/privilege_parser.cpp index 140fbde2d7f..d1131c8881f 100644 --- a/src/mongo/db/auth/privilege_parser.cpp +++ b/src/mongo/db/auth/privilege_parser.cpp @@ -93,8 +93,13 @@ bool ParsedResource::isValid(std::string* errMsg) const { } if (isCollectionSet() && (!NamespaceString::validCollectionName(getCollection()) && !getCollection().empty())) { - *errMsg = stream() << getCollection() << " is not a valid collection name"; - return false; + // local.oplog.$main is a real collection that the server will create. But, collection + // names with a '$' character are illegal. We must make an exception for this collection + // here so we can grant users access to it. + if (!(getDb() == "local" && getCollection() == "oplog.$main")) { + *errMsg = stream() << getCollection() << " is not a valid collection name"; + return false; + } } return true; } diff --git a/src/mongo/db/auth/role_graph_builtin_roles.cpp b/src/mongo/db/auth/role_graph_builtin_roles.cpp index 8a7cf4f3b8a..ba0785aef09 100644 --- a/src/mongo/db/auth/role_graph_builtin_roles.cpp +++ b/src/mongo/db/auth/role_graph_builtin_roles.cpp @@ -361,14 +361,17 @@ void addClusterMonitorPrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyNormalResource(), clusterMonitorRoleDatabaseActions)); - addReadOnlyDbPrivileges(privileges, "config"); Privilege::addPrivilegeToPrivilegeVector( privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("local.system.replset")), - ActionType::find)); + Privilege(ResourcePattern::forDatabaseName("config"), clusterMonitorRoleDatabaseActions)); Privilege::addPrivilegeToPrivilegeVector( privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("local.sources")), + Privilege(ResourcePattern::forDatabaseName("local"), clusterMonitorRoleDatabaseActions)); + addReadOnlyDbPrivileges(privileges, "local"); + addReadOnlyDbPrivileges(privileges, "config"); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace(NamespaceString("local", "system.replset")), ActionType::find)); Privilege::addPrivilegeToPrivilegeVector( privileges, @@ -391,22 +394,26 @@ void addClusterManagerPrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyNormalResource(), clusterManagerRoleDatabaseActions)); - addReadOnlyDbPrivileges(privileges, "config"); - ActionSet configSettingsActions; - configSettingsActions << ActionType::insert << ActionType::update << ActionType::remove; Privilege::addPrivilegeToPrivilegeVector( privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("config", "settings")), - configSettingsActions)); + Privilege(ResourcePattern::forDatabaseName("config"), clusterManagerRoleDatabaseActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forDatabaseName("local"), clusterManagerRoleDatabaseActions)); + Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forExactNamespace(NamespaceString("local", "system.replset")), readRoleActions)); + addReadOnlyDbPrivileges(privileges, "config"); + + ActionSet writeActions; + writeActions << ActionType::insert << ActionType::update << ActionType::remove; Privilege::addPrivilegeToPrivilegeVector( - privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("config", "tags")), - configSettingsActions)); + privileges, Privilege(ResourcePattern::forDatabaseName("config"), writeActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("local"), writeActions)); } void addClusterAdminPrivileges(PrivilegeVector* privileges) { @@ -433,6 +440,12 @@ void addBackupPrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forClusterResource(), clusterActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("config"), ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("local"), ActionType::find)); + Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forCollectionName("system.indexes"), ActionType::find)); @@ -503,6 +516,12 @@ void addRestorePrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyResource(), ActionType::listCollections)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("config"), actions)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("local"), actions)); + // Privileges for user/role management Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyNormalResource(), userAdminRoleActions)); @@ -560,6 +579,7 @@ void addRootRolePrivileges(PrivilegeVector* privileges) { addUserAdminAnyDbPrivileges(privileges); addDbAdminAnyDbPrivileges(privileges); addReadWriteAnyDbPrivileges(privileges); + addBackupPrivileges(privileges); addRestorePrivileges(privileges); Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyResource(), ActionType::validate)); diff --git a/src/mongo/db/auth/sasl_plain_server_conversation.cpp b/src/mongo/db/auth/sasl_plain_server_conversation.cpp index b5f0b9e3c8f..d1e993db4a1 100644 --- a/src/mongo/db/auth/sasl_plain_server_conversation.cpp +++ b/src/mongo/db/auth/sasl_plain_server_conversation.cpp @@ -80,18 +80,16 @@ StatusWith SaslPLAINServerConversation::step(StringData inputData, std::st } } else { // Handle schemaVersion28SCRAM (SCRAM only mode) - unsigned char storedKey[scram::hashSize]; - unsigned char serverKey[scram::hashSize]; - - scram::generateSecrets( + std::string decodedSalt = base64::decode(creds.scram.salt); + scram::SCRAMSecrets secrets = scram::generateSecrets(scram::SCRAMPresecrets( authDigest, - reinterpret_cast(base64::decode(creds.scram.salt).c_str()), - 16, - creds.scram.iterationCount, - storedKey, - serverKey); + std::vector(reinterpret_cast(decodedSalt.c_str()), + reinterpret_cast(decodedSalt.c_str()) + + 16), + creds.scram.iterationCount)); if (creds.scram.storedKey != - base64::encode(reinterpret_cast(storedKey), scram::hashSize)) { + base64::encode(reinterpret_cast(secrets->storedKey.data()), + secrets->storedKey.size())) { return StatusWith(ErrorCodes::AuthenticationFailed, mongoutils::str::stream() << "Incorrect user name or password"); } diff --git a/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp b/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp index 9fd8496b7bc..5809306456d 100644 --- a/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp +++ b/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp @@ -35,8 +35,8 @@ #include #include -#include "mongo/crypto/crypto.h" #include "mongo/crypto/mechanism_scram.h" +#include "mongo/crypto/sha1_block.h" #include "mongo/db/auth/sasl_options.h" #include "mongo/platform/random.h" #include "mongo/util/base64.h" @@ -275,60 +275,23 @@ StatusWith SaslSCRAMSHA1ServerConversation::_secondStep(const std::vector< // ClientKey := ClientSignature XOR ClientProof // ServerSignature := HMAC(ServerKey, AuthMessage) - unsigned int hashLen = 0; - unsigned char clientSignature[scram::hashSize]; - - std::string decodedStoredKey = base64::decode(_creds.scram.storedKey); - // ClientSignature := HMAC(StoredKey, AuthMessage) - fassert(18662, - crypto::hmacSha1(reinterpret_cast(decodedStoredKey.c_str()), - scram::hashSize, - reinterpret_cast(_authMessage.c_str()), - _authMessage.size(), - clientSignature, - &hashLen)); - - fassert(18658, hashLen == scram::hashSize); - - try { - clientProof = base64::decode(clientProof); - } catch (const DBException& ex) { - return StatusWith(ex.toStatus()); - } - const unsigned char* decodedClientProof = - reinterpret_cast(clientProof.c_str()); - - // ClientKey := ClientSignature XOR ClientProof - unsigned char clientKey[scram::hashSize]; - for (size_t i = 0; i < scram::hashSize; i++) { - clientKey[i] = clientSignature[i] ^ decodedClientProof[i]; - } - - // StoredKey := H(ClientKey) - unsigned char computedStoredKey[scram::hashSize]; - fassert(18659, crypto::sha1(clientKey, scram::hashSize, computedStoredKey)); - - if (memcmp(decodedStoredKey.c_str(), computedStoredKey, scram::hashSize) != 0) { + if (!scram::verifyClientProof( + base64::decode(clientProof), base64::decode(_creds.scram.storedKey), _authMessage)) { return StatusWith(ErrorCodes::AuthenticationFailed, mongoutils::str::stream() << "SCRAM-SHA-1 authentication failed, storedKey mismatch"); } // ServerSignature := HMAC(ServerKey, AuthMessage) - unsigned char serverSignature[scram::hashSize]; std::string decodedServerKey = base64::decode(_creds.scram.serverKey); - fassert(18660, - crypto::hmacSha1(reinterpret_cast(decodedServerKey.c_str()), - scram::hashSize, - reinterpret_cast(_authMessage.c_str()), - _authMessage.size(), - serverSignature, - &hashLen)); - - fassert(18661, hashLen == scram::hashSize); + SHA1Block serverSignature = + SHA1Block::computeHmac(reinterpret_cast(decodedServerKey.c_str()), + decodedServerKey.size(), + reinterpret_cast(_authMessage.c_str()), + _authMessage.size()); StringBuilder sb; - sb << "v=" << base64::encode(reinterpret_cast(serverSignature), scram::hashSize); + sb << "v=" << serverSignature.toString(); *outputData = sb.str(); return StatusWith(false); diff --git a/src/mongo/db/auth/sasl_scramsha1_test.cpp b/src/mongo/db/auth/sasl_scramsha1_test.cpp new file mode 100644 index 00000000000..66cddabe02e --- /dev/null +++ b/src/mongo/db/auth/sasl_scramsha1_test.cpp @@ -0,0 +1,501 @@ +/* + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/client/native_sasl_client_session.h" +#include "mongo/client/scram_sha1_client_cache.h" +#include "mongo/crypto/mechanism_scram.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" +#include "mongo/db/auth/authz_session_external_state_mock.h" +#include "mongo/db/auth/native_sasl_authentication_session.h" +#include "mongo/db/auth/sasl_scramsha1_server_conversation.h" +#include "mongo/db/service_context_noop.h" +#include "mongo/stdx/memory.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/base64.h" +#include "mongo/util/password_digest.h" + +namespace mongo { + +BSONObj generateSCRAMUserDocument(StringData username, StringData password) { + const size_t scramIterationCount = 10000; + std::string database = "test"; + + std::string digested = createPasswordDigest(username, password); + BSONObj scramCred = scram::generateCredentials(digested, scramIterationCount); + return BSON("_id" << (str::stream() << database << "." << username).operator std::string() + << AuthorizationManager::USER_NAME_FIELD_NAME << username + << AuthorizationManager::USER_DB_FIELD_NAME << database << "credentials" + << BSON("SCRAM-SHA-1" << scramCred) << "roles" << BSONArray() << "privileges" + << BSONArray()); +} + +BSONObj generateMONGODBCRUserDocument(StringData username, StringData password) { + std::string database = "test"; + + std::string digested = createPasswordDigest(username, password); + return BSON("_id" << (str::stream() << database << "." << username).operator std::string() + << AuthorizationManager::USER_NAME_FIELD_NAME << username + << AuthorizationManager::USER_DB_FIELD_NAME << database << "credentials" + << BSON("MONGODB-CR" << digested) << "roles" << BSONArray() << "privileges" + << BSONArray()); +} + +std::string corruptEncodedPayload(const std::string& message, + std::string::const_iterator begin, + std::string::const_iterator end) { + std::string raw = base64::decode( + message.substr(std::distance(message.begin(), begin), std::distance(begin, end))); + if (raw[0] == std::numeric_limits::max()) { + raw[0] -= 1; + } else { + raw[0] += 1; + } + return base64::encode(raw); +} + +class SaslTestState { +public: + enum Participant { kClient, kServer }; + SaslTestState() : SaslTestState(kClient, 0) {} + SaslTestState(Participant participant, size_t stage) : participant(participant), stage(stage) {} + +private: + // Define members here, so that they can be used in declaration of lens(). In C++14, lens() + // can be declared with a return of decltype(auto), without a trailing return type, and these + // members can go at the end of the class. + Participant participant; + size_t stage; + +public: + std::tuple lens() const { + return std::tie(stage, participant); + } + + friend bool operator==(const SaslTestState& lhs, const SaslTestState& rhs) { + return lhs.lens() == rhs.lens(); + } + + friend bool operator<(const SaslTestState& lhs, const SaslTestState& rhs) { + return lhs.lens() < rhs.lens(); + } + + void next() { + if (participant == kClient) { + participant = kServer; + } else { + participant = kClient; + stage++; + } + } + + std::string toString() const { + std::stringstream ss; + if (participant == kClient) { + ss << "Client"; + } else { + ss << "Server"; + } + ss << "Step" << stage; + + return ss.str(); + } +}; + +class SCRAMMutators { +public: + SCRAMMutators() {} + + void setMutator(SaslTestState state, stdx::function fun) { + mutators.insert(std::make_pair(state, fun)); + } + + void execute(SaslTestState state, std::string& str) { + auto it = mutators.find(state); + if (it != mutators.end()) { + it->second(str); + } + } + +private: + std::map> mutators; +}; + +struct SCRAMStepsResult { + SCRAMStepsResult() : outcome(SaslTestState::kClient, 1), status(Status::OK()) {} + SCRAMStepsResult(SaslTestState outcome, Status status) : outcome(outcome), status(status) {} + bool operator==(const SCRAMStepsResult& other) const { + return outcome == other.outcome && status.code() == other.status.code() && + status.reason() == other.status.reason(); + } + SaslTestState outcome; + Status status; + + friend std::ostream& operator<<(std::ostream& os, const SCRAMStepsResult& result) { + return os << "{outcome: " << result.outcome.toString() << ", status: " << result.status + << "}"; + } +}; + +SCRAMStepsResult runSteps(NativeSaslAuthenticationSession* saslServerSession, + NativeSaslClientSession* saslClientSession, + SCRAMMutators interposers = SCRAMMutators{}) { + SCRAMStepsResult result{}; + std::string clientOutput = ""; + std::string serverOutput = ""; + + for (size_t step = 1; step <= 3; step++) { + ASSERT_FALSE(saslClientSession->isDone()); + ASSERT_FALSE(saslServerSession->isDone()); + + // Client step + result.status = saslClientSession->step(serverOutput, &clientOutput); + if (result.status != Status::OK()) { + return result; + } + std::cout << result.outcome.toString() << ": " << clientOutput << std::endl; + interposers.execute(result.outcome, clientOutput); + result.outcome.next(); + + // Server step + result.status = saslServerSession->step(clientOutput, &serverOutput); + if (result.status != Status::OK()) { + return result; + } + interposers.execute(result.outcome, serverOutput); + std::cout << result.outcome.toString() << ": " << serverOutput << std::endl; + result.outcome.next(); + } + ASSERT_TRUE(saslClientSession->isDone()); + ASSERT_TRUE(saslServerSession->isDone()); + + return result; +} + +class SCRAMSHA1Fixture : public mongo::unittest::Test { +protected: + const SCRAMStepsResult goalState = + SCRAMStepsResult(SaslTestState(SaslTestState::kClient, 4), Status::OK()); + + ServiceContextNoop serviceContext; + ServiceContextNoop::UniqueClient client; + ServiceContextNoop::UniqueOperationContext txn; + + AuthzManagerExternalStateMock* authzManagerExternalState; + std::unique_ptr authzManager; + std::unique_ptr authzSession; + + std::unique_ptr saslServerSession; + std::unique_ptr saslClientSession; + + void setUp() { + client = serviceContext.makeClient("test"); + txn = serviceContext.makeOperationContext(client.get()); + + auto uniqueAuthzManagerExternalStateMock = + stdx::make_unique(); + authzManagerExternalState = uniqueAuthzManagerExternalStateMock.get(); + authzManager = + stdx::make_unique(std::move(uniqueAuthzManagerExternalStateMock)); + authzSession = stdx::make_unique( + stdx::make_unique(authzManager.get())); + + saslServerSession = stdx::make_unique(authzSession.get()); + saslServerSession->setOpCtxt(txn.get()); + saslServerSession->start("test", "SCRAM-SHA-1", "mongodb", "MockServer.test", 1, false); + saslClientSession = stdx::make_unique(); + saslClientSession->setParameter(NativeSaslClientSession::parameterMechanism, "SCRAM-SHA-1"); + saslClientSession->setParameter(NativeSaslClientSession::parameterServiceName, "mongodb"); + saslClientSession->setParameter(NativeSaslClientSession::parameterServiceHostname, + "MockServer.test"); + saslClientSession->setParameter(NativeSaslClientSession::parameterServiceHostAndPort, + "MockServer.test:27017"); + } +}; + +/*TEST_F(SCRAMSHA1Fixture, testServerStep1DoesNotIncludeNonceFromClientStep1) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + SCRAMMutators mutator; + mutator.setMutator(SaslTestState(SaslTestState::kServer, 1), [](std::string& serverMessage) { + std::string::iterator nonceBegin = serverMessage.begin() + serverMessage.find("r="); + std::string::iterator nonceEnd = std::find(nonceBegin, serverMessage.end(), ','); + serverMessage = serverMessage.replace(nonceBegin, nonceEnd, "r="); + + }); + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kClient, 2), + Status(ErrorCodes::BadValue, + "Server SCRAM-SHA-1 nonce does not match client nonce: r=")), + runSteps(saslServerSession.get(), saslClientSession.get(), mutator)); +}*/ + +TEST_F(SCRAMSHA1Fixture, testClientStep2DoesNotIncludeNonceFromServerStep1) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + SCRAMMutators mutator; + mutator.setMutator(SaslTestState(SaslTestState::kClient, 2), + [](std::string& clientMessage) { + std::string::iterator nonceBegin = + clientMessage.begin() + clientMessage.find("r="); + std::string::iterator nonceEnd = + std::find(nonceBegin, clientMessage.end(), ','); + clientMessage = clientMessage.replace(nonceBegin, nonceEnd, "r="); + }); + ASSERT_EQ(SCRAMStepsResult( + SaslTestState(SaslTestState::kServer, 2), + Status(ErrorCodes::BadValue, "Incorrect SCRAM-SHA-1 client|server nonce: r=")), + runSteps(saslServerSession.get(), saslClientSession.get(), mutator)); +} + +TEST_F(SCRAMSHA1Fixture, testClientStep2GivesBadProof) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + SCRAMMutators mutator; + mutator.setMutator( + SaslTestState(SaslTestState::kClient, 2), + [](std::string& clientMessage) { + std::string::iterator proofBegin = clientMessage.begin() + clientMessage.find("p=") + 2; + std::string::iterator proofEnd = std::find(proofBegin, clientMessage.end(), ','); + clientMessage = clientMessage.replace( + proofBegin, proofEnd, corruptEncodedPayload(clientMessage, proofBegin, proofEnd)); + + }); + + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kServer, 2), + Status(ErrorCodes::AuthenticationFailed, + "SCRAM-SHA-1 authentication failed, storedKey mismatch")), + runSteps(saslServerSession.get(), saslClientSession.get(), mutator)); +} + +TEST_F(SCRAMSHA1Fixture, testServerStep2GivesBadVerifier) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + std::string encodedVerifier; + SCRAMMutators mutator; + mutator.setMutator( + SaslTestState(SaslTestState::kServer, 2), + [&encodedVerifier](std::string& serverMessage) { + std::string::iterator verifierBegin = + serverMessage.begin() + serverMessage.find("v=") + 2; + std::string::iterator verifierEnd = std::find(verifierBegin, serverMessage.end(), ','); + encodedVerifier = corruptEncodedPayload(serverMessage, verifierBegin, verifierEnd); + + serverMessage = serverMessage.replace(verifierBegin, verifierEnd, encodedVerifier); + + }); + + auto result = runSteps(saslServerSession.get(), saslClientSession.get(), mutator); + + ASSERT_EQ( + SCRAMStepsResult( + SaslTestState(SaslTestState::kClient, 3), + Status(ErrorCodes::BadValue, + str::stream() << "Client failed to verify SCRAM-SHA-1 ServerSignature, received " + << encodedVerifier)), + result); +} + + +TEST_F(SCRAMSHA1Fixture, testSCRAM) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testNULLInPassword) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "saj\0ack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "saj\0ack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + + +TEST_F(SCRAMSHA1Fixture, testCommasInUsernameAndPassword) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("s,a,jack", "s,a,jack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "s,a,jack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("s,a,jack", "s,a,jack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testIncorrectUser) { + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kServer, 1), + Status(ErrorCodes::UserNotFound, "Could not find user sajack@test")), + runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testIncorrectPassword) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "invalidPassword")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kServer, 2), + Status(ErrorCodes::AuthenticationFailed, + "SCRAM-SHA-1 authentication failed, storedKey mismatch")), + runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testMONGODBCR) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateMONGODBCRUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST(SCRAMSHA1Cache, testGetFromEmptyCache) { + SCRAMSHA1ClientCache cache; + std::string saltStr("saltsaltsaltsalt"); + std::vector salt(saltStr.begin(), saltStr.end()); + HostAndPort host("localhost:27017"); + + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000))); +} + + +TEST(SCRAMSHA1Cache, testSetAndGet) { + SCRAMSHA1ClientCache cache; + std::string saltStr("saltsaltsaltsalt"); + std::string badSaltStr("s@lts@lts@lts@lt"); + std::vector salt(saltStr.begin(), saltStr.end()); + std::vector badSalt(badSaltStr.begin(), badSaltStr.end()); + HostAndPort host("localhost:27017"); + + auto secret = scram::generateSecrets(scram::SCRAMPresecrets("aaa", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000), secret); + auto cachedSecret = cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000)); + ASSERT_TRUE(cachedSecret); + ASSERT_TRUE(secret->clientKey == cachedSecret->clientKey); + ASSERT_TRUE(secret->serverKey == cachedSecret->serverKey); + ASSERT_TRUE(secret->storedKey == cachedSecret->storedKey); +} + + +TEST(SCRAMSHA1Cache, testSetAndGetWithDifferentParameters) { + SCRAMSHA1ClientCache cache; + std::string saltStr("saltsaltsaltsalt"); + std::string badSaltStr("s@lts@lts@lts@lt"); + std::vector salt(saltStr.begin(), saltStr.end()); + std::vector badSalt(badSaltStr.begin(), badSaltStr.end()); + HostAndPort host("localhost:27017"); + + auto secret = scram::generateSecrets(scram::SCRAMPresecrets("aaa", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000), secret); + + ASSERT_FALSE(cache.getCachedSecrets(HostAndPort("localhost:27018"), + scram::SCRAMPresecrets("aaa", salt, 10000))); + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aab", salt, 10000))); + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", badSalt, 10000))); + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10001))); +} + + +TEST(SCRAMSHA1Cache, testSetAndReset) { + SCRAMSHA1ClientCache cache; + StringData saltStr("saltsaltsaltsalt"); + std::vector salt(saltStr.begin(), saltStr.end()); + HostAndPort host("localhost:27017"); + + auto secret = scram::generateSecrets(scram::SCRAMPresecrets("aaa", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000), secret); + auto newSecret = scram::generateSecrets(scram::SCRAMPresecrets("aab", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aab", salt, 10000), newSecret); + + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000))); + auto cachedSecret = cache.getCachedSecrets(host, scram::SCRAMPresecrets("aab", salt, 10000)); + ASSERT_TRUE(cachedSecret); + ASSERT_TRUE(newSecret->clientKey == cachedSecret->clientKey); + ASSERT_TRUE(newSecret->serverKey == cachedSecret->serverKey); + ASSERT_TRUE(newSecret->storedKey == cachedSecret->storedKey); +} + +} // namespace mongo diff --git a/src/mongo/db/catalog/apply_ops.cpp b/src/mongo/db/catalog/apply_ops.cpp index 6d00127b9af..0afea74e776 100644 --- a/src/mongo/db/catalog/apply_ops.cpp +++ b/src/mongo/db/catalog/apply_ops.cpp @@ -132,7 +132,7 @@ Status applyOps(OperationContext* txn, try { MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { if (*opType == 'c') { - status = repl::applyCommand_inlock(txn, temp); + status = repl::applyCommand_inlock(txn, temp, true); break; } else { OldClientContext ctx(txn, ns); diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 0f2304a6f49..700d50beb45 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -41,6 +41,99 @@ #include "mongo/db/service_context.h" namespace mongo { + +struct CollModRequest { + const IndexDescriptor* idx = nullptr; + BSONElement indexExpireAfterSeconds = {}; + BSONElement collValidator = {}; + std::string collValidationAction = {}; + std::string collValidationLevel = {}; + BSONElement usePowerOf2Sizes = {}; + BSONElement noPadding = {}; +}; + +StatusWith parseCollModRequest(OperationContext* txn, + const NamespaceString& nss, + Collection* coll, + const BSONObj& cmdObj) { + CollModRequest cmr; + + BSONForEach(e, cmdObj) { + if (str::equals("collMod", e.fieldName())) { + // no-op + } else if (str::startsWith(e.fieldName(), "$")) { + // no-op ignore top-level fields prefixed with $. They are for the command processor + } else if (LiteParsedQuery::cmdOptionMaxTimeMS == e.fieldNameStringData()) { + // no-op + } else if (str::equals("index", e.fieldName())) { + BSONObj indexObj = e.Obj(); + BSONObj keyPattern = indexObj.getObjectField("keyPattern"); + + if (keyPattern.isEmpty()) { + return Status(ErrorCodes::InvalidOptions, "no keyPattern specified"); + } + cmr.indexExpireAfterSeconds = indexObj["expireAfterSeconds"]; + if (cmr.indexExpireAfterSeconds.eoo()) { + return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field"); + } + if (!cmr.indexExpireAfterSeconds.isNumber()) { + return Status(ErrorCodes::InvalidOptions, + "expireAfterSeconds field must be a number"); + } + + const IndexDescriptor* idx = + coll->getIndexCatalog()->findIndexByKeyPattern(txn, keyPattern); + if (idx == NULL) { + return Status(ErrorCodes::IndexNotFound, + str::stream() << "cannot find index " << keyPattern << " for ns " + << nss.ns()); + } + cmr.idx = idx; + + BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); + + if (oldExpireSecs.eoo()) { + return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update"); + } + if (!oldExpireSecs.isNumber()) { + return Status(ErrorCodes::InvalidOptions, + "existing expireAfterSeconds field is not a number"); + } + } else if (str::equals("validator", e.fieldName())) { + auto statusW = coll->parseValidator(e.Obj()); + if (!statusW.isOK()) + return statusW.getStatus(); + + cmr.collValidator = e; + } else if (str::equals("validationLevel", e.fieldName())) { + auto statusW = coll->parseValidationLevel(e.String()); + if (!statusW.isOK()) + return statusW.getStatus(); + + cmr.collValidationLevel = e.String(); + } else if (str::equals("validationAction", e.fieldName())) { + auto statusW = coll->parseValidationAction(e.String()); + if (!statusW.isOK()) + return statusW.getStatus(); + + cmr.collValidationAction = e.String(); + } else { + // As of SERVER-17312 we only support these two options. When SERVER-17320 is + // resolved this will need to be enhanced to handle other options. + const StringData name = e.fieldNameStringData(); + if (name == "usePowerOf2Sizes") + cmr.usePowerOf2Sizes = e; + else if (name == "noPadding") + cmr.noPadding = e; + else + return Status(ErrorCodes::InvalidOptions, + str::stream() << "unknown option to collMod: " << name); + } + } + + return {std::move(cmr)}; +} + Status collMod(OperationContext* txn, const NamespaceString& nss, const BSONObj& cmdObj, @@ -49,7 +142,7 @@ Status collMod(OperationContext* txn, ScopedTransaction transaction(txn, MODE_IX); AutoGetDb autoDb(txn, dbName, MODE_X); Database* const db = autoDb.getDb(); - Collection* coll = db ? db->getCollection(nss) : NULL; + Collection* coll = db ? db->getCollection(nss) : nullptr; // This can kill all cursors so don't allow running it while a background operation is in // progress. @@ -71,122 +164,80 @@ Status collMod(OperationContext* txn, << nss.ns()); } + auto statusW = parseCollModRequest(txn, nss, coll, cmdObj); + if (!statusW.isOK()) { + return statusW.getStatus(); + } + + CollModRequest cmr = statusW.getValue(); + WriteUnitOfWork wunit(txn); - Status errorStatus = Status::OK(); + if (!cmr.indexExpireAfterSeconds.eoo()) { + BSONElement& newExpireSecs = cmr.indexExpireAfterSeconds; + BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); + + if (oldExpireSecs != newExpireSecs) { + result->appendAs(oldExpireSecs, "expireAfterSeconds_old"); + // Change the value of "expireAfterSeconds" on disk. + coll->getCatalogEntry()->updateTTLSetting( + txn, cmr.idx->indexName(), newExpireSecs.safeNumberLong()); + // Notify the index catalog that the definition of this index changed. + cmr.idx = coll->getIndexCatalog()->refreshEntry(txn, cmr.idx); + result->appendAs(newExpireSecs, "expireAfterSeconds_new"); + } + } - BSONForEach(e, cmdObj) { - if (str::equals("collMod", e.fieldName())) { - // no-op - } else if (str::startsWith(e.fieldName(), "$")) { - // no-op ignore top-level fields prefixed with $. They are for the command processor - } else if (LiteParsedQuery::cmdOptionMaxTimeMS == e.fieldNameStringData()) { - // no-op - } else if (str::equals("index", e.fieldName())) { - BSONObj indexObj = e.Obj(); - BSONObj keyPattern = indexObj.getObjectField("keyPattern"); + if (!cmr.collValidator.eoo()) + coll->setValidator(txn, cmr.collValidator.Obj()); - if (keyPattern.isEmpty()) { - errorStatus = Status(ErrorCodes::InvalidOptions, "no keyPattern specified"); - continue; - } + if (!cmr.collValidationAction.empty()) + coll->setValidationAction(txn, cmr.collValidationAction); - BSONElement newExpireSecs = indexObj["expireAfterSeconds"]; - if (newExpireSecs.eoo()) { - errorStatus = Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field"); - continue; - } - if (!newExpireSecs.isNumber()) { - errorStatus = - Status(ErrorCodes::InvalidOptions, "expireAfterSeconds field must be a number"); - continue; - } + if (!cmr.collValidationLevel.empty()) + coll->setValidationLevel(txn, cmr.collValidationLevel); - const IndexDescriptor* idx = - coll->getIndexCatalog()->findIndexByKeyPattern(txn, keyPattern); - if (idx == NULL) { - errorStatus = Status(ErrorCodes::InvalidOptions, - str::stream() << "cannot find index " << keyPattern - << " for ns " << nss.ns()); - continue; - } - BSONElement oldExpireSecs = idx->infoObj().getField("expireAfterSeconds"); - if (oldExpireSecs.eoo()) { - errorStatus = - Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update"); - continue; - } - if (!oldExpireSecs.isNumber()) { - errorStatus = Status(ErrorCodes::InvalidOptions, - "existing expireAfterSeconds field is not a number"); - continue; - } + auto setCollectionOption = [&](BSONElement& COElement) { + typedef CollectionOptions CO; + const StringData name = COElement.fieldNameStringData(); - if (oldExpireSecs != newExpireSecs) { - result->appendAs(oldExpireSecs, "expireAfterSeconds_old"); - // Change the value of "expireAfterSeconds" on disk. - coll->getCatalogEntry()->updateTTLSetting( - txn, idx->indexName(), newExpireSecs.numberLong()); - // Notify the index catalog that the definition of this index changed. - idx = coll->getIndexCatalog()->refreshEntry(txn, idx); - result->appendAs(newExpireSecs, "expireAfterSeconds_new"); - } - } else if (str::equals("validator", e.fieldName())) { - auto status = coll->setValidator(txn, e.Obj()); - if (!status.isOK()) - errorStatus = std::move(status); - } else if (str::equals("validationLevel", e.fieldName())) { - auto status = coll->setValidationLevel(txn, e.String()); - if (!status.isOK()) - errorStatus = std::move(status); - } else if (str::equals("validationAction", e.fieldName())) { - auto status = coll->setValidationAction(txn, e.String()); - if (!status.isOK()) - errorStatus = std::move(status); - } else { - // As of SERVER-17312 we only support these two options. When SERVER-17320 is - // resolved this will need to be enhanced to handle other options. - typedef CollectionOptions CO; - const StringData name = e.fieldNameStringData(); - const int flag = (name == "usePowerOf2Sizes") - ? CO::Flag_UsePowerOf2Sizes - : (name == "noPadding") ? CO::Flag_NoPadding : 0; - if (!flag) { - errorStatus = Status(ErrorCodes::InvalidOptions, - str::stream() << "unknown option to collMod: " << name); - continue; - } + int flag = (name == "usePowerOf2Sizes") ? CO::Flag_UsePowerOf2Sizes + : (name == "noPadding") ? CO::Flag_NoPadding : 0; - CollectionCatalogEntry* cce = coll->getCatalogEntry(); + CollectionCatalogEntry* cce = coll->getCatalogEntry(); - const int oldFlags = cce->getCollectionOptions(txn).flags; - const bool oldSetting = oldFlags & flag; - const bool newSetting = e.trueValue(); + const int oldFlags = cce->getCollectionOptions(txn).flags; + const bool oldSetting = oldFlags & flag; + const bool newSetting = COElement.trueValue(); - result->appendBool(name.toString() + "_old", oldSetting); - result->appendBool(name.toString() + "_new", newSetting); + result->appendBool(name.toString() + "_old", oldSetting); + result->appendBool(name.toString() + "_new", newSetting); - const int newFlags = newSetting ? (oldFlags | flag) // set flag - : (oldFlags & ~flag); // clear flag + const int newFlags = newSetting ? (oldFlags | flag) // set flag + : (oldFlags & ~flag); // clear flag - // NOTE we do this unconditionally to ensure that we note that the user has - // explicitly set flags, even if they are just setting the default. - cce->updateFlags(txn, newFlags); + // NOTE we do this unconditionally to ensure that we note that the user has + // explicitly set flags, even if they are just setting the default. + cce->updateFlags(txn, newFlags); - const CollectionOptions newOptions = cce->getCollectionOptions(txn); - invariant(newOptions.flags == newFlags); - invariant(newOptions.flagsSet); - } + const CollectionOptions newOptions = cce->getCollectionOptions(txn); + invariant(newOptions.flags == newFlags); + invariant(newOptions.flagsSet); + }; + + if (!cmr.usePowerOf2Sizes.eoo()) { + setCollectionOption(cmr.usePowerOf2Sizes); } - if (!errorStatus.isOK()) { - return errorStatus; + if (!cmr.noPadding.eoo()) { + setCollectionOption(cmr.noPadding); } getGlobalServiceContext()->getOpObserver()->onCollMod( txn, (dbName.toString() + ".$cmd").c_str(), cmdObj); wunit.commit(); + return Status::OK(); } } // namespace mongo diff --git a/src/mongo/db/catalog/coll_mod.h b/src/mongo/db/catalog/coll_mod.h index eb8644b74d1..bb86bbc50d1 100644 --- a/src/mongo/db/catalog/coll_mod.h +++ b/src/mongo/db/catalog/coll_mod.h @@ -27,13 +27,22 @@ */ #include "mongo/base/status.h" +#include "mongo/base/status_with.h" namespace mongo { class BSONObj; class BSONObjBuilder; +class Collection; class NamespaceString; class OperationContext; +struct CollModRequest; + +StatusWith parseCollModRequest(OperationContext* txn, + const NamespaceString& nss, + Collection* coll, + const BSONObj& cmdObj); + /** * Performs the collection modification described in "cmdObj" on the collection "ns". */ diff --git a/src/mongo/db/catalog/collection.cpp b/src/mongo/db/catalog/collection.cpp index 0043b20a4c5..04e94eab47d 100644 --- a/src/mongo/db/catalog/collection.cpp +++ b/src/mongo/db/catalog/collection.cpp @@ -180,9 +180,9 @@ Collection::Collection(OperationContext* txn, _validatorDoc(_details->getCollectionOptions(txn).validator.getOwned()), _validator(uassertStatusOK(parseValidator(_validatorDoc))), _validationAction(uassertStatusOK( - _parseValidationAction(_details->getCollectionOptions(txn).validationAction))), + parseValidationAction(_details->getCollectionOptions(txn).validationAction))), _validationLevel(uassertStatusOK( - _parseValidationLevel(_details->getCollectionOptions(txn).validationLevel))), + parseValidationLevel(_details->getCollectionOptions(txn).validationLevel))), _cursorManager(fullNS), _cappedNotifier(_recordStore->isCapped() ? new CappedInsertNotifier() : nullptr), _mustTakeCappedLockOnInsert(isCapped() && !_ns.isSystemDotProfile() && !_ns.isOplog()) { @@ -823,7 +823,7 @@ Status Collection::setValidator(OperationContext* txn, BSONObj validatorDoc) { return Status::OK(); } -StatusWith Collection::_parseValidationLevel(StringData newLevel) { +StatusWith Collection::parseValidationLevel(StringData newLevel) { if (newLevel == "") { // default return STRICT_V; @@ -839,7 +839,7 @@ StatusWith Collection::_parseValidationLevel(String } } -StatusWith Collection::_parseValidationAction(StringData newAction) { +StatusWith Collection::parseValidationAction(StringData newAction) { if (newAction == "") { // default return ERROR_V; @@ -878,7 +878,7 @@ StringData Collection::getValidationAction() const { Status Collection::setValidationLevel(OperationContext* txn, StringData newLevel) { invariant(txn->lockState()->isCollectionLockedForMode(ns().toString(), MODE_X)); - StatusWith status = _parseValidationLevel(newLevel); + StatusWith status = parseValidationLevel(newLevel); if (!status.isOK()) { return status.getStatus(); } @@ -893,7 +893,7 @@ Status Collection::setValidationLevel(OperationContext* txn, StringData newLevel Status Collection::setValidationAction(OperationContext* txn, StringData newAction) { invariant(txn->lockState()->isCollectionLockedForMode(ns().toString(), MODE_X)); - StatusWith status = _parseValidationAction(newAction); + StatusWith status = parseValidationAction(newAction); if (!status.isOK()) { return status.getStatus(); } @@ -1047,4 +1047,8 @@ Status Collection::touch(OperationContext* txn, return Status::OK(); } + +UpdateNotifier* Collection::getUpdateNotifier() { + return this; +} } diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 6823045b952..8f516f72bbd 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -38,6 +38,7 @@ #include "mongo/base/status_with.h" #include "mongo/base/string_data.h" #include "mongo/bson/mutable/damage_vector.h" +#include "mongo/db/catalog/coll_mod.h" #include "mongo/db/catalog/collection_info_cache.h" #include "mongo/db/catalog/cursor_manager.h" #include "mongo/db/catalog/index_catalog.h" @@ -362,6 +363,16 @@ public: */ void temp_cappedTruncateAfter(OperationContext* txn, RecordId end, bool inclusive); + enum ValidationAction { WARN, ERROR_V }; + enum ValidationLevel { OFF, MODERATE, STRICT_V }; + + /** + * Returns a non-ok Status if validator is not legal for this collection. + */ + StatusWithMatchExpression parseValidator(const BSONObj& validator) const; + + static StatusWith parseValidationLevel(StringData); + static StatusWith parseValidationAction(StringData); /** * Sets the validator for this collection. * @@ -422,16 +433,17 @@ public: */ void notifyCappedWaitersIfNeeded(); -private: /** - * Returns a non-ok Status if document does not pass this collection's validator. + * This function is necessary for a 3.2 backport. We have a better fix for the + * underlying issue in later versions. */ - Status checkValidation(OperationContext* txn, const BSONObj& document) const; + UpdateNotifier* getUpdateNotifier(); +private: /** - * Returns a non-ok Status if validator is not legal for this collection. + * Returns a non-ok Status if document does not pass this collection's validator. */ - StatusWithMatchExpression parseValidator(const BSONObj& validator) const; + Status checkValidation(OperationContext* txn, const BSONObj& document) const; Status recordStoreGoingToMove(OperationContext* txn, const RecordId& oldLocation, @@ -470,11 +482,9 @@ private: BSONObj _validatorDoc; // Points into _validatorDoc. Null means no filter. std::unique_ptr _validator; - enum ValidationAction { WARN, ERROR_V } _validationAction; - enum ValidationLevel { OFF, MODERATE, STRICT_V } _validationLevel; - static StatusWith _parseValidationLevel(StringData); - static StatusWith _parseValidationAction(StringData); + ValidationAction _validationAction; + ValidationLevel _validationLevel; // this is mutable because read only users of the Collection class // use it keep state. This seems valid as const correctness of Collection diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp index a54a5e4798c..13b4b7faa01 100644 --- a/src/mongo/db/catalog/database.cpp +++ b/src/mongo/db/catalog/database.cpp @@ -40,16 +40,14 @@ #include "mongo/db/audit.h" #include "mongo/db/auth/auth_index_d.h" #include "mongo/db/background.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog_entry.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/catalog/database_catalog_entry.h" #include "mongo/db/catalog/database_holder.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbhelpers.h" -#include "mongo/db/service_context.h" -#include "mongo/db/service_context_d.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/instance.h" #include "mongo/db/introspect.h" @@ -57,10 +55,12 @@ #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" +#include "mongo/db/service_context_d.h" #include "mongo/db/stats/top.h" -#include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/storage_engine.h" #include "mongo/db/storage/recovery_unit.h" +#include "mongo/db/storage/storage_engine.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/util/log.h" namespace mongo { @@ -215,35 +215,6 @@ Database::Database(OperationContext* txn, StringData name, DatabaseCatalogEntry* } } - -/*static*/ -string Database::duplicateUncasedName(const string& name, set* duplicates) { - if (duplicates) { - duplicates->clear(); - } - - set allShortNames; - dbHolder().getAllShortNames(allShortNames); - - for (const auto& dbname : allShortNames) { - if (strcasecmp(dbname.c_str(), name.c_str())) - continue; - - if (strcmp(dbname.c_str(), name.c_str()) == 0) - continue; - - if (duplicates) { - duplicates->insert(dbname); - } else { - return dbname; - } - } - if (duplicates) { - return duplicates->empty() ? "" : *duplicates->begin(); - } - return ""; -} - void Database::clearTmpCollections(OperationContext* txn) { invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); @@ -344,14 +315,8 @@ void Database::getStats(OperationContext* opCtx, BSONObjBuilder* output, double } Status Database::dropCollection(OperationContext* txn, StringData fullns) { - invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); - - LOG(1) << "dropCollection: " << fullns << endl; - massertNamespaceNotIndex(fullns, "dropCollection"); - - Collection* collection = getCollection(fullns); - if (!collection) { - // collection doesn't exist + if (!getCollection(fullns)) { + // Collection doesn't exist so don't bother validating if it can be dropped. return Status::OK(); } @@ -370,9 +335,24 @@ Status Database::dropCollection(OperationContext* txn, StringData fullns) { } } + return dropCollectionEvenIfSystem(txn, nss); +} + +Status Database::dropCollectionEvenIfSystem(OperationContext* txn, const NamespaceString& fullns) { + invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); + + LOG(1) << "dropCollection: " << fullns; + + Collection* collection = getCollection(fullns); + if (!collection) { + return Status::OK(); // Post condition already met. + } + + massertNamespaceNotIndex(fullns.toString(), "dropCollection"); + BackgroundOperation::assertNoBgOpInProgForNs(fullns); - audit::logDropCollection(&cc(), fullns); + audit::logDropCollection(&cc(), fullns.toString()); Status s = collection->getIndexCatalog()->dropAllIndexes(txn, true); if (!s.isOK()) { @@ -384,13 +364,13 @@ Status Database::dropCollection(OperationContext* txn, StringData fullns) { verify(collection->_details->getTotalIndexCount(txn) == 0); LOG(1) << "\t dropIndexes done" << endl; - Top::get(txn->getClient()->getServiceContext()).collectionDropped(fullns); + Top::get(txn->getClient()->getServiceContext()).collectionDropped(fullns.toString()); // We want to destroy the Collection object before telling the StorageEngine to destroy the // RecordStore. - _clearCollectionCache(txn, fullns, "collection dropped"); + _clearCollectionCache(txn, fullns.toString(), "collection dropped"); - s = _dbEntry->dropCollection(txn, fullns); + s = _dbEntry->dropCollection(txn, fullns.toString()); if (!s.isOK()) return s; @@ -407,7 +387,7 @@ Status Database::dropCollection(OperationContext* txn, StringData fullns) { } } - getGlobalServiceContext()->getOpObserver()->onDropCollection(txn, nss); + getGlobalServiceContext()->getOpObserver()->onDropCollection(txn, fullns); return Status::OK(); } diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h index 53c3cb5712d..77b6ccc865b 100644 --- a/src/mongo/db/catalog/database.h +++ b/src/mongo/db/catalog/database.h @@ -144,7 +144,12 @@ public: const DatabaseCatalogEntry* getDatabaseCatalogEntry() const; + /** + * dropCollection() will refuse to drop system collections. Use dropCollectionEvenIfSystem() if + * that is required. + */ Status dropCollection(OperationContext* txn, StringData fullns); + Status dropCollectionEvenIfSystem(OperationContext* txn, const NamespaceString& fullns); Collection* createCollection(OperationContext* txn, StringData ns, @@ -167,15 +172,6 @@ public: StringData toNS, bool stayTemp); - /** - * @return name of an existing database with same text name but different - * casing, if one exists. Otherwise the empty std::string is returned. If - * 'duplicates' is specified, it is filled with all duplicate names. - // TODO move??? - */ - static std::string duplicateUncasedName(const std::string& name, - std::set* duplicates = 0); - static Status validateDBName(StringData dbname); const std::string& getSystemIndexesName() const { diff --git a/src/mongo/db/catalog/database_holder.cpp b/src/mongo/db/catalog/database_holder.cpp index f4a2cf62970..b6633f5c328 100644 --- a/src/mongo/db/catalog/database_holder.cpp +++ b/src/mongo/db/catalog/database_holder.cpp @@ -35,14 +35,15 @@ #include "mongo/db/audit.h" #include "mongo/db/auth/auth_index_d.h" #include "mongo/db/background.h" -#include "mongo/db/client.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_catalog_entry.h" -#include "mongo/db/service_context.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/operation_context.h" +#include "mongo/db/service_context.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/util/log.h" +#include "mongo/util/scopeguard.h" namespace mongo { @@ -91,56 +92,81 @@ Database* DatabaseHolder::get(OperationContext* txn, StringData ns) const { return NULL; } +std::set DatabaseHolder::_getNamesWithConflictingCasing_inlock(StringData name) { + std::set duplicates; + + for (const auto& nameAndPointer : _dbs) { + // A name that's equal with case-insensitive match must be identical, or it's a duplicate. + if (name.equalCaseInsensitive(nameAndPointer.first) && name != nameAndPointer.first) + duplicates.insert(nameAndPointer.first); + } + return duplicates; +} + +std::set DatabaseHolder::getNamesWithConflictingCasing(StringData name) { + stdx::lock_guard lk(_m); + return _getNamesWithConflictingCasing_inlock(name); +} + Database* DatabaseHolder::openDb(OperationContext* txn, StringData ns, bool* justCreated) { const StringData dbname = _todb(ns); invariant(txn->lockState()->isDbLockedForMode(dbname, MODE_X)); - Database* db = get(txn, ns); - if (db) { - if (justCreated) { - *justCreated = false; - } + if (justCreated) + *justCreated = false; // Until proven otherwise. - return db; - } + stdx::unique_lock lk(_m); - // Check casing - const string duplicate = Database::duplicateUncasedName(dbname.toString()); - if (!duplicate.empty()) { - stringstream ss; - ss << "db already exists with different case already have: [" << duplicate - << "] trying to create [" << dbname.toString() << "]"; - uasserted(ErrorCodes::DatabaseDifferCase, ss.str()); - } + // The following will insert a nullptr for dbname, which will treated the same as a non- + // existant database by the get method, yet still counts in getNamesWithConflictingCasing. + if (auto db = _dbs[dbname]) + return db; + // We've inserted a nullptr entry for dbname: make sure to remove it on unsuccessful exit. + auto removeDbGuard = MakeGuard([this, &lk, dbname] { + if (!lk.owns_lock()) + lk.lock(); + _dbs.erase(dbname); + }); + + // Check casing in lock to avoid transient duplicates. + auto duplicates = _getNamesWithConflictingCasing_inlock(dbname); + uassert(ErrorCodes::DatabaseDifferCase, + str::stream() << "db already exists with different case already have: [" + << *duplicates.cbegin() << "] trying to create [" << dbname.toString() + << "]", + duplicates.empty()); + + + // Do the catalog lookup and database creation outside of the scoped lock, because these may + // block. Only one thread can be inside this method for the same DB name, because of the + // requirement for X-lock on the database when we enter. So there is no way we can insert two + // different databases for the same name. + lk.unlock(); StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine(); - invariant(storageEngine); - DatabaseCatalogEntry* entry = storageEngine->getDatabaseCatalogEntry(txn, dbname); - invariant(entry); - const bool exists = entry->exists(); - if (!exists) { - audit::logCreateDatabase(&cc(), dbname); - } - if (justCreated) { - *justCreated = !exists; + if (!entry->exists()) { + audit::logCreateDatabase(&cc(), dbname); + if (justCreated) + *justCreated = true; } - // Do this outside of the scoped lock, because database creation does transactional - // operations which may block. Only one thread can be inside this method for the same DB - // name, because of the requirement for X-lock on the database when we enter. So there is - // no way we can insert two different databases for the same name. - db = new Database(txn, dbname, entry); + auto newDb = stdx::make_unique(txn, dbname, entry); - stdx::lock_guard lk(_m); - _dbs[dbname] = db; + // Finally replace our nullptr entry with the new Database pointer. + removeDbGuard.Dismiss(); + lk.lock(); + auto it = _dbs.find(dbname); + invariant(it != _dbs.end() && it->second == nullptr); + Database* newDbPointer = newDb.release(); + _dbs[dbname] = newDbPointer; + invariant(_getNamesWithConflictingCasing_inlock(dbname.toString()).empty()); - return db; + return newDbPointer; } void DatabaseHolder::close(OperationContext* txn, StringData ns) { - // TODO: This should be fine if only a DB X-lock invariant(txn->lockState()->isW()); const StringData dbName = _todb(ns); diff --git a/src/mongo/db/catalog/database_holder.h b/src/mongo/db/catalog/database_holder.h index ce99747937b..91850197771 100644 --- a/src/mongo/db/catalog/database_holder.h +++ b/src/mongo/db/catalog/database_holder.h @@ -29,6 +29,7 @@ #pragma once #include +#include #include "mongo/base/string_data.h" #include "mongo/db/namespace_string.h" @@ -87,9 +88,15 @@ public: } } + /** + * Returns the set of existing database names that differ only in casing. + */ + std::set getNamesWithConflictingCasing(StringData name); + private: - typedef StringMap DBs; + std::set _getNamesWithConflictingCasing_inlock(StringData name); + typedef StringMap DBs; mutable SimpleMutex _m; DBs _dbs; }; diff --git a/src/mongo/db/catalog/index_catalog.cpp b/src/mongo/db/catalog/index_catalog.cpp index b572a66426e..be51e69ef83 100644 --- a/src/mongo/db/catalog/index_catalog.cpp +++ b/src/mongo/db/catalog/index_catalog.cpp @@ -133,6 +133,17 @@ IndexCatalogEntry* IndexCatalog::_setupInMemoryStructures(OperationContext* txn, Status status = _isSpecOk(descriptor->infoObj()); if (!status.isOK() && status != ErrorCodes::IndexAlreadyExists) { + if (_collection->ns().ns() == "admin.system.version") { + auto indexNameElem = descriptor->infoObj()["name"]; + + if (indexNameElem.type() == BSONType::String && + indexNameElem.valueStringData() == StringData("incompatible_with_version_32")) { + severe() + << "Cannot start mongod when the featureCompatibilityVersion is higher than " + "3.2. See http://dochub.mongodb.org/core/3.4-feature-compatibility."; + fassertFailedNoTrace(40352); + } + } severe() << "Found an invalid index " << descriptor->infoObj() << " on the " << _collection->ns().ns() << " collection: " << status.reason(); fassertFailedNoTrace(28782); @@ -393,6 +404,8 @@ void IndexCatalog::IndexBuildBlock::fail() { void IndexCatalog::IndexBuildBlock::success() { Collection* collection = _catalog->_collection; fassert(17207, collection->ok()); + NamespaceString ns(_indexNamespace); + invariant(_txn->lockState()->isDbLockedForMode(ns.db(), MODE_X)); collection->getCatalogEntry()->indexBuildSuccess(_txn, _indexName); @@ -402,6 +415,8 @@ void IndexCatalog::IndexBuildBlock::success() { fassert(17331, entry && entry == _entry); OperationContext* txn = _txn; + LOG(2) << "marking index " << _indexName << " as ready in snapshot id " + << txn->recoveryUnit()->getSnapshotId(); _txn->recoveryUnit()->onCommit([txn, entry, collection] { // Note: this runs after the WUOW commits but before we release our X lock on the // collection. This means that any snapshot created after this must include the full index, @@ -842,13 +857,12 @@ Status IndexCatalog::_dropIndex(OperationContext* txn, IndexCatalogEntry* entry) invariant(_entries.release(entry->descriptor()) == entry); txn->recoveryUnit()->registerChange(new IndexRemoveChange(txn, _collection, &_entries, entry)); + _collection->infoCache()->droppedIndex(txn, indexName); entry = NULL; _deleteIndexFromDisk(txn, indexName, indexNamespace); _checkMagic(); - _collection->infoCache()->droppedIndex(txn, indexName); - return Status::OK(); } @@ -1154,9 +1168,12 @@ Status IndexCatalog::_unindexRecord(OperationContext* txn, options.logIfError = logIfError; options.dupsAllowed = isDupsAllowed(index->descriptor()); - // For unindex operations, dupsAllowed=false really means that it is safe to delete anything - // that matches the key, without checking the RecordID, since dups are impossible. We need - // to disable this behavior for in-progress indexes. See SERVER-17487 for more details. + // On WiredTiger, we do blind unindexing of records for efficiency. However, when duplicates + // are allowed in unique indexes, WiredTiger does not do blind unindexing, and instead confirms + // that the recordid matches the element we are removing. + // We need to disable blind-deletes for in-progress indexes, in order to force recordid-matching + // for unindex operations, since initial sync can build an index over a collection with + // duplicates. See SERVER-17487 for more details. options.dupsAllowed = options.dupsAllowed || !index->isReady(txn); int64_t removed; diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp index b18e90ac880..a5cbd7a920b 100644 --- a/src/mongo/db/catalog/index_create.cpp +++ b/src/mongo/db/catalog/index_create.cpp @@ -47,6 +47,7 @@ #include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/operation_context.h" +#include "mongo/db/server_parameters.h" #include "mongo/stdx/mutex.h" #include "mongo/util/fail_point.h" #include "mongo/util/fail_point_service.h" @@ -63,6 +64,30 @@ using std::endl; MONGO_FP_DECLARE(crashAfterStartingIndexBuild); +std::atomic maxIndexBuildMemoryUsageMegabytes(500); // NOLINT + +class ExportedMaxIndexBuildMemoryUsageParameter + : public ExportedServerParameter { +public: + ExportedMaxIndexBuildMemoryUsageParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "maxIndexBuildMemoryUsageMegabytes", + &maxIndexBuildMemoryUsageMegabytes) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 100) { + return Status( + ErrorCodes::BadValue, + "maxIndexBuildMemoryUsageMegabytes must be greater than or equal to 100 MB"); + } + + return Status::OK(); + } + +} exportedMaxIndexBuildMemoryUsageParameter; + + /** * On rollback sets MultiIndexBlock::_needToCleanup to true. */ @@ -169,6 +194,14 @@ Status MultiIndexBlock::init(const std::vector& indexSpecs) { _buildInBackground = (_buildInBackground && info["background"].trueValue()); } + std::vector indexInfoObjs; + indexInfoObjs.reserve(indexSpecs.size()); + std::size_t eachIndexBuildMaxMemoryUsageBytes = 0; + if (!indexSpecs.empty()) { + eachIndexBuildMaxMemoryUsageBytes = + std::size_t(maxIndexBuildMemoryUsageMegabytes) * 1024 * 1024 / indexSpecs.size(); + } + for (size_t i = 0; i < indexSpecs.size(); i++) { BSONObj info = indexSpecs[i]; StatusWith statusWithInfo = @@ -192,7 +225,7 @@ Status MultiIndexBlock::init(const std::vector& indexSpecs) { if (!_buildInBackground) { // Bulk build process requires foreground building as it assumes nothing is changing // under it. - index.bulk = index.real->initiateBulk(); + index.bulk = index.real->initiateBulk(eachIndexBuildMaxMemoryUsageBytes); } const IndexDescriptor* descriptor = index.block->getEntry()->descriptor(); @@ -203,7 +236,8 @@ Status MultiIndexBlock::init(const std::vector& indexSpecs) { log() << "build index on: " << ns << " properties: " << descriptor->toString(); if (index.bulk) - log() << "\t building index using bulk method"; + log() << "\t building index using bulk method; build may temporarily use up to " + << eachIndexBuildMaxMemoryUsageBytes / 1024 / 1024 << " megabytes of RAM"; index.filterExpression = index.block->getEntry()->getFilterExpression(); diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp index 6607f49e98f..d8e8cfe0c85 100644 --- a/src/mongo/db/catalog/rename_collection.cpp +++ b/src/mongo/db/catalog/rename_collection.cpp @@ -41,6 +41,8 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_create.h" #include "mongo/db/client.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/index_builder.h" @@ -115,7 +117,7 @@ Status renameCollection(OperationContext* txn, Database* const targetDB = dbHolder().openDb(txn, target.db()); - { + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { WriteUnitOfWork wunit(txn); // Check if the target namespace exists and if dropTarget is true. @@ -149,6 +151,7 @@ Status renameCollection(OperationContext* txn, wunit.commit(); } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "renameCollection", target.ns()); // If we get here, we are renaming across databases, so we must copy all the data and // indexes, then remove the source collection. diff --git a/src/mongo/db/commands/apply_ops.cpp b/src/mongo/db/commands/apply_ops.cpp index f4f0b930a8c..77e89db3aca 100644 --- a/src/mongo/db/commands/apply_ops.cpp +++ b/src/mongo/db/commands/apply_ops.cpp @@ -121,22 +121,31 @@ public: txn->setWriteConcern(wcResult.getValue()); setupSynchronousCommit(txn); + auto applyOpsRes = [&]() { + try { + // Note: this scope guard must go out of scope before the waitForWriteConcern below! + auto client = txn->getClient(); + auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); + ScopeGuard lastOpSetterGuard = + MakeObjGuard(repl::ReplClientInfo::forClient(client), + &repl::ReplClientInfo::setLastOpToSystemLastOpTime, + txn); + + auto res = appendCommandStatus(result, applyOps(txn, dbname, cmdObj, &result)); + + if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { + // If this operation has already generated a new lastOp, don't bother setting it + // here. No-op applyOps will not generate a new lastOp, so we still need the + // guard to fire in that case. + lastOpSetterGuard.Dismiss(); + } - auto client = txn->getClient(); - auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); - ScopeGuard lastOpSetterGuard = - MakeObjGuard(repl::ReplClientInfo::forClient(client), - &repl::ReplClientInfo::setLastOpToSystemLastOpTime, - txn); - - auto applyOpsStatus = appendCommandStatus(result, applyOps(txn, dbname, cmdObj, &result)); - - if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { - // If this operation has already generated a new lastOp, don't bother setting it - // here. No-op applyOps will not generate a new lastOp, so we still need the guard to - // fire in that case. - lastOpSetterGuard.Dismiss(); - } + return res; + } catch (const DBException& e) { + appendCommandStatus(result, e.toStatus()); + return false; + } + }(); WriteConcernResult res; auto waitForWCStatus = @@ -146,7 +155,7 @@ public: &res); appendCommandWCStatus(result, waitForWCStatus); - return applyOpsStatus; + return applyOpsRes; } private: diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index ddd40f0ed09..4e33d11ef2f 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -41,17 +41,17 @@ #include "mongo/db/commands.h" #include "mongo/db/db_raii.h" #include "mongo/db/exec/working_set_common.h" -#include "mongo/db/service_context.h" #include "mongo/db/matcher/extensions_callback_real.h" -#include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/query/explain.h" #include "mongo/db/query/find.h" #include "mongo/db/query/find_common.h" #include "mongo/db/query/get_executor.h" +#include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/s/operation_shard_version.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" #include "mongo/db/stats/counters.h" #include "mongo/s/stale_exception.h" #include "mongo/util/log.h" @@ -243,37 +243,34 @@ public: ShardingState* const shardingState = ShardingState::get(txn); - if (OperationShardVersion::get(txn).hasShardVersion() && shardingState->enabled()) { - ChunkVersion receivedVersion = OperationShardVersion::get(txn).getShardVersion(nss); - ChunkVersion latestVersion; - // Wait for migration completion to get the correct chunk version. + // Parse, canonicalize, plan, transcribe, and get a plan executor. + boost::optional optionalCtx; + try { + optionalCtx.emplace(txn, nss); + } catch (const StaleConfigException& sce) { + // Wait for migration completion to get the correct chunk version const int maxTimeoutSec = 30; int timeoutSec = cq->getParsed().getMaxTimeMS() / 1000; if (!timeoutSec || timeoutSec > maxTimeoutSec) { timeoutSec = maxTimeoutSec; } - if (!shardingState->waitTillNotInCriticalSection(timeoutSec)) { - uasserted(ErrorCodes::LockTimeout, "Timeout while waiting for migration commit"); + if (shardingState->waitTillNotInCriticalSection(maxTimeoutSec)) { + ChunkVersion unused; + shardingState->refreshMetadataIfNeeded( + txn, nss.ns(), sce.getVersionReceived(), &unused); } - - // If the received version is newer than the version cached in 'shardingState', then we - // have to refresh 'shardingState' from the config servers. We do this before acquiring - // locks so that we don't hold locks while waiting on the network. - uassertStatusOK(shardingState->refreshMetadataIfNeeded( - txn, nss.ns(), receivedVersion, &latestVersion)); + throw; } - // Acquire locks. - AutoGetCollectionForRead ctx(txn, nss); + const auto& ctx = *optionalCtx; Collection* collection = ctx.getCollection(); const int dbProfilingLevel = ctx.getDb() ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile; - // It is possible that the sharding version will change during yield while we are - // retrieving a plan executor. If this happens we will throw an error and mongos will - // retry. + // It is possible that the sharding version will change during yield while we are retrieving + // a plan executor. If this happens we will throw an error and mongos will retry. const ChunkVersion shardingVersionAtStart = shardingState->getVersion(nss.ns()); // Get the execution plan for the query. diff --git a/src/mongo/db/commands/generic.cpp b/src/mongo/db/commands/generic.cpp index 47f47442452..bc8963273da 100644 --- a/src/mongo/db/commands/generic.cpp +++ b/src/mongo/db/commands/generic.cpp @@ -32,6 +32,7 @@ #include +#include "mongo/bson/util/bson_extract.h" #include "mongo/bson/util/builder.h" #include "mongo/client/dbclient_rs.h" #include "mongo/db/auth/action_set.h" @@ -448,6 +449,61 @@ public: } getLogCmd; +class ClearLogCmd : public Command { +public: + ClearLogCmd() : Command("clearLog") {} + + virtual bool slaveOk() const { + return true; + } + virtual bool adminOnly() const { + return true; + } + virtual bool isWriteCommandForConfigServer() const { + return false; + } + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) override { + // No access control needed since this command is a testing-only command that must be + // enabled at the command line. + return Status::OK(); + } + virtual void help(stringstream& help) const { + help << "{ clearLog : 'global' }"; + } + + virtual bool run(OperationContext* opCtx, + const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result) { + std::string logName; + Status status = bsonExtractStringField(cmdObj, "clearLog", &logName); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (logName != "global") { + return appendCommandStatus( + result, Status(ErrorCodes::InvalidOptions, "Only the 'global' log can be cleared")); + } + RamLog* ramlog = RamLog::getIfExists(logName); + invariant(ramlog); + ramlog->clear(); + return true; + } +}; + +MONGO_INITIALIZER(RegisterClearLogCmd)(InitializerContext* context) { + if (Command::testCommandsEnabled) { + // Leaked intentionally: a Command registers itself when constructed. + new ClearLogCmd(); + } + return Status::OK(); +} + class CmdGetCmdLineOpts : Command { public: CmdGetCmdLineOpts() : Command("getCmdLineOpts") {} diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index bb545b2ebff..5aca8c8ad1b 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -301,7 +301,7 @@ public: auto planSummary = Explain::getPlanSummary(exec); { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); CurOp::get(txn)->setPlanSummary_inlock(planSummary); } diff --git a/src/mongo/db/commands/list_databases.cpp b/src/mongo/db/commands/list_databases.cpp index 93d07479b8f..ee049091efc 100644 --- a/src/mongo/db/commands/list_databases.cpp +++ b/src/mongo/db/commands/list_databases.cpp @@ -37,6 +37,9 @@ #include "mongo/db/storage/storage_engine.h" namespace mongo { +namespace { +static const StringData kNameOnlyField{"nameOnly"}; +} // namespace using std::set; using std::string; @@ -61,7 +64,8 @@ public: return false; } virtual void help(stringstream& help) const { - help << "list databases on this server"; + help << "{ listDatabases:1, [filter: ] [, nameOnly: true ] }\n" + "list databases on this server"; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, @@ -79,6 +83,8 @@ public: int, string& errmsg, BSONObjBuilder& result) { + bool nameOnly = jsobj[kNameOnlyField].trueValue(); + vector dbNames; StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine(); storageEngine->listDatabases(&dbNames); @@ -93,7 +99,7 @@ public: BSONObjBuilder b; b.append("name", dbname); - { + if (!nameOnly) { ScopedTransaction transaction(txn, MODE_IS); Lock::DBLock dbLock(txn->lockState(), dbname, MODE_IS); @@ -117,7 +123,9 @@ public: } result.append("databases", dbInfos); - result.append("totalSize", double(totalSize)); + if (!nameOnly) { + result.append("totalSize", double(totalSize)); + } return true; } } cmdListDatabases; diff --git a/src/mongo/db/concurrency/lock_manager.cpp b/src/mongo/db/concurrency/lock_manager.cpp index 6d2450086fe..b8ab480e6a2 100644 --- a/src/mongo/db/concurrency/lock_manager.cpp +++ b/src/mongo/db/concurrency/lock_manager.cpp @@ -754,6 +754,7 @@ void LockManager::_onLockModeChanged(LockHead* lock, bool checkConflictQueue) { LockRequest* iterNext = NULL; + bool newlyCompatibleFirst = false; // Set on enabling compatibleFirst mode. for (LockRequest* iter = lock->conflictList._front; (iter != NULL) && checkConflictQueue; iter = iterNext) { invariant(iter->status == LockRequest::STATUS_WAITING); @@ -765,11 +766,11 @@ void LockManager::_onLockModeChanged(LockHead* lock, bool checkConflictQueue) { if (conflicts(iter->mode, lock->grantedModes)) { // If iter doesn't have a previous pointer, this means that it is at the front of the // queue. If we continue scanning the queue beyond this point, we will starve it by - // granting more and more requests. - if (!iter->prev) { + // granting more and more requests. However, if we newly transition to compatibleFirst + // mode, grant any waiting compatible requests. + if (!iter->prev && !newlyCompatibleFirst) { break; } - continue; } @@ -782,13 +783,13 @@ void LockManager::_onLockModeChanged(LockHead* lock, bool checkConflictQueue) { lock->decConflictModeCount(iter->mode); if (iter->compatibleFirst) { - lock->compatibleFirstCount++; + newlyCompatibleFirst |= (lock->compatibleFirstCount++ == 0); } iter->notify->notify(lock->resourceId, LOCK_OK); - // Small optimization - nothing is compatible with MODE_X, so no point in looking - // further in the conflict queue. + // Small optimization - nothing is compatible with a newly granted MODE_X, so no point in + // looking further in the conflict queue. Conflicting MODE_X requests are skipped above. if (iter->mode == MODE_X) { break; } diff --git a/src/mongo/db/concurrency/lock_manager_test.cpp b/src/mongo/db/concurrency/lock_manager_test.cpp index 218dd6227ad..4fcda52ae89 100644 --- a/src/mongo/db/concurrency/lock_manager_test.cpp +++ b/src/mongo/db/concurrency/lock_manager_test.cpp @@ -26,6 +26,7 @@ * it in the license file. */ +#include "mongo/db/concurrency/lock_manager_defs.h" #include "mongo/db/concurrency/lock_manager_test_help.h" #include "mongo/unittest/unittest.h" @@ -743,6 +744,66 @@ TEST(LockManager, CompatibleFirstImmediateGrant) { ASSERT(lockMgr.unlock(&requestX)); } +TEST(LockManager, CompatibleFirstGrantAlreadyQueued) { + LockManager lockMgr; + const ResourceId resId(RESOURCE_GLOBAL, 0); + + // This tests the following behaviors (alternatives indicated with '|'): + // Lock held in X, queue: S X|IX IS, where S is compatibleFirst. + // Once X unlocks|downgrades both the S and IS requests should proceed. + + + enum UnblockMethod { kDowngrading, kUnlocking }; + LockMode conflictingModes[2] = {MODE_IX, MODE_X}; + UnblockMethod unblockMethods[2] = {kDowngrading, kUnlocking}; + + for (LockMode writerMode : conflictingModes) { + for (UnblockMethod unblockMethod : unblockMethods) { + MMAPV1LockerImpl locker1; + LockRequestCombo request1(&locker1); + + MMAPV1LockerImpl locker2; + LockRequestCombo request2(&locker2); + request2.compatibleFirst = true; + + MMAPV1LockerImpl locker3; + LockRequestCombo request3(&locker3); + + MMAPV1LockerImpl locker4; + LockRequestCombo request4(&locker4); + + // Hold the lock in X and establish the S IX|X IS queue. + ASSERT(LOCK_OK == lockMgr.lock(resId, &request1, MODE_X)); + ASSERT(LOCK_WAITING == lockMgr.lock(resId, &request2, MODE_S)); + ASSERT(LOCK_WAITING == lockMgr.lock(resId, &request3, writerMode)); + ASSERT(LOCK_WAITING == lockMgr.lock(resId, &request4, MODE_IS)); + + // Now unlock the initial X, so all readers should be able to proceed, while the writer + // remains queued. + if (unblockMethod == kUnlocking) { + ASSERT(lockMgr.unlock(&request1)); + } else { + invariant(unblockMethod == kDowngrading); + lockMgr.downgrade(&request1, MODE_S); + } + ASSERT(request2.lastResult == LOCK_OK); + ASSERT(request3.lastResult == LOCK_INVALID); + ASSERT(request4.lastResult == LOCK_OK); + + // Now unlock the readers, and the writer succeeds as well. + ASSERT(lockMgr.unlock(&request2)); + ASSERT(lockMgr.unlock(&request4)); + if (unblockMethod == kDowngrading) { + ASSERT(lockMgr.unlock(&request1)); + } + ASSERT(request3.lastResult == LOCK_OK); + + // Unlock the writer + ASSERT(lockMgr.unlock(&request3)); + } + } +} + TEST(LockManager, CompatibleFirstDelayedGrant) { LockManager lockMgr; const ResourceId resId(RESOURCE_GLOBAL, 0); diff --git a/src/mongo/db/concurrency/lock_state_test.cpp b/src/mongo/db/concurrency/lock_state_test.cpp index ec44c9dc096..c3a29275416 100644 --- a/src/mongo/db/concurrency/lock_state_test.cpp +++ b/src/mongo/db/concurrency/lock_state_test.cpp @@ -30,11 +30,13 @@ #include "mongo/platform/basic.h" +#include #include #include #include "mongo/config.h" #include "mongo/db/concurrency/lock_manager_test_help.h" +#include "mongo/db/concurrency/locker.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" #include "mongo/util/timer.h" @@ -361,5 +363,85 @@ TEST(Locker, PerformanceLocker) { } #endif // MONGO_CONFIG_DEBUG_BUILD +namespace { +/** + * Helper function to determine if 'lockerInfo' contains a lock with ResourceId 'resourceId' and + * lock mode 'mode' within 'lockerInfo.locks'. + */ +bool lockerInfoContainsLock(const Locker::LockerInfo& lockerInfo, + const ResourceId& resourceId, + const LockMode& mode) { + return (1U == std::count_if(lockerInfo.locks.begin(), + lockerInfo.locks.end(), + [&resourceId, &mode](const Locker::OneLock& lock) { + return lock.resourceId == resourceId && lock.mode == mode; + })); +} +} // namespace + +TEST(LockerImpl, GetLockerInfoShouldReportHeldLocks) { + const ResourceId globalId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL); + const ResourceId dbId(RESOURCE_DATABASE, std::string("TestDB")); + const ResourceId collectionId(RESOURCE_COLLECTION, std::string("TestDB.collection")); + + // Take an exclusive lock on the collection. + DefaultLockerImpl locker; + ASSERT_EQ(LOCK_OK, locker.lockGlobal(MODE_IX)); + ASSERT_EQ(LOCK_OK, locker.lock(dbId, MODE_IX)); + ASSERT_EQ(LOCK_OK, locker.lock(collectionId, MODE_X)); + + // Assert it shows up in the output of getLockerInfo(). + Locker::LockerInfo lockerInfo; + locker.getLockerInfo(&lockerInfo); + + ASSERT(lockerInfoContainsLock(lockerInfo, globalId, MODE_IX)); + ASSERT(lockerInfoContainsLock(lockerInfo, dbId, MODE_IX)); + ASSERT(lockerInfoContainsLock(lockerInfo, collectionId, MODE_X)); + ASSERT_EQ(3U, lockerInfo.locks.size()); + + ASSERT(locker.unlockAll()); +} + +TEST(LockerImpl, GetLockerInfoShouldReportPendingLocks) { + const ResourceId globalId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL); + const ResourceId dbId(RESOURCE_DATABASE, std::string("TestDB")); + const ResourceId collectionId(RESOURCE_COLLECTION, std::string("TestDB.collection")); + + // Take an exclusive lock on the collection. + DefaultLockerImpl successfulLocker; + ASSERT_EQ(LOCK_OK, successfulLocker.lockGlobal(MODE_IX)); + ASSERT_EQ(LOCK_OK, successfulLocker.lock(dbId, MODE_IX)); + ASSERT_EQ(LOCK_OK, successfulLocker.lock(collectionId, MODE_X)); + + // Now attempt to get conflicting locks. + DefaultLockerImpl conflictingLocker; + ASSERT_EQ(LOCK_OK, conflictingLocker.lockGlobal(MODE_IS)); + ASSERT_EQ(LOCK_OK, conflictingLocker.lock(dbId, MODE_IS)); + ASSERT_EQ(LOCK_WAITING, conflictingLocker.lockBegin(collectionId, MODE_IS)); + + // Assert the held locks show up in the output of getLockerInfo(). + Locker::LockerInfo lockerInfo; + conflictingLocker.getLockerInfo(&lockerInfo); + ASSERT(lockerInfoContainsLock(lockerInfo, globalId, MODE_IS)); + ASSERT(lockerInfoContainsLock(lockerInfo, dbId, MODE_IS)); + ASSERT(lockerInfoContainsLock(lockerInfo, collectionId, MODE_IS)); + ASSERT_EQ(3U, lockerInfo.locks.size()); + + // Assert it reports that it is waiting for the collection lock. + ASSERT_EQ(collectionId, lockerInfo.waitingResource); + + // Make sure it no longer reports waiting once unlocked. + ASSERT(successfulLocker.unlockAll()); + + const unsigned timeoutMs = 0; + const bool checkDeadlock = false; + ASSERT_EQ(LOCK_OK, + conflictingLocker.lockComplete(collectionId, MODE_IS, timeoutMs, checkDeadlock)); + + conflictingLocker.getLockerInfo(&lockerInfo); + ASSERT_FALSE(lockerInfo.waitingResource.isValid()); + + ASSERT(conflictingLocker.unlockAll()); +} } // namespace mongo diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index e18b60efb17..6b0dda65150 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -149,7 +149,10 @@ void (*snmpInit)() = NULL; extern int diagLogging; -static const NamespaceString startupLogCollectionName("local.startup_log"); +namespace { + +const NamespaceString startupLogCollectionName("local.startup_log"); +const NamespaceString kSystemReplSetCollection("local.system.replset"); #ifdef _WIN32 ntservice::NtServiceDefaultStrings defaultServiceStrings = { @@ -215,7 +218,7 @@ public: } }; -static void logStartup(OperationContext* txn) { +void logStartup(OperationContext* txn) { BSONObjBuilder toLog; stringstream id; id << getHostNameCached() << "-" << jsTime().asInt64(); @@ -255,7 +258,7 @@ static void logStartup(OperationContext* txn) { wunit.commit(); } -static void checkForIdIndexes(OperationContext* txn, Database* db) { +void checkForIdIndexes(OperationContext* txn, Database* db) { if (db->name() == "local") { // we do not need an _id index on anything in the local database return; @@ -293,13 +296,10 @@ static void checkForIdIndexes(OperationContext* txn, Database* db) { * @returns the number of documents in local.system.replset or 0 if this was started with * --replset. */ -static unsigned long long checkIfReplMissingFromCommandLine(OperationContext* txn) { - // This is helpful for the query below to work as you can't open files when readlocked - ScopedTransaction transaction(txn, MODE_X); - Lock::GlobalWrite lk(txn->lockState()); +unsigned long long checkIfReplMissingFromCommandLine(OperationContext* txn) { if (!repl::getGlobalReplicationCoordinator()->getSettings().usingReplSets()) { DBDirectClient c(txn); - return c.count("local.system.replset"); + return c.count(kSystemReplSetCollection.ns()); } return 0; } @@ -311,7 +311,7 @@ static unsigned long long checkIfReplMissingFromCommandLine(OperationContext* tx * start up or get promoted to be replica set primaries, newer nodes clear the temp flags left by * these versions. */ -static bool isSubjectToSERVER23299(OperationContext* txn) { +bool isSubjectToSERVER23299(OperationContext* txn) { dbHolder().openDb(txn, startupLogCollectionName.db()); AutoGetCollectionForRead autoColl(txn, startupLogCollectionName); // No startup log or an empty one means either that the user was not running an affected @@ -353,7 +353,7 @@ static bool isSubjectToSERVER23299(OperationContext* txn) { return true; } -static void handleSERVER23299ForDb(OperationContext* txn, Database* db) { +void handleSERVER23299ForDb(OperationContext* txn, Database* db) { log() << "Scanning " << db->name() << " db for SERVER-23299 eligibility"; const auto dbEntry = db->getDatabaseCatalogEntry(); list collNames; @@ -374,7 +374,22 @@ static void handleSERVER23299ForDb(OperationContext* txn, Database* db) { log() << "Done scanning " << db->name() << " for SERVER-23299 eligibility"; } -static void repairDatabasesAndCheckVersion(OperationContext* txn) { +/** + * Check that the oplog is capped, and abort the process if it is not. + * Caller must lock DB before calling this function. + */ +void checkForCappedOplog(OperationContext* txn, Database* db) { + const NamespaceString oplogNss(repl::rsOplogName); + invariant(txn->lockState()->isDbLockedForMode(oplogNss.db(), MODE_IS)); + Collection* oplogCollection = db->getCollection(oplogNss); + if (oplogCollection && !oplogCollection->isCapped()) { + severe() << "The oplog collection " << oplogNss + << " is not capped; a capped oplog is a requirement for replication to function."; + fassertFailedNoTrace(40116); + } +} + +void repairDatabasesAndCheckVersion(OperationContext* txn) { LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl; ScopedTransaction transaction(txn, MODE_X); @@ -397,6 +412,13 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { const repl::ReplSettings& replSettings = repl::getGlobalReplicationCoordinator()->getSettings(); + // We open the "local" database before calling checkIfReplMissingFromCommandLine() to ensure the + // in-memory catalog entries for the 'kSystemReplSetCollection' collection have been populated + // if the collection exists. If the "local" database didn't exist at this point yet, then it + // will be created. + Lock::DBLock dbLock(txn->lockState(), kSystemReplSetCollection.db(), MODE_X); + dbHolder().openDb(txn, kSystemReplSetCollection.db()); + // On replica set members we only clear temp collections on DBs other than "local" during // promotion to primary. On pure slaves, they are only cleared when the oplog tells them // to. The local DB is special because it is not replicated. See SERVER-10927 for more @@ -480,6 +502,11 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { if (replSettings.usingReplSets()) { // We only care about the _id index if we are in a replset checkForIdIndexes(txn, db); + // Ensure oplog is capped (mmap does not guarantee order of inserts on noncapped + // collections) + if (db->name() == "local") { + checkForCappedOplog(txn, db); + } } if (shouldDoCleanupForSERVER23299) { @@ -503,7 +530,7 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { LOG(1) << "done repairDatabases" << endl; } -static void _initWireSpec() { +void _initWireSpec() { WireSpec& spec = WireSpec::instance(); // accept from any version spec.minWireVersionIncoming = RELEASE_2_4_AND_BEFORE; @@ -513,8 +540,7 @@ static void _initWireSpec() { spec.maxWireVersionOutgoing = FIND_COMMAND; } - -static void _initAndListen(int listenPort) { +void _initAndListen(int listenPort) { Client::initThread("initandlisten"); _initWireSpec(); @@ -721,6 +747,8 @@ static void _initAndListen(int listenPort) { "data." << startupWarningsLog; } + logStartup(startupOpCtx.get()); + getDeleter()->startWorkers(); restartInProgressIndexesFromLastShutdown(startupOpCtx.get()); @@ -757,8 +785,6 @@ static void _initAndListen(int listenPort) { uassertStatusOK(ShardingStateRecovery::recover(startupOpCtx.get())); } - logStartup(startupOpCtx.get()); - // MessageServer::run will return when exit code closes its socket and we don't need the // operation context anymore startupOpCtx.reset(); @@ -785,6 +811,8 @@ ExitCode initAndListen(int listenPort) { } } +} // namespace + #if defined(_WIN32) ExitCode initService() { ntservice::reportStatus(SERVICE_RUNNING); diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp index b0da72c5bb8..4e7045061c1 100644 --- a/src/mongo/db/db_raii.cpp +++ b/src/mongo/db/db_raii.cpp @@ -134,8 +134,10 @@ void AutoGetCollectionForRead::_ensureMajorityCommittedSnapshotIsValid(const Nam uassertStatusOK(_txn->recoveryUnit()->setReadFromMajorityCommittedSnapshot()); - stdx::lock_guard lk(*_txn->getClient()); - CurOp::get(_txn)->yielded(); + { + stdx::lock_guard lk(*_txn->getClient()); + CurOp::get(_txn)->yielded(); + } // Relock. _autoColl.emplace(_txn, nss, MODE_IS); diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index e65da17dd07..53b92f56bd3 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -92,6 +92,7 @@ #include "mongo/rpc/reply_builder_interface.h" #include "mongo/rpc/metadata.h" #include "mongo/rpc/metadata/config_server_metadata.h" +#include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/rpc/metadata/server_selection_metadata.h" #include "mongo/rpc/metadata/sharding_metadata.h" #include "mongo/rpc/protocol.h" @@ -141,7 +142,7 @@ public: BSONObjBuilder& result) { bool force = cmdObj.hasField("force") && cmdObj["force"].trueValue(); - long long timeoutSecs = 0; + long long timeoutSecs = 10; if (cmdObj.hasField("timeoutSecs")) { timeoutSecs = cmdObj["timeoutSecs"].numberLong(); } @@ -1257,7 +1258,16 @@ void Command::execCommand(OperationContext* txn, replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet && !replCoord->canAcceptWritesForDatabase(dbname) && !replCoord->getMemberState().secondary()) { - uasserted(ErrorCodes::NotMasterOrSecondary, "node is recovering"); + uassert(ErrorCodes::NotMasterOrSecondary, + "node is recovering", + !replCoord->getMemberState().recovering()); + uassert(ErrorCodes::NotMasterOrSecondary, + "node is not in primary or recovering state", + replCoord->getMemberState().primary()); + // Check ticket SERVER-21432, slaveOk commands are allowed in drain mode + uassert(ErrorCodes::NotMasterOrSecondary, + "node is in drain mode", + commandIsOverriddenToRunOnSecondary || commandCanRunOnSecondary); } } @@ -1466,8 +1476,9 @@ bool Command::run(OperationContext* txn, if (isReplSet) { repl::OpTime lastOpTimeFromClient = repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(); - replCoord->prepareReplResponseMetadata(request, lastOpTimeFromClient, &metadataBob); - + if (request.getMetadata().hasField(rpc::kReplSetMetadataFieldName)) { + replCoord->prepareReplMetadata(lastOpTimeFromClient, &metadataBob); + } // For commands from mongos, append some info to help getLastError(w) work. // TODO: refactor out of here as part of SERVER-18326 if (isShardingAware || serverGlobalParams.configsvr) { diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index e0d6da23040..d3c62ea0ef8 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -351,6 +351,12 @@ long long Helpers::removeRange(OperationContext* txn, IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern(txn, indexKeyPattern.toBSON()); + if (!desc) { + warning(LogComponent::kSharding) << "shard key index " << indexKeyPattern.toBSON() + << " on '" << ns << "' was dropped"; + return -1; + } + unique_ptr exec( InternalPlanner::indexScan(txn, collection, diff --git a/src/mongo/db/exec/geo_near.cpp b/src/mongo/db/exec/geo_near.cpp index c4ae337ab16..71a9b50e749 100644 --- a/src/mongo/db/exec/geo_near.cpp +++ b/src/mongo/db/exec/geo_near.cpp @@ -264,8 +264,13 @@ class GeoNear2DStage::DensityEstimator { public: DensityEstimator(PlanStage::Children* children, const IndexDescriptor* twoDindex, - const GeoNearParams* nearParams) - : _children(children), _twoDIndex(twoDindex), _nearParams(nearParams), _currentLevel(0) { + const GeoNearParams* nearParams, + const R2Annulus& fullBounds) + : _children(children), + _twoDIndex(twoDindex), + _nearParams(nearParams), + _fullBounds(fullBounds), + _currentLevel(0) { GeoHashConverter::Parameters hashParams; Status status = GeoHashConverter::parseParameters(_twoDIndex->infoObj(), &hashParams); // The index status should always be valid. @@ -292,7 +297,8 @@ private: PlanStage::Children* _children; // Points to PlanStage::_children in the NearStage. const IndexDescriptor* _twoDIndex; // Not owned here. const GeoNearParams* _nearParams; // Not owned here. - IndexScan* _indexScan = nullptr; // Owned in PlanStage::_children. + const R2Annulus& _fullBounds; + IndexScan* _indexScan = nullptr; // Owned in PlanStage::_children. unique_ptr _converter; GeoHash _centroidCell; unsigned _currentLevel; @@ -359,6 +365,39 @@ PlanStage::StageState GeoNear2DStage::DensityEstimator::work(OperationContext* t if (state == PlanStage::IS_EOF) { // We ran through the neighbors but found nothing. + // + // Before going to the next-coarsest level, check whether our search area contains the + // entire search annulus, since we don't want to spend time doing density estimation over + // areas that are much larger than the requested $maxDistance. + // + // The search area consists of four cells with side length S. Within its cell, the closest + // vertex to the search point must be the vertex shared with the other three cells. If the + // search point lies in the upper left cell, this means that it must lie in the lower right + // quadrant of that cell. Furthermore, this lower-right quadrant has a side-length of S/2. + // + // +-----------+-----------+ + // | | | + // | S/2 | | + // + +-----+ | + // | | o | | + // | | | | + // +-----+-----+-----------+ + // | | | + // | | | + // | | | + // | | | + // | | | + // +-----------+-----------+ + // S + // + // As long as the outer radius of the search annulus is less than S/2, it must be entirely + // contained within these four cells. + if (_fullBounds.getOuter() < (0.5 * _converter->sizeEdge(_currentLevel))) { + // We're covering the entire search annulus. Return EOF to indicate we're done. + *estimatedDistance = 0.5 * _converter->sizeEdge(_currentLevel); + return PlanStage::IS_EOF; + } + if (_currentLevel > 0u) { // Advance to the next level and search again. _currentLevel--; @@ -391,7 +430,8 @@ PlanStage::StageState GeoNear2DStage::initialize(OperationContext* txn, Collection* collection, WorkingSetID* out) { if (!_densityEstimator) { - _densityEstimator.reset(new DensityEstimator(&_children, _twoDIndex, &_nearParams)); + _densityEstimator.reset( + new DensityEstimator(&_children, _twoDIndex, &_nearParams, _fullBounds)); } double estimatedDistance; @@ -793,11 +833,13 @@ public: DensityEstimator(PlanStage::Children* children, const IndexDescriptor* s2Index, const GeoNearParams* nearParams, - const S2IndexingParams& indexParams) + const S2IndexingParams& indexParams, + const R2Annulus& fullBounds) : _children(children), _s2Index(s2Index), _nearParams(nearParams), _indexParams(indexParams), + _fullBounds(fullBounds), _currentLevel(0) { // cellId.AppendVertexNeighbors(level, output) requires level < finest, // so we use the minimum of max_level - 1 and the user specified finest @@ -820,6 +862,7 @@ private: const IndexDescriptor* _s2Index; // Not owned here. const GeoNearParams* _nearParams; // Not owned here. const S2IndexingParams _indexParams; + const R2Annulus& _fullBounds; int _currentLevel; IndexScan* _indexScan = nullptr; // Owned in PlanStage::_children. }; @@ -873,6 +916,41 @@ PlanStage::StageState GeoNear2DSphereStage::DensityEstimator::work(OperationCont if (state == PlanStage::IS_EOF) { // We ran through the neighbors but found nothing. + // + // Before going to the next-coarsest level, check whether our search area contains the + // entire search annulus, since we don't want to spend time doing density estimation over + // areas that are much larger than the requested $maxDistance. + // + // The search area consists of four cells at level L. Within its cell, the closest vertex to + // the search point must be the vertex shared with the other three cells. If the search + // point lies in the upper left cell, this means that it must lie in the lower right + // sub-cell at level L+1. + // + // +-----------+-----------+ + // | | | + // | S | | + // + +-----+ | + // | | o | | + // | | | | + // +-----+-----+-----------+ + // | | | + // | | | + // | | | + // | | | + // | | | + // +-----------+-----------+ + // + // In the diagram above, S is the width of the cell at level L+1. We can determine a lower + // bound for the width any cell at this level, i.e. S > minWidth(L+1). As long as the outer + // radius of the search annulus is less than minWidth(L+1), it must be entirely contained + // within these four level L cells. + if (_fullBounds.getOuter() < + (S2::kMinWidth.GetValue(_currentLevel + 1) * kRadiusOfEarthInMeters)) { + // We're covering the entire search annulus. Return EOF to indicate we're done. + *estimatedDistance = S2::kMinWidth.GetValue(_currentLevel + 1) * kRadiusOfEarthInMeters; + return PlanStage::IS_EOF; + } + if (_currentLevel > 0) { // Advance to the next level and search again. _currentLevel--; @@ -907,7 +985,7 @@ PlanStage::StageState GeoNear2DSphereStage::initialize(OperationContext* txn, WorkingSetID* out) { if (!_densityEstimator) { _densityEstimator.reset( - new DensityEstimator(&_children, _s2Index, &_nearParams, _indexParams)); + new DensityEstimator(&_children, _s2Index, &_nearParams, _indexParams, _fullBounds)); } double estimatedDistance; diff --git a/src/mongo/db/ftdc/SConscript b/src/mongo/db/ftdc/SConscript index 67601b2d3ab..5b8c2d58455 100644 --- a/src/mongo/db/ftdc/SConscript +++ b/src/mongo/db/ftdc/SConscript @@ -32,21 +32,32 @@ ftdcEnv.Library( ], ) +platform_libs = [] + +if env.TargetOSIs('linux'): + platform_libs = [ + '$BUILD_DIR/mongo/util/procparser' + ] +elif env.TargetOSIs('windows'): + platform_libs = [ + '$BUILD_DIR/mongo/util/perfctr_collect' + ] + env.Library( target='ftdc_mongod', source=[ + 'ftdc_commands.cpp', 'ftdc_mongod.cpp', + 'ftdc_system_stats.cpp', + 'ftdc_system_stats_${TARGET_OS}.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/server_parameters', + '$BUILD_DIR/mongo/db/storage/storage_options', + '$BUILD_DIR/mongo/util/processinfo', 'ftdc' - ], - LIBDEPS_TAGS=[ - # Needs 'mongo::storageGlobalParams' in storage_options.cpp which is in serveronly - # And this library is already part of serveronly - 'incomplete', - ], + ] + platform_libs, ) env.CppUnitTest( diff --git a/src/mongo/db/ftdc/config.h b/src/mongo/db/ftdc/config.h index ef7ee8b9143..feb57aaa1e6 100644 --- a/src/mongo/db/ftdc/config.h +++ b/src/mongo/db/ftdc/config.h @@ -86,7 +86,7 @@ struct FTDCConfig { static const bool kEnabledDefault = true; static const std::uint64_t kPeriodMillisDefault; - static const std::uint64_t kMaxDirectorySizeBytesDefault = 100 * 1024 * 1024; + static const std::uint64_t kMaxDirectorySizeBytesDefault = 200 * 1024 * 1024; static const std::uint64_t kMaxFileSizeBytesDefault = 10 * 1024 * 1024; static const std::uint64_t kMaxFileUniqifier = 65000; diff --git a/src/mongo/db/ftdc/controller.cpp b/src/mongo/db/ftdc/controller.cpp index 65819c7e380..c13f6fc7fcf 100644 --- a/src/mongo/db/ftdc/controller.cpp +++ b/src/mongo/db/ftdc/controller.cpp @@ -99,6 +99,13 @@ void FTDCController::addOnRotateCollector(std::unique_ptr lock(_mutex); + return _mostRecentPeriodicDocument.getOwned(); + } +} + void FTDCController::start() { log() << "Initializing full-time diagnostic data capture with directory '" << _path.generic_string() << "'"; @@ -211,6 +218,12 @@ void FTDCController::doLoop() { client, std::get<0>(collectSample), std::get<1>(collectSample)); uassertStatusOK(s); + + // Store a reference to the most recent document from the periodic collectors + { + stdx::lock_guard lock(_mutex); + _mostRecentPeriodicDocument = std::get<0>(collectSample); + } } } } catch (...) { diff --git a/src/mongo/db/ftdc/controller.h b/src/mongo/db/ftdc/controller.h index 3db82b6b171..44b5e838506 100644 --- a/src/mongo/db/ftdc/controller.h +++ b/src/mongo/db/ftdc/controller.h @@ -43,6 +43,8 @@ namespace mongo { +class ServiceContext; + /** * Responsible for periodic collection of samples, writing them to disk, * and rotation. @@ -118,6 +120,16 @@ public: */ void stop(); + /** + * Get the FTDCController from ServiceContext. + */ + static FTDCController* get(ServiceContext* serviceContext); + + /** + * Get a reference to most recent document from the periodic collectors. + */ + BSONObj getMostRecentPeriodicDocument(); + private: /** * Do periodic statistics collection, and all other work on the background thread. @@ -162,7 +174,7 @@ private: // Directory to store files const boost::filesystem::path _path; - // Mutex to protect the condvar, and configuration changes. + // Mutex to protect the condvar, configuration changes, and most recent periodic document. stdx::mutex _mutex; stdx::condition_variable _condvar; @@ -176,6 +188,10 @@ private: // Set of periodic collectors FTDCCollectorCollection _periodicCollectors; + // Last seen sample document from periodic collectors + // Owned + BSONObj _mostRecentPeriodicDocument; + // Set of file rotation collectors FTDCCollectorCollection _rotateCollectors; diff --git a/src/mongo/db/ftdc/file_writer.cpp b/src/mongo/db/ftdc/file_writer.cpp index 576f946700a..1f290249c67 100644 --- a/src/mongo/db/ftdc/file_writer.cpp +++ b/src/mongo/db/ftdc/file_writer.cpp @@ -57,9 +57,6 @@ Status FTDCFileWriter::open(const boost::filesystem::path& file) { _archiveFile = file; - // Disable file buffering - _archiveStream.rdbuf()->pubsetbuf(0, 0); - // Ideally, we create a file from scratch via O_CREAT but there is not portable way via C++ // iostreams to do this. _archiveStream.open(_archiveFile.c_str(), @@ -87,9 +84,6 @@ Status FTDCFileWriter::writeInterimFileBuffer(ConstDataRange buf) { // Fixed size interim stream std::ofstream interimStream; - // Disable file buffering - interimStream.rdbuf()->pubsetbuf(0, 0); - // Open up a temporary interim file interimStream.open(_interimTempFile.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); @@ -134,6 +128,18 @@ Status FTDCFileWriter::writeArchiveFileBuffer(ConstDataRange buf) { << _archiveFile.generic_string()}; } + // Flush the stream explictly, this is preferred over "pubsetbuf(0,0)" which has implementation + // defined behavior of "before any I/O has occurred". + _archiveStream.flush(); + + if (_archiveStream.fail()) { + return { + ErrorCodes::FileStreamFailed, + str::stream() + << "Failed to flush to archive file buffer for full-time diagnostic data capture: " + << _archiveFile.generic_string()}; + } + _size += buf.length(); return Status::OK(); diff --git a/src/mongo/db/ftdc/file_writer_test.cpp b/src/mongo/db/ftdc/file_writer_test.cpp index 6555a13c115..62443894254 100644 --- a/src/mongo/db/ftdc/file_writer_test.cpp +++ b/src/mongo/db/ftdc/file_writer_test.cpp @@ -45,6 +45,7 @@ namespace mongo { const char* kTestFile = "metrics.test"; +const char* kTestFileCopy = "metrics.test.copy"; // File Sanity check TEST(FTDCFileTest, TestFileBasicMetadata) { @@ -160,7 +161,27 @@ public: } private: - void validate(bool forceCompress = true) { + void validate() { + // Verify we are flushing writes correctly by copying the file, and then reading it. + auto tempfile(boost::filesystem::path(_tempdir.path()) / kTestFileCopy); + boost::filesystem::copy_file(_path, tempfile); + + // Read the file to make sure it is correct. + // We do not verify contents because the compressor may not have flushed the final records + // which is expected. + { + FTDCFileReader reader; + + ASSERT_OK(reader.open(tempfile)); + + auto sw = reader.hasNext(); + while (sw.isOK() && sw.getValue()) { + sw = reader.hasNext(); + } + + ASSERT_OK(sw); + } + _writer.close(); ValidateDocumentList(_path, _docs); @@ -244,6 +265,25 @@ TEST(FTDCFileTest, TestFull) { } } +// Test a large documents so that we cause multiple 4kb buffers to flush on Windows. +TEST(FTDCFileTest, TestLargeDocuments) { + FileTestTie c; + + for (int j = 0; j < 5; j++) { + for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault; i++) { + BSONObjBuilder b; + b.append("name", "joe"); + for (size_t k = 0; k < 200; k++) { + b.appendNumber( + "num", + static_cast(i * j + 5000 - (sin(static_cast(k)) * 100))); + } + + c.addSample(b.obj()); + } + } +} + // Test a bad file TEST(FTDCFileTest, TestBadFile) { unittest::TempDir tempdir("metrics_testpath"); diff --git a/src/mongo/db/ftdc/ftdc_commands.cpp b/src/mongo/db/ftdc/ftdc_commands.cpp new file mode 100644 index 00000000000..13487171aa6 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_commands.cpp @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/init.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/operation_context.h" + +namespace mongo { +namespace { + +/** + * Get the most recent document FTDC collected from its periodic collectors. + * + * Document will be empty if FTDC has never run. + */ +class GetDiagnosticDataCommand final : public Command { +public: + GetDiagnosticDataCommand() : Command("getDiagnosticData") {} + + bool adminOnly() const override { + return true; + } + + void help(std::stringstream& help) const override { + help << "get latest diagnostic data collection snapshot"; + } + + bool slaveOk() const override { + return true; + } + + bool isWriteCommandForConfigServer() const override { + return false; + } + + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) override { + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::serverStatus)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::replSetGetStatus)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString("local", "oplog.rs")), + ActionType::collStats)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + return Status::OK(); + } + + bool run(OperationContext* txn, + const std::string& db, + BSONObj& cmdObj, + int options, + std::string& errmsg, + BSONObjBuilder& result) override { + result.append( + "data", FTDCController::get(txn->getServiceContext())->getMostRecentPeriodicDocument()); + + return true; + } +}; + +Command* ftdcCommand; + +MONGO_INITIALIZER(CreateDiagnosticDataCommand)(InitializerContext* context) { + ftdcCommand = new GetDiagnosticDataCommand(); + + return Status::OK(); +} + +} // namespace + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_mongod.cpp b/src/mongo/db/ftdc/ftdc_mongod.cpp index e328dbd3dff..f9861186af0 100644 --- a/src/mongo/db/ftdc/ftdc_mongod.cpp +++ b/src/mongo/db/ftdc/ftdc_mongod.cpp @@ -28,6 +28,8 @@ #include "mongo/platform/basic.h" +#include "mongo/db/ftdc/ftdc_mongod.h" + #include #include #include @@ -35,11 +37,11 @@ #include "mongo/base/init.h" #include "mongo/base/status.h" #include "mongo/bson/bsonobjbuilder.h" - #include "mongo/db/commands.h" #include "mongo/db/ftdc/collector.h" #include "mongo/db/ftdc/config.h" #include "mongo/db/ftdc/controller.h" +#include "mongo/db/ftdc/ftdc_system_stats.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/replication_coordinator_global.h" @@ -288,7 +290,6 @@ private: } // namespace - // Register the FTDC system // Note: This must be run before the server parameters are parsed during startup // so that the FTDCController is initialized. @@ -310,6 +311,8 @@ void startFTDC() { // Install periodic collectors // These are collected on the period interval in FTDCConfig. + // NOTE: For each command here, there must be an equivalent privilege check in + // GetDiagnosticDataCommand // CmdServerStatus controller->addPeriodicCollector(stdx::make_unique( @@ -327,6 +330,9 @@ void startFTDC() { "collStats", "local.oplog.rs.stats", "local.oplog.rs", BSONObj())); } + // Install System Metric Collector as a periodic collector + installSystemMetricsCollector(controller.get()); + // Install file rotation collectors // These are collected on each file rotation. @@ -358,4 +364,8 @@ void stopFTDC() { } } +FTDCController* FTDCController::get(ServiceContext* serviceContext) { + return getFTDCController(serviceContext).get(); +} + } // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats.cpp b/src/mongo/db/ftdc/ftdc_system_stats.cpp new file mode 100644 index 00000000000..42a2892d964 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats.cpp @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/collector.h" + +namespace mongo { + +namespace { + +/** + * Name of FTDC collector to create. + */ +auto kSystemMetricsCollector = "systemMetrics"; + +} // namespace + +std::string SystemMetricsCollector::name() const { + return kSystemMetricsCollector; +} + +void SystemMetricsCollector::processStatusErrors(Status s, BSONObjBuilder* builder) { + if (!s.isOK()) { + builder->append("error", s.toString()); + } +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats.h b/src/mongo/db/ftdc/ftdc_system_stats.h new file mode 100644 index 00000000000..20d21ef4f39 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats.h @@ -0,0 +1,64 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/ftdc/controller.h" + +namespace mongo { + +class FTDCController; + +/** + * Base class for system metrics collectors. Sets collector name to a common name all system metrics + * collectors to use. + */ +class SystemMetricsCollector : public FTDCCollectorInterface { +public: + std::string name() const final; + +protected: + /** + * Convert any errors we see into BSON for the user to see in the final FTDC document. It is + * acceptable for the collector to fail, but we do not want to shutdown the FTDC loop because + * of it. We assume that the BSONBuilder is not corrupt on non-OK Status but nothing else with + * regards to the final document output. + */ + static void processStatusErrors(Status s, BSONObjBuilder* builder); +}; + + +/** + * Install a system metrics collector if it exists as a periodic collector. + */ +void installSystemMetricsCollector(FTDCController* controller); + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp b/src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp new file mode 100644 index 00000000000..5ed3c6e89ba --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/procparser.h" + +namespace mongo { + +namespace { + +static const std::vector kCpuKeys{ + "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running"}; + +static const std::vector kMemKeys{ + "MemTotal", + "MemFree", + "Cached", + "Dirty", + "Buffers", + "SwapTotal", + "SwapCached", + "SwapFree", + "Active", + "Inactive", + "Active(anon)", + "Inactive(anon)", + "Active(file)", + "Inactive(file)", +}; + +/** + * Collect metrics from the Linux /proc file system. + */ +class LinuxSystemMetricsCollector final : public SystemMetricsCollector { +public: + LinuxSystemMetricsCollector() : _disks(procparser::findPhysicalDisks("/sys/block")) { + for (const auto& disk : _disks) { + _disksStringData.emplace_back(disk); + } + } + + void collect(OperationContext* txn, BSONObjBuilder& builder) override { + { + BSONObjBuilder subObjBuilder(builder.subobjStart("cpu")); + + // Include the number of cpus to simplify client calculations + ProcessInfo p; + subObjBuilder.append("num_cpus", p.getNumCores()); + + processStatusErrors( + procparser::parseProcStatFile("/proc/stat", kCpuKeys, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + + { + BSONObjBuilder subObjBuilder(builder.subobjStart("memory")); + processStatusErrors( + procparser::parseProcMemInfoFile("/proc/meminfo", kMemKeys, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + + // Skip the disks section if we could not find any disks. + // This can happen when we do not have permission to /sys/block for instance. + if (!_disksStringData.empty()) { + BSONObjBuilder subObjBuilder(builder.subobjStart("disks")); + processStatusErrors(procparser::parseProcDiskStatsFile( + "/proc/diskstats", _disksStringData, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + } + +private: + // List of physical disks to collect stats from as string from findPhysicalDisks. + std::vector _disks; + + // List of physical disks to collect stats from as StringData to pass to parseProcDiskStatsFile. + std::vector _disksStringData; +}; + +} // namespace + +void installSystemMetricsCollector(FTDCController* controller) { + controller->addPeriodicCollector(stdx::make_unique()); +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp b/src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_osx.cpp b/src/mongo/db/ftdc/ftdc_system_stats_osx.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_osx.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp b/src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_windows.cpp b/src/mongo/db/ftdc/ftdc_system_stats_windows.cpp new file mode 100644 index 00000000000..e684ebcf326 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_windows.cpp @@ -0,0 +1,154 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/log.h" +#include "mongo/util/perfctr_collect.h" + +namespace mongo { + +namespace { + +const std::vector kCpuCounters = { + "\\Processor(_Total)\\% Idle Time", + "\\Processor(_Total)\\% Interrupt Time", + "\\Processor(_Total)\\% Privileged Time", + "\\Processor(_Total)\\% Processor Time", + "\\Processor(_Total)\\% User Time", + "\\Processor(_Total)\\Interrupts/sec", + "\\System\\Context Switches/sec", + "\\System\\Processes", + "\\System\\Processor Queue Length", + "\\System\\System Up Time", + "\\System\\Threads", +}; + +const std::vector kMemoryCounters = { + "\\Memory\\Available Bytes", + "\\Memory\\Cache Bytes", + "\\Memory\\Cache Faults/sec", + "\\Memory\\Committed Bytes", + "\\Memory\\Commit Limit", + "\\Memory\\Page Reads/sec", + "\\Memory\\Page Writes/sec", + "\\Memory\\Pages Input/sec", + "\\Memory\\Pages Output/sec", + "\\Memory\\Pool Nonpaged Bytes", + "\\Memory\\Pool Paged Bytes", + "\\Memory\\Pool Paged Resident Bytes", + "\\Memory\\System Cache Resident Bytes", + "\\Memory\\System Code Total Bytes", + +}; + +const std::vector kDiskCounters = { + "\\PhysicalDisk(*)\\% Disk Read Time", + "\\PhysicalDisk(*)\\% Disk Write Time", + "\\PhysicalDisk(*)\\Avg. Disk Read Queue Length", + "\\PhysicalDisk(*)\\Avg. Disk Write Queue Length", + "\\PhysicalDisk(*)\\Disk Read Bytes/sec", + "\\PhysicalDisk(*)\\Disk Write Bytes/sec", + "\\PhysicalDisk(*)\\Disk Reads/sec", + "\\PhysicalDisk(*)\\Disk Writes/sec", + "\\PhysicalDisk(*)\\Current Disk Queue Length", +}; + + +/** + * Collect metrics from Windows Performance Counters. + */ +class WindowsSystemMetricsCollector final : public SystemMetricsCollector { +public: + WindowsSystemMetricsCollector(std::unique_ptr collector) + : _collector(std::move(collector)) {} + + void collect(OperationContext* txn, BSONObjBuilder& builder) override { + processStatusErrors(_collector->collect(&builder), &builder); + } + +private: + std::unique_ptr _collector; +}; + + +StatusWith> createCollector() { + PerfCounterCollection collection; + + Status s = collection.addCountersGroup("cpu", kCpuCounters); + if (!s.isOK()) { + return s; + } + + // TODO: Should we capture the Heap Counters for the current process? + s = collection.addCountersGroup("memory", kMemoryCounters); + if (!s.isOK()) { + return s; + } + + s = collection.addCountersGroupedByInstanceName("disks", kDiskCounters); + if (!s.isOK()) { + return s; + } + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + if (!swCollector.getStatus().isOK()) { + return swCollector.getStatus(); + } + + return {std::move(swCollector.getValue())}; +} + +} // namespace + +void installSystemMetricsCollector(FTDCController* controller) { + auto swCollector = createCollector(); + if (!swCollector.getStatus().isOK()) { + warning() << "Failed to initialize Performance Counters for FTDC: " + << swCollector.getStatus(); + return; + } + + controller->addPeriodicCollector( + stdx::make_unique(std::move(swCollector.getValue()))); +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_test.cpp b/src/mongo/db/ftdc/ftdc_test.cpp index 33fcce62eff..28910def1a1 100644 --- a/src/mongo/db/ftdc/ftdc_test.cpp +++ b/src/mongo/db/ftdc/ftdc_test.cpp @@ -58,6 +58,8 @@ void ValidateDocumentList(const boost::filesystem::path& p, const std::vector IndexAccessMethod::initiateBulk() { - return std::unique_ptr(new BulkBuilder(this, _descriptor)); +std::unique_ptr IndexAccessMethod::initiateBulk( + size_t maxMemoryUsageBytes) { + return std::unique_ptr(new BulkBuilder(this, _descriptor, maxMemoryUsageBytes)); } IndexAccessMethod::BulkBuilder::BulkBuilder(const IndexAccessMethod* index, - const IndexDescriptor* descriptor) + const IndexDescriptor* descriptor, + size_t maxMemoryUsageBytes) : _sorter(Sorter::make( SortOptions() .TempDir(storageGlobalParams.dbpath + "/_tmp") .ExtSortAllowed() - .MaxMemoryUsageBytes(100 * 1024 * 1024), + .MaxMemoryUsageBytes(maxMemoryUsageBytes), BtreeExternalSortComparison(descriptor->keyPattern(), descriptor->version()))), _real(index) {} diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h index 401f0dbe0e4..8ea1d58fde8 100644 --- a/src/mongo/db/index/index_access_method.h +++ b/src/mongo/db/index/index_access_method.h @@ -202,7 +202,9 @@ public: using Sorter = mongo::Sorter; - BulkBuilder(const IndexAccessMethod* index, const IndexDescriptor* descriptor); + BulkBuilder(const IndexAccessMethod* index, + const IndexDescriptor* descriptor, + size_t maxMemoryUsageBytes); std::unique_ptr _sorter; const IndexAccessMethod* _real; @@ -216,8 +218,11 @@ public: * This can return NULL, meaning bulk mode is not available. * * It is only legal to initiate bulk when the index is new and empty. + * + * maxMemoryUsageBytes: amount of memory consumed before the external sorter starts spilling to + * disk */ - std::unique_ptr initiateBulk(); + std::unique_ptr initiateBulk(size_t maxMemoryUsageBytes); /** * Call this when you are ready to finish your bulk work. diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp index 2bd2d003b49..ef292b96f35 100644 --- a/src/mongo/db/instance.cpp +++ b/src/mongo/db/instance.cpp @@ -886,7 +886,7 @@ bool receivedGetMore(OperationContext* txn, DbResponse& dbresponse, Message& m, curop.debug().cursorid = cursorid; { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); CurOp::get(txn)->setNS_inlock(ns); } @@ -1158,7 +1158,7 @@ void receivedInsert(OperationContext* txn, const NamespaceString& nsString, Mess DbMessage d(m); const char* ns = d.getns(); { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); CurOp::get(txn)->setNS_inlock(nsString.ns()); } diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index e8facbaa850..610adbdbc3d 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -44,9 +44,9 @@ #include "mongo/db/server_options.h" #include "mongo/db/server_options_helpers.h" #include "mongo/db/storage/mmap_v1/mmap_v1_options.h" -#include "mongo/s/catalog/catalog_manager.h" #include "mongo/logger/console_appender.h" #include "mongo/logger/message_event_utf8_encoder.h" +#include "mongo/s/catalog/catalog_manager.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/net/ssl_options.h" @@ -332,12 +332,12 @@ Status addMongodOptions(moe::OptionSection* options) { "storage.mmapv1.journal.commitIntervalMs"); // Deprecated option that we don't want people to use for performance reasons - storage_options.addOptionChaining("nopreallocj", + storage_options.addOptionChaining("storage.mmapv1.journal.nopreallocj", "nopreallocj", moe::Switch, "don't preallocate journal files") .hidden() - .setSources(moe::SourceAllLegacy); + .setSources(moe::SourceAll); #if defined(__linux__) general_options.addOptionChaining( @@ -1032,8 +1032,8 @@ Status storeMongodOptions(const moe::Environment& params, const std::vector(); } - if (params.count("nopreallocj")) { - mmapv1GlobalOptions.preallocj = !params["nopreallocj"].as(); + if (params.count("storage.mmapv1.journal.nopreallocj")) { + mmapv1GlobalOptions.preallocj = !params["storage.mmapv1.journal.nopreallocj"].as(); } if (params.count("net.http.RESTInterfaceEnabled")) { diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 1e70c1fa376..715cd170c48 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -174,5 +174,6 @@ env.CppUnitTest( 'pipeline', '$BUILD_DIR/mongo/db/auth/authorization_manager_mock_init', '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/auth/authorization_session_for_test' ], ) diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index 6a2845ecddc..a6b81feb420 100644 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -30,12 +30,12 @@ #include "mongo/platform/basic.h" -#include #include +#include +#include #include #include #include -#include #include #include @@ -502,7 +502,7 @@ private: typedef std::vector> Accumulators; - typedef std::unordered_map GroupsMap; + typedef boost::unordered_map GroupsMap; GroupsMap groups; /* diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 04ed07ae797..eba7877ed5a 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -331,45 +331,67 @@ Status Pipeline::checkAuthForCommand(ClientBasic* client, std::vector privileges; - if (cmdObj.getFieldDotted("pipeline.0.$indexStats")) { - Privilege::addPrivilegeToPrivilegeVector( - &privileges, - Privilege(ResourcePattern::forAnyNormalResource(), ActionType::indexStats)); - } else { - // If no source requiring an alternative permission scheme is specified then default to - // requiring find() privileges on the given namespace. + BSONElement pipelineElem = cmdObj["pipeline"]; + if (pipelineElem.type() != BSONType::Array) { + return Status(ErrorCodes::TypeMismatch, "'pipeline' must be specified as an array"); + } + + BSONObj pipeline = pipelineElem.embeddedObject(); + if (pipeline.isEmpty()) { + // The pipeline is empty, so we require only the find action. Privilege::addPrivilegeToPrivilegeVector(&privileges, Privilege(inputResource, ActionType::find)); - } + } else { + if (pipeline.firstElementType() != BSONType::Object) { + // The pipeline contains something that's not an object. + return Status(ErrorCodes::TypeMismatch, + "'pipeline' cannot contain non-object elements"); + } - BSONObj pipeline = cmdObj.getObjectField("pipeline"); - BSONForEach(stageElem, pipeline) { - BSONObj stage = stageElem.embeddedObjectUserCheck(); - StringData stageName = stage.firstElementFieldName(); - if (stageName == "$out" && stage.firstElementType() == String) { - NamespaceString outputNs(db, stage.firstElement().str()); - uassert(17139, - mongoutils::str::stream() << "Invalid $out target namespace, " << outputNs.ns(), - outputNs.isValid()); - - ActionSet actions; - actions.addAction(ActionType::remove); - actions.addAction(ActionType::insert); - if (shouldBypassDocumentValidationForCommand(cmdObj)) { - actions.addAction(ActionType::bypassDocumentValidation); - } + // We treat the first stage in the pipeline specially, as $indexStats changes the authz + // requirements when appearing in this position. + BSONObj firstPipelineStage = pipeline.firstElement().embeddedObject(); + if (str::equals("$indexStats", firstPipelineStage.firstElementFieldName())) { Privilege::addPrivilegeToPrivilegeVector( - &privileges, Privilege(ResourcePattern::forExactNamespace(outputNs), actions)); - } else if (stageName == "$lookup" && stage.firstElementType() == Object) { - NamespaceString fromNs(db, stage.firstElement()["from"].str()); - Privilege::addPrivilegeToPrivilegeVector( - &privileges, - Privilege(ResourcePattern::forExactNamespace(fromNs), ActionType::find)); + &privileges, Privilege(inputResource, ActionType::indexStats)); + } else { + // If no source requiring an alternative permission scheme is specified then default to + // requiring find() privileges on the given namespace. + Privilege::addPrivilegeToPrivilegeVector(&privileges, + Privilege(inputResource, ActionType::find)); + } + + BSONObj pipelineStages = cmdObj.getObjectField("pipeline"); + BSONForEach(stageElem, pipelineStages) { + BSONObj stage = stageElem.embeddedObjectUserCheck(); + StringData stageName = stage.firstElementFieldName(); + if (stageName == "$out" && stage.firstElementType() == String) { + NamespaceString outputNs(db, stage.firstElement().str()); + uassert(17139, + mongoutils::str::stream() << "Invalid $out target namespace, " + << outputNs.ns(), + outputNs.isValid()); + + ActionSet actions; + actions.addAction(ActionType::remove); + actions.addAction(ActionType::insert); + if (shouldBypassDocumentValidationForCommand(cmdObj)) { + actions.addAction(ActionType::bypassDocumentValidation); + } + Privilege::addPrivilegeToPrivilegeVector( + &privileges, Privilege(ResourcePattern::forExactNamespace(outputNs), actions)); + } else if (stageName == "$lookup" && stage.firstElementType() == Object) { + NamespaceString fromNs(db, stage.firstElement()["from"].str()); + Privilege::addPrivilegeToPrivilegeVector( + &privileges, + Privilege(ResourcePattern::forExactNamespace(fromNs), ActionType::find)); + } } } - if (AuthorizationSession::get(client)->isAuthorizedForPrivileges(privileges)) + if (AuthorizationSession::get(client)->isAuthorizedForPrivileges(privileges)) { return Status::OK(); + } return Status(ErrorCodes::Unauthorized, "unauthorized"); } diff --git a/src/mongo/db/pipeline/pipeline_test.cpp b/src/mongo/db/pipeline/pipeline_test.cpp index 333fc5963de..edf9b30ca87 100644 --- a/src/mongo/db/pipeline/pipeline_test.cpp +++ b/src/mongo/db/pipeline/pipeline_test.cpp @@ -28,12 +28,20 @@ #include "mongo/platform/basic.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session_for_test.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" +#include "mongo/db/auth/authz_session_external_state_mock.h" +#include "mongo/db/operation_context_noop.h" #include "mongo/db/pipeline/document.h" #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/field_path.h" #include "mongo/db/pipeline/pipeline.h" -#include "mongo/db/operation_context_noop.h" +#include "mongo/db/service_context.h" +#include "mongo/db/service_context_noop.h" #include "mongo/dbtests/dbtests.h" +#include "mongo/stdx/memory.h" namespace mongo { bool isMongos() { @@ -638,4 +646,215 @@ public: SuiteInstance myall; +namespace AuthzTests { +const NamespaceString testFooNss("test.foo"); +const NamespaceString testBarNss("test.bar"); +const std::string testFooDb = testFooNss.db().toString(); +const std::string testBarDb = testBarNss.db().toString(); +const ResourcePattern testFooCollResource(ResourcePattern::forExactNamespace(testFooNss)); +const ResourcePattern testBarCollResource(ResourcePattern::forExactNamespace(testBarNss)); + +class PipelineAuthzTest : public ::mongo::unittest::Test { +public: + AuthzManagerExternalStateMock* managerState; + OperationContextNoop txn; + AuthzSessionExternalStateMock* sessionState; + std::unique_ptr authzManager; + std::unique_ptr service; + ServiceContext::UniqueClient client; + AuthorizationSessionForTest* authzSession; + + void setUp() override { + auto localManagerState = stdx::make_unique(); + managerState = localManagerState.get(); + managerState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); + authzManager = stdx::make_unique(std::move(localManagerState)); + authzManager->setAuthEnabled(true); + auto localSessionState = + stdx::make_unique(authzManager.get()); + sessionState = localSessionState.get(); + service = stdx::make_unique(); + client = service->makeClient("PipelineAuthzTest"); + auto authzSessionForTest = + stdx::make_unique(std::move(localSessionState)); + authzSession = authzSessionForTest.get(); + AuthorizationSession::set(client.get(), std::move(authzSessionForTest)); + } +}; + +TEST_F(PipelineAuthzTest, CheckAuthForCommandFailsIfPipelineIsNotAnArray) { + BSONObj cmdObjIntPipeline = BSON("aggregate" << testFooNss.coll() << "pipeline" << 7); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjIntPipeline)); + + BSONObj cmdObjObjPipeline = BSON("aggregate" << testFooNss.coll() << "pipeline" << BSONObj()); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjObjPipeline)); + + BSONObj cmdObjNoPipeline = BSON("aggregate" << testFooNss.coll()); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjNoPipeline)); +} + +TEST_F(PipelineAuthzTest, CheckAuthForCommandFailsIfPipelineFirstStageIsNotAnObject) { + BSONObj cmdObjFirstStageInt = + BSON("aggregate" << testFooNss.coll() << "pipeline" << BSON_ARRAY(7)); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjFirstStageInt)); + + BSONObj cmdObjFirstStageArray = + BSON("aggregate" << testFooNss.coll() << "pipeline" << BSON_ARRAY(BSONArray())); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjFirstStageArray)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateEmptyPipelineWithoutFindAction) { + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << BSONArray()); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateEmptyPipelineWithFindAction) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << BSONArray()); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateWithoutFindActionIfFirstStageNotIndexStats) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::indexStats})); + + BSONArray pipeline = BSON_ARRAY(BSON("$limit" << 1) << BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateWithFindActionIfFirstStageNotIndexStats) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$limit" << 1) << BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateIndexStatsWithoutIndexStatsAction) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateIndexStatsWithIndexStatsAction) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::indexStats})); + + BSONArray pipeline = BSON_ARRAY(BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CheckAuthForCommandFailsIfOutNamespaceIsNotValid) { + BSONArray pipeline = BSON_ARRAY(BSON("$out" + << "")); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_THROWS_CODE( + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj), UserException, 17139); +} + +TEST_F(PipelineAuthzTest, CannotAggregateOutWithoutInsertAndRemoveOnTargetNamespace) { + // We only have find on the aggregation namespace. + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); + + // We have insert but not remove on the $out namespace. + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::insert})}); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); + + // We have remove but not insert on the $out namespace. + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::remove})}); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateOutWithInsertAndRemoveOnTargetNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::insert, ActionType::remove})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); + + BSONObj cmdObjNoBypassDocumentValidation = + BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline + << "bypassDocumentValidation" << false); + ASSERT_OK( + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjNoBypassDocumentValidation)); +} + +TEST_F(PipelineAuthzTest, + CannotAggregateOutBypassingValidationWithoutBypassDocumentValidationOnTargetNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::insert, ActionType::remove})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline + << "bypassDocumentValidation" << true); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, + CanAggregateOutBypassingValidationWithBypassDocumentValidationOnTargetNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, + ActionSet{ActionType::insert, + ActionType::remove, + ActionType::bypassDocumentValidation})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline + << "bypassDocumentValidation" << true); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateLookupWithoutFindOnJoinedNamespace) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$lookup" << BSON("from" << testBarNss.coll()))); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateLookupWithFindOnJoinedNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::find})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$lookup" << BSON("from" << testBarNss.coll()))); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} +} // namespace AuthzTests } // namespace PipelineTests diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp index 813b019aba7..dc2784b2dd8 100644 --- a/src/mongo/db/query/find.cpp +++ b/src/mongo/db/query/find.cpp @@ -42,7 +42,6 @@ #include "mongo/db/db_raii.h" #include "mongo/db/exec/filter.h" #include "mongo/db/exec/working_set_common.h" -#include "mongo/db/service_context.h" #include "mongo/db/keypattern.h" #include "mongo/db/matcher/extensions_callback_real.h" #include "mongo/db/query/explain.h" @@ -54,6 +53,7 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/server_options.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" #include "mongo/db/storage/storage_options.h" #include "mongo/s/chunk_version.h" #include "mongo/s/stale_exception.h" @@ -171,7 +171,7 @@ void endQueryOp(OperationContext* txn, if (dbProfilingLevel > 0 || curop->elapsedMillis() > serverGlobalParams.slowMS || logger::globalLogDomain()->shouldLog(commandLogComponent, logLevelOne)) { // Generate plan summary string. - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); curop->setPlanSummary_inlock(Explain::getPlanSummary(&exec)); } @@ -389,7 +389,7 @@ QueryResult::View getMore(OperationContext* txn, auto planSummary = Explain::getPlanSummary(exec); { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); curop.setPlanSummary_inlock(planSummary); } @@ -525,13 +525,38 @@ std::string runQuery(OperationContext* txn, LOG(5) << "Running query:\n" << cq->toString(); LOG(2) << "Running query: " << cq->toStringShort(); + ShardingState* const shardingState = ShardingState::get(txn); + // Parse, canonicalize, plan, transcribe, and get a plan executor. - AutoGetCollectionForRead ctx(txn, nss); + boost::optional optionalCtx; + try { + optionalCtx.emplace(txn, nss); + } catch (const StaleConfigException& sce) { + // Wait for migration completion to get the correct chunk version + const int maxTimeoutSec = 30; + int timeoutSec = cq->getParsed().getMaxTimeMS() / 1000; + if (!timeoutSec || timeoutSec > maxTimeoutSec) { + timeoutSec = maxTimeoutSec; + } + + if (shardingState->waitTillNotInCriticalSection(maxTimeoutSec)) { + ChunkVersion unused; + shardingState->refreshMetadataIfNeeded( + txn, nss.ns(), sce.getVersionReceived(), &unused); + } + throw; + } + + const auto& ctx = *optionalCtx; Collection* collection = ctx.getCollection(); const int dbProfilingLevel = ctx.getDb() ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile; + // It is possible that the sharding version will change during yield while we are retrieving a + // plan executor. If this happens we will throw an error and mongos will retry. + const ChunkVersion shardingVersionAtStart = shardingState->getVersion(nss.ns()); + // We have a parsed query. Time to get the execution plan for it. std::unique_ptr exec = uassertStatusOK( getExecutorFind(txn, collection, nss, std::move(cq), PlanExecutor::YIELD_AUTO)); @@ -568,11 +593,6 @@ std::string runQuery(OperationContext* txn, return ""; } - ShardingState* const shardingState = ShardingState::get(txn); - - // We freak out later if this changes before we're done with the query. - const ChunkVersion shardingVersionAtStart = shardingState->getVersion(nss.ns()); - // Handle query option $maxTimeMS (not used with commands). curop.setMaxTimeMicros(static_cast(pq.getMaxTimeMS()) * 1000); txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. diff --git a/src/mongo/db/query/lite_parsed_query.cpp b/src/mongo/db/query/lite_parsed_query.cpp index a4f9696b7a0..7dfdd093f12 100644 --- a/src/mongo/db/query/lite_parsed_query.cpp +++ b/src/mongo/db/query/lite_parsed_query.cpp @@ -375,7 +375,7 @@ StatusWith> LiteParsedQuery::makeFromFindCommand(Nam pq->addMetaProjection(); - Status validateStatus = pq->validateFindCmd(); + Status validateStatus = pq->validate(); if (!validateStatus.isOK()) { return validateStatus; } @@ -662,6 +662,11 @@ Status LiteParsedQuery::validate() const { } } + // AwaitData is only valid with a tailable cursor. + if (_awaitData && !_tailable) { + return Status(ErrorCodes::BadValue, "Cannot set awaitData without tailable"); + } + return Status::OK(); } @@ -975,14 +980,6 @@ void LiteParsedQuery::addMetaProjection() { } } -Status LiteParsedQuery::validateFindCmd() { - if (isAwaitData() && !isTailable()) { - return Status(ErrorCodes::BadValue, "Cannot set awaitData without tailable"); - } - - return validate(); -} - boost::optional LiteParsedQuery::getEffectiveBatchSize() const { return _batchSize ? _batchSize : _ntoreturn; } diff --git a/src/mongo/db/query/lite_parsed_query.h b/src/mongo/db/query/lite_parsed_query.h index 0287ad159e6..57c1ccd0a04 100644 --- a/src/mongo/db/query/lite_parsed_query.h +++ b/src/mongo/db/query/lite_parsed_query.h @@ -331,11 +331,6 @@ private: */ void addMetaProjection(); - /** - * Returns OK if this is valid in the find command context. - */ - Status validateFindCmd(); - const NamespaceString _nss; BSONObj _filter; diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 007a37c5c25..af7515855f9 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -250,6 +250,7 @@ env.Library('topology_coordinator_impl', 'repl_settings', 'rslog', 'topology_coordinator', + '$BUILD_DIR/mongo/util/fail_point', ]) env.CppUnitTest('repl_set_heartbeat_response_test', diff --git a/src/mongo/db/repl/applier.h b/src/mongo/db/repl/applier.h index a8e48c0edb9..d035e0d80ef 100644 --- a/src/mongo/db/repl/applier.h +++ b/src/mongo/db/repl/applier.h @@ -64,7 +64,7 @@ public: /** * Type of function to to apply a single operation. In production, this function - * would have the same outcome as calling SyncTail::syncApply() ('convertUpdatesToUpserts' + * would have the same outcome as calling SyncTail::syncApply() ('inSteadyStateReplication' * value will be embedded in the function implementation). */ using ApplyOperationFn = stdx::function; diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index a0312fa30f2..98d86f0c7c0 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -52,9 +52,12 @@ #include "mongo/db/repl/rollback_source_impl.h" #include "mongo/db/repl/rs_rollback.h" #include "mongo/db/repl/rs_sync.h" +#include "mongo/db/repl/storage_interface.h" #include "mongo/db/stats/timer_stats.h" #include "mongo/executor/network_interface_factory.h" +#include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" +#include "mongo/rpc/metadata/server_selection_metadata.h" #include "mongo/stdx/memory.h" #include "mongo/util/concurrency/thread_pool.h" #include "mongo/util/exit.h" @@ -115,7 +118,7 @@ Status checkRemoteOplogStart(stdx::function()> getNextOperat } // namespace -MONGO_FP_DECLARE(rsBgSyncProduce); +MONGO_FP_DECLARE(stopReplProducer); BackgroundSync* BackgroundSync::s_instance = 0; stdx::mutex BackgroundSync::s_mutex; @@ -131,6 +134,12 @@ static ServerStatusMetricField displayOpsRead("repl.network.ops", &op static Counter64 networkByteStats; static ServerStatusMetricField displayBytesRead("repl.network.bytes", &networkByteStats); +// Failpoint which causes rollback to hang before starting. +MONGO_FP_DECLARE(rollbackHangBeforeStart); + +// Failpoint which causes the oplog fetcher to hang before the first fetch. +MONGO_FP_DECLARE(fetcherHangBeforeStart); + // The count of items in the buffer static Counter64 bufferCountGauge; static ServerStatusMetricField displayBufferCount("repl.buffer.count", @@ -154,6 +163,8 @@ size_t getSize(const BSONObj& o) { } } // namespace +const NamespaceString BackgroundSync::kLocalOplogNss("local.oplog.rs"); + BackgroundSync::BackgroundSync() : _buffer(bufferMaxSizeGauge, &getSize), _threadPoolTaskExecutor(makeThreadPool(), @@ -242,6 +253,8 @@ void BackgroundSync::_producerThread() { return; } + invariant(!state.rollback()); + // We need to wait until initial sync has started. if (_replCoord->getMyLastAppliedOpTime().isNull()) { sleepsecs(1); @@ -258,6 +271,21 @@ void BackgroundSync::_producerThread() { } void BackgroundSync::_produce(OperationContext* txn) { + if (MONGO_FAIL_POINT(stopReplProducer)) { + // This log output is used in js tests so please leave it. + log() << "bgsync - stopReplProducer fail point " + "enabled. Blocking until fail point is disabled."; + + // TODO(SERVER-27120): Remove the return statement and uncomment the while loop. + // Currently we cannot block here or we prevent primaries from being fully elected since + // we'll never call _signalNoNewDataForApplier. + // while (MONGO_FAIL_POINT(stopReplProducer) && !inShutdown()) { + // mongo::sleepsecs(1); + // } + mongo::sleepsecs(1); + return; + } + // this oplog reader does not do a handshake because we don't want the server it's syncing // from to track how far it has synced { @@ -276,16 +304,13 @@ void BackgroundSync::_produce(OperationContext* txn) { } } - while (MONGO_FAIL_POINT(rsBgSyncProduce)) { - sleepmillis(0); - } - - + HostAndPort oldSyncSource; // find a target to sync from the last optime fetched OpTime lastOpTimeFetched; { stdx::unique_lock lock(_mutex); lastOpTimeFetched = _lastOpTimeFetched; + oldSyncSource = _syncSourceHost; _syncSourceHost = HostAndPort(); } OplogReader syncSourceReader; @@ -296,7 +321,9 @@ void BackgroundSync::_produce(OperationContext* txn) { minValid = minValidSaved; } } - syncSourceReader.connectToSyncSource(txn, lastOpTimeFetched, minValid, _replCoord); + + int rbid; + syncSourceReader.connectToSyncSource(txn, lastOpTimeFetched, minValid, _replCoord, &rbid); // no server found if (syncSourceReader.getHost().empty()) { @@ -305,6 +332,17 @@ void BackgroundSync::_produce(OperationContext* txn) { return; } + // If our sync source has not changed, it is likely caused by our heartbeat data map being + // out of date. In that case we sleep for 1 second to reduce the amount we spin waiting + // for our map to update. + if (syncSourceReader.getHost() == oldSyncSource) { + log() << "Chose same sync source candidate as last time, " << oldSyncSource + << ". Sleeping for 1 second to avoid immediately choosing a new sync source for " + "the same reason as last time."; + + sleepsecs(1); + } + long long lastHashFetched; { stdx::lock_guard lock(_mutex); @@ -317,7 +355,7 @@ void BackgroundSync::_produce(OperationContext* txn) { _replCoord->signalUpstreamUpdater(); } - const Milliseconds oplogSocketTimeout(OplogReader::kSocketTimeout); + const Milliseconds kRollbackOplogSocketTimeout(10 * 60 * 1000); const auto isV1ElectionProtocol = _replCoord->isV1ElectionProtocol(); // Under protocol version 1, make the awaitData timeout (maxTimeMS) dependent on the election @@ -348,7 +386,8 @@ void BackgroundSync::_produce(OperationContext* txn) { lastOpTimeFetched, lastHashFetched, fetcherMaxTimeMS, - &fetcherReturnStatus); + &fetcherReturnStatus, + rbid); BSONObjBuilder cmdBob; @@ -365,16 +404,32 @@ void BackgroundSync::_produce(OperationContext* txn) { metadataBob.append(rpc::kReplSetMetadataFieldName, 1); } + if (MONGO_FAIL_POINT(fetcherHangBeforeStart)) { + // This log output is used in js tests so please leave it. + log() << "BackgroundSync - fetcherHangBeforeStart fail point " + "enabled. Blocking until fail point is disabled."; + while (MONGO_FAIL_POINT(fetcherHangBeforeStart) && !inShutdown()) { + mongo::sleepsecs(1); + } + + // If the sync source candidate rolls back while in this fail point, it will close all + // connections and the next request will fail. + // We manually drop all connections here so that the following Fetcher request succeeds. + _threadPoolTaskExecutor.dropConnections(source); + } + auto dbName = nsToDatabase(rsOplogName); auto cmdObj = cmdBob.obj(); auto metadataObj = metadataBob.obj(); + // 5 seconds more than the find command's 1 minute maxTimeMs + const Milliseconds oplogQueryNetworkTimeout = duration_cast(Seconds(65)); Fetcher fetcher(&_threadPoolTaskExecutor, source, dbName, cmdObj, fetcherCallback, metadataObj, - _replCoord->getConfig().getElectionTimeoutPeriod()); + oplogQueryNetworkTimeout); LOG(1) << "scheduling fetcher to read remote oplog on " << source << " starting at " << cmdObj["filter"]; @@ -409,10 +464,10 @@ void BackgroundSync::_produce(OperationContext* txn) { ConnectionPool connectionPool(messagingPortTags); std::unique_ptr connection; auto getConnection = - [&connection, &connectionPool, oplogSocketTimeout, source]() -> DBClientBase* { + [&connection, &connectionPool, kRollbackOplogSocketTimeout, source]() -> DBClientBase* { if (!connection.get()) { connection.reset(new ConnectionPool::ConnectionPtr( - &connectionPool, source, Date_t::now(), oplogSocketTimeout)); + &connectionPool, source, Date_t::now(), kRollbackOplogSocketTimeout)); }; return connection->get(); }; @@ -436,32 +491,55 @@ void BackgroundSync::_produce(OperationContext* txn) { } } } - // check that we are at minvalid, otherwise we cannot roll back as we may be in an - // inconsistent state - const auto minValid = getMinValid(txn); - if (lastApplied < minValid) { - fassertNoTrace(18750, - Status(ErrorCodes::UnrecoverableRollbackError, - str::stream() - << "need to rollback, but in inconsistent state. " - << "minvalid: " << minValid.toString() - << " > our last optime: " << lastApplied.toString())); - } - _rollback(txn, source, getConnection); + _rollback(txn, source, rbid, getConnection); stop(); } else if (!fetcherReturnStatus.isOK()) { warning() << "Fetcher error querying oplog: " << fetcherReturnStatus.toString(); } } +void BackgroundSync::_lastAppliedFetcherCallback(const StatusWith& result, + OpTime lastOpTimeFetched, + Status* returnStatus) { + if (!result.isOK()) { + *returnStatus = result.getStatus(); + return; + } + + const auto& queryResponse = result.getValue(); + if (queryResponse.documents.empty()) { + *returnStatus = Status(ErrorCodes::InvalidSyncSource, "Upstream node had an empty oplog."); + return; + } + + const auto& remoteLastAppliedDocument = queryResponse.documents.front(); + const auto remoteLastAppliedOpTime = OpTime::parseFromOplogEntry(remoteLastAppliedDocument); + if (!remoteLastAppliedOpTime.isOK()) { + *returnStatus = Status(ErrorCodes::InvalidBSON, + str::stream() << "Received invalid oplog entry from upstream node: " + << remoteLastAppliedDocument.toString() << ". Error: " + << remoteLastAppliedOpTime.getStatus().toString()); + return; + } + if (remoteLastAppliedOpTime.getValue() <= lastOpTimeFetched) { + *returnStatus = Status(ErrorCodes::InvalidSyncSource, + str::stream() << "Upstream node's last applied OpTime " + << remoteLastAppliedOpTime.getValue().toString() + << " is not greater than our last fetched OpTime " + << lastOpTimeFetched.toString()); + return; + } +} + void BackgroundSync::_fetcherCallback(const StatusWith& result, BSONObjBuilder* bob, const HostAndPort& source, OpTime lastOpTimeFetched, long long lastFetchedHash, Milliseconds fetcherMaxTimeMS, - Status* returnStatus) { + Status* returnStatus, + int rbid) { // if target cut connections between connecting and querying (for // example, because it stepped down) we might not have a cursor if (!result.isOK()) { @@ -481,26 +559,6 @@ void BackgroundSync::_fetcherCallback(const StatusWith& bool syncSourceHasSyncSource = false; OpTime sourcesLastOp; - // Forward metadata (containing liveness information) to replication coordinator. - bool receivedMetadata = - queryResponse.otherFields.metadata.hasElement(rpc::kReplSetMetadataFieldName); - if (receivedMetadata) { - auto metadataResult = - rpc::ReplSetMetadata::readFromMetadata(queryResponse.otherFields.metadata); - if (!metadataResult.isOK()) { - error() << "invalid replication metadata from sync source " << source << ": " - << metadataResult.getStatus() << ": " << queryResponse.otherFields.metadata; - return; - } - const auto& metadata = metadataResult.getValue(); - _replCoord->processReplSetMetadata(metadata); - if (metadata.getPrimaryIndex() != rpc::ReplSetMetadata::kNoPrimary) { - _replCoord->cancelAndRescheduleElectionTimeout(); - } - syncSourceHasSyncSource = metadata.getSyncSourceIndex() != -1; - sourcesLastOp = metadata.getLastOpVisible(); - } - const auto& documents = queryResponse.documents; auto firstDocToApply = documents.cbegin(); auto lastDocToApply = documents.cend(); @@ -515,6 +573,64 @@ void BackgroundSync::_fetcherCallback(const StatusWith& // Check start of remote oplog and, if necessary, stop fetcher to execute rollback. if (queryResponse.first) { + // Once we establish our cursor, we need to ensure that our upstream node hasn't rolled back + // since that could cause it to not have our required minValid point. The cursor will be + // killed if the upstream node rolls back so we don't need to keep checking. This must be + // blocking since the Fetcher doesn't give us a way to defer sending the getmores after we + // return. + auto handle = _threadPoolTaskExecutor.scheduleRemoteCommand( + {source, "admin", BSON("replSetGetRBID" << 1)}, + [&](const executor::TaskExecutor::RemoteCommandCallbackArgs& rbidReply) { + *returnStatus = rbidReply.response.getStatus(); + if (!returnStatus->isOK()) + return; + + const auto& rbidReplyObj = rbidReply.response.getValue().data; + *returnStatus = getStatusFromCommandResult(rbidReplyObj); + if (!returnStatus->isOK()) + return; + + const auto rbidElem = rbidReplyObj["rbid"]; + if (rbidElem.type() != NumberInt) { + *returnStatus = + Status(ErrorCodes::InvalidSyncSource, + str::stream() << "Upstream node returned an " + << "rbid with invalid type " << rbidElem.type()); + return; + } + if (rbidElem.Int() != rbid) { + *returnStatus = Status(ErrorCodes::InvalidSyncSource, + "Upstream node rolled back after verifying " + "that it had our MinValid point. Retrying."); + } + }); + if (!handle.isOK()) { + *returnStatus = handle.getStatus(); + return; + } + + _threadPoolTaskExecutor.wait(handle.getValue()); + if (!returnStatus->isOK()) + return; + + // Check that the upstream last applied OpTime is newer than our last fetched OpTime. + Fetcher lastAppliedFetcher(&_threadPoolTaskExecutor, + source, + kLocalOplogNss.db().toString(), + BSON("find" << kLocalOplogNss.coll() << "limit" << 1 << "sort" + << BSON("$natural" << -1)), + stdx::bind(&BackgroundSync::_lastAppliedFetcherCallback, + this, + stdx::placeholders::_1, + lastOpTimeFetched, + returnStatus), + rpc::ServerSelectionMetadata(true, boost::none).toBSON(), + Seconds(30)); + lastAppliedFetcher.schedule(); + lastAppliedFetcher.wait(); + if (!returnStatus->isOK()) + return; + auto getNextOperation = [&firstDocToApply, lastDocToApply]() -> StatusWith { if (firstDocToApply == lastDocToApply) { return Status(ErrorCodes::OplogStartMissing, "remote oplog start missing"); @@ -539,6 +655,32 @@ void BackgroundSync::_fetcherCallback(const StatusWith& return; } + if (MONGO_FAIL_POINT(stopReplProducer)) { + return; + } + + // Process replset metadata. It is important that this happen after we've validated the + // first batch, so we don't progress our knowledge of the commit point from a + // response that triggers a rollback. + bool receivedMetadata = + queryResponse.otherFields.metadata.hasElement(rpc::kReplSetMetadataFieldName); + if (receivedMetadata) { + auto metadataResult = + rpc::ReplSetMetadata::readFromMetadata(queryResponse.otherFields.metadata); + if (!metadataResult.isOK()) { + error() << "invalid replication metadata from sync source " << source << ": " + << metadataResult.getStatus() << ": " << queryResponse.otherFields.metadata; + return; + } + const auto& metadata = metadataResult.getValue(); + _replCoord->processReplSetMetadata(metadata, true /*advance commit point*/); + if (metadata.getPrimaryIndex() != rpc::ReplSetMetadata::kNoPrimary) { + _replCoord->cancelAndRescheduleElectionTimeout(); + } + syncSourceHasSyncSource = metadata.getSyncSourceIndex() != -1; + sourcesLastOp = metadata.getLastOpVisible(); + } + // The count of the bytes of the documents read off the network. int networkDocumentBytes = 0; Timestamp lastTS; @@ -702,40 +844,86 @@ void BackgroundSync::consume() { void BackgroundSync::_rollback(OperationContext* txn, const HostAndPort& source, + boost::optional requiredRBID, stdx::function getConnection) { - // Abort only when syncRollback detects we are in a unrecoverable state. - // In other cases, we log the message contained in the error status and retry later. - auto status = syncRollback(txn, - OplogInterfaceLocal(txn, rsOplogName), - RollbackSourceImpl(getConnection, source, rsOplogName), - _replCoord); - if (status.isOK()) { - // When the syncTail thread sees there is no new data by adding something to the buffer. - _signalNoNewDataForApplier(); - // Wait until the buffer is empty. - // This is an indication that syncTail has removed the sentinal marker from the buffer - // and reset its local lastAppliedOpTime via the replCoord. - while (!_buffer.empty()) { - sleepmillis(10); - if (inShutdown()) { - return; - } + if (MONGO_FAIL_POINT(rollbackHangBeforeStart)) { + // This log output is used in js tests so please leave it. + log() << "rollback - rollbackHangBeforeStart fail point " + "enabled. Blocking until fail point is disabled."; + while (MONGO_FAIL_POINT(rollbackHangBeforeStart) && !inShutdown()) { + mongo::sleepsecs(1); } + } - // It is now safe to clear the ROLLBACK state, which may result in the applier thread - // transitioning to SECONDARY. This is safe because the applier thread has now reloaded - // the new rollback minValid from the database. - if (!_replCoord->setFollowerMode(MemberState::RS_RECOVERING)) { - warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) - << "; expected to be in state " << MemberState(MemberState::RS_ROLLBACK) - << " but found self in " << _replCoord->getMemberState(); + // Set state to ROLLBACK while we are in this function. This prevents serving reads, even from + // the oplog. This can fail if we are elected PRIMARY, in which case we better not do any + // rolling back. If we successfully enter ROLLBACK we will only exit this function fatally or + // after transitioning to RECOVERING. We always transition to RECOVERING regardless of success + // or (recoverable) failure since we may be in an inconsistent state. If rollback failed before + // writing anything, SyncTail will quickly take us to SECONDARY since are are still at our + // original MinValid, which is fine because we may choose a sync source that doesn't require + // rollback. If it failed after we wrote to MinValid, then we will pick a sync source that will + // cause us to roll back to the same common point, which is fine. If we succeeded, we will be + // consistent as soon as we apply up to/through MinValid and SyncTail will make us SECONDARY + // then. + { + log() << "rollback 0"; + Lock::GlobalWrite globalWrite(txn->lockState()); + if (!_replCoord->setFollowerMode(MemberState::RS_ROLLBACK)) { + log() << "Cannot transition from " << _replCoord->getMemberState().toString() << " to " + << MemberState(MemberState::RS_ROLLBACK).toString(); + return; } - return; } - if (ErrorCodes::UnrecoverableRollbackError == status.code()) { - fassertNoTrace(28723, status); + + try { + auto status = syncRollback(txn, + OplogInterfaceLocal(txn, rsOplogName), + RollbackSourceImpl(getConnection, source, rsOplogName), + requiredRBID, + _replCoord); + + // Abort only when syncRollback detects we are in a unrecoverable state. + // WARNING: these statuses sometimes have location codes which are lost with uassertStatusOK + // so we need to check here first. + if (ErrorCodes::UnrecoverableRollbackError == status.code()) { + severe() << "Unable to complete rollback. A full resync may be needed: " << status; + fassertFailedNoTrace(28723); + } + + // In other cases, we log the message contained in the error status and retry later. + uassertStatusOK(status); + } catch (const DBException& ex) { + // UnrecoverableRollbackError should only come from a returned status which is handled + // above. + invariant(ex.getCode() != ErrorCodes::UnrecoverableRollbackError); + + warning() << "rollback cannot complete at this time (retrying later): " << ex + << " appliedThrough=" << _replCoord->getMyLastAppliedOpTime() + << " minvalid=" << getMinValid(txn); + + // Sleep a bit to allow upstream node to coalesce, if that was the cause of the failure. If + // we failed in a way that will keep failing, but wasn't flagged as a fatal failure, this + // will also prevent us from hot-looping and putting too much load on upstream nodes. + sleepsecs(5); // 5 seconds was chosen as a completely arbitrary amount of time. + } catch (...) { + std::terminate(); + } + + // At this point we are about to leave rollback. Before we do, wait for any writes done + // as part of rollback to be durable, and then do any necessary checks that we didn't + // wind up rolling back something illegal. We must wait for the rollback to be durable + // so that if we wind up shutting down uncleanly in response to something we rolled back + // we know that we won't wind up right back in the same situation when we start back up + // because the rollback wasn't durable. + txn->recoveryUnit()->waitUntilDurable(); + + if (!_replCoord->setFollowerMode(MemberState::RS_RECOVERING)) { + severe() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) + << "; expected to be in state " << MemberState(MemberState::RS_ROLLBACK) + << " but found self in " << _replCoord->getMemberState(); + fassertFailedNoTrace(40364); } - warning() << "rollback cannot proceed at this time (retrying later): " << status; } HostAndPort BackgroundSync::getSyncTarget() { diff --git a/src/mongo/db/repl/bgsync.h b/src/mongo/db/repl/bgsync.h index 489790f988d..20ea6629ba0 100644 --- a/src/mongo/db/repl/bgsync.h +++ b/src/mongo/db/repl/bgsync.h @@ -77,6 +77,8 @@ public: */ class BackgroundSync : public BackgroundSyncInterface { public: + static const NamespaceString kLocalOplogNss; + // Allow index prefetching to be turned on/off enum IndexPrefetchConfig { UNINITIALIZED = 0, @@ -187,7 +189,16 @@ private: OpTime lastOpTimeFetched, long long lastFetchedHash, Milliseconds fetcherMaxTimeMS, - Status* returnStatus); + Status* returnStatus, + int rbid); + + /** + * A callback to a Fetcher that checks that the remote last applied OpTime is newer than the + * local last fetched OpTime. + */ + void _lastAppliedFetcherCallback(const StatusWith& result, + OpTime lastOpTimeFetched, + Status* returnStatus); /** * Executes a rollback. @@ -195,6 +206,7 @@ private: */ void _rollback(OperationContext* txn, const HostAndPort& source, + boost::optional requiredRBID, stdx::function getConnection); /** diff --git a/src/mongo/db/repl/data_replicator.cpp b/src/mongo/db/repl/data_replicator.cpp index aed481df509..145ffc2e92c 100644 --- a/src/mongo/db/repl/data_replicator.cpp +++ b/src/mongo/db/repl/data_replicator.cpp @@ -386,10 +386,12 @@ Status DatabasesCloner::start() { _status = Status::OK(); log() << "starting cloning of all databases"; - // Schedule listDatabase command which will kick off the database cloner per result db. + // Schedule listDatabase command which will kick off the database cloner per result db. We only + // retrieve database names since computing & fetching all database stats can be costly on the + // remote node when there are a large number of collections. Request listDBsReq(_source, "admin", - BSON("listDatabases" << true), + BSON("listDatabases" << true << "nameOnly" << true), rpc::ServerSelectionMetadata(true, boost::none).toBSON()); CBHStatus s = _exec->scheduleRemoteCommand( listDBsReq, diff --git a/src/mongo/db/repl/freshness_checker.cpp b/src/mongo/db/repl/freshness_checker.cpp index 71b7b2e419b..ef1afe02f92 100644 --- a/src/mongo/db/repl/freshness_checker.cpp +++ b/src/mongo/db/repl/freshness_checker.cpp @@ -149,7 +149,8 @@ void FreshnessChecker::Algorithm::processResponse(const RemoteCommandRequest& re LOG(2) << "FreshnessChecker: Got response from " << request.target << " of " << res; if (res["fresher"].trueValue()) { - log() << "not electing self, we are not freshest"; + log() << "not electing self, " << request.target.toString() + << " knows a node is fresher than us"; _abortReason = FresherNodeFound; return; } @@ -162,10 +163,15 @@ void FreshnessChecker::Algorithm::processResponse(const RemoteCommandRequest& re } Timestamp remoteTime(res["opTime"].date()); if (remoteTime == _lastOpTimeApplied) { + log() << "not electing self, " << request.target.toString() + << " has same OpTime as us: " << remoteTime.toBSON(); _abortReason = FreshnessTie; } if (remoteTime > _lastOpTimeApplied) { // something really wrong (rogue command?) + log() << "not electing self, " << request.target.toString() + << " has newer OpTime than us. Our OpTime: " << _lastOpTimeApplied.toBSON() + << ", their OpTime: " << remoteTime.toBSON(); _abortReason = FresherNodeFound; return; } diff --git a/src/mongo/db/repl/freshness_checker_test.cpp b/src/mongo/db/repl/freshness_checker_test.cpp index a8a269b627b..606be981611 100644 --- a/src/mongo/db/repl/freshness_checker_test.cpp +++ b/src/mongo/db/repl/freshness_checker_test.cpp @@ -262,7 +262,8 @@ TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshest) { stopCapturingLogMessages(); ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound); - ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest")); + ASSERT_EQUALS( + 1, countLogLinesContaining("not electing self, h1:27017 knows a node is fresher than us")); } TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestOpTime) { @@ -471,7 +472,8 @@ TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestManyNodes) { waitOnChecker(); stopCapturingLogMessages(); ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound); - ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest")); + ASSERT_EQUALS( + 1, countLogLinesContaining("not electing self, h1:27017 knows a node is fresher than us")); } TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestOpTimeManyNodes) { diff --git a/src/mongo/db/repl/last_vote.cpp b/src/mongo/db/repl/last_vote.cpp index 0b125624019..e7cc78ace9f 100644 --- a/src/mongo/db/repl/last_vote.cpp +++ b/src/mongo/db/repl/last_vote.cpp @@ -38,27 +38,31 @@ namespace { const std::string kCandidateIndexFieldName = "candidateIndex"; const std::string kTermFieldName = "term"; +const std::string kIdFieldName = "_id"; -const std::string kLegalFieldNames[] = { - kCandidateIndexFieldName, kTermFieldName, -}; +const std::string kLegalFieldNames[] = {kCandidateIndexFieldName, kTermFieldName, kIdFieldName}; } // namespace -Status LastVote::initialize(const BSONObj& argsObj) { - Status status = bsonCheckOnlyHasFields("VotedFar", argsObj, kLegalFieldNames); +LastVote::LastVote(long long term, long long candidateIndex) + : _candidateIndex(candidateIndex), _term(term) {} + +StatusWith LastVote::readFromLastVote(const BSONObj& doc) { + Status status = bsonCheckOnlyHasFields("LastVote", doc, kLegalFieldNames); if (!status.isOK()) return status; - status = bsonExtractIntegerField(argsObj, kTermFieldName, &_term); + long long term; + status = bsonExtractIntegerField(doc, kTermFieldName, &term); if (!status.isOK()) return status; - status = bsonExtractIntegerField(argsObj, kCandidateIndexFieldName, &_candidateIndex); + long long candidateIndex; + status = bsonExtractIntegerField(doc, kCandidateIndexFieldName, &candidateIndex); if (!status.isOK()) return status; - return Status::OK(); + return LastVote{term, candidateIndex}; } void LastVote::setTerm(long long term) { diff --git a/src/mongo/db/repl/last_vote.h b/src/mongo/db/repl/last_vote.h index 0466ffccbec..db1a9ab75e0 100644 --- a/src/mongo/db/repl/last_vote.h +++ b/src/mongo/db/repl/last_vote.h @@ -28,17 +28,20 @@ #pragma once +#include "mongo/base/status_with.h" + namespace mongo { class BSONObj; class BSONObjBuilder; -class Status; namespace repl { class LastVote { public: - Status initialize(const BSONObj& argsObj); + LastVote(long long term, long long candidateIndex); + + static StatusWith readFromLastVote(const BSONObj& doc); long long getTerm() const; long long getCandidateIndex() const; @@ -48,8 +51,8 @@ public: BSONObj toBSON() const; private: - long long _candidateIndex = -1; - long long _term = -1; + long long _candidateIndex; + long long _term; }; } // namespace repl diff --git a/src/mongo/db/repl/master_slave.cpp b/src/mongo/db/repl/master_slave.cpp index 1cb12ba561e..dcd5d77f395 100644 --- a/src/mongo/db/repl/master_slave.cpp +++ b/src/mongo/db/repl/master_slave.cpp @@ -537,7 +537,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, // missing from master after optime "ts". return false; } - if (Database::duplicateUncasedName(db).empty()) { + if (dbHolder().getNamesWithConflictingCasing(db).empty()) { // No duplicate database names are present. return true; } @@ -547,7 +547,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, { // This is always a GlobalWrite lock (so no ns/db used from the context) invariant(txn->lockState()->isW()); - Lock::TempRelease(txn->lockState()); + Lock::TempRelease tempRelease(txn->lockState()); // We always log an operation after executing it (never before), so // a database list will always be valid as of an oplog entry generated @@ -594,8 +594,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, } // Check for duplicates again, since we released the lock above. - set duplicates; - Database::duplicateUncasedName(db, &duplicates); + auto duplicates = dbHolder().getNamesWithConflictingCasing(db); // The database is present on the master and no conflicting databases // are present on the master. Drop any local conflicts. @@ -610,20 +609,20 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, massert(14034, "Duplicate database names present after attempting to delete duplicates", - Database::duplicateUncasedName(db).empty()); + dbHolder().getNamesWithConflictingCasing(db).empty()); return true; } void ReplSource::applyCommand(OperationContext* txn, const BSONObj& op) { try { - Status status = applyCommand_inlock(txn, op); + Status status = applyCommand_inlock(txn, op, true); if (!status.isOK()) { SyncTail sync(nullptr, SyncTail::MultiSyncApplyFunc()); sync.setHostname(hostName); if (sync.shouldRetry(txn, op)) { uassert(28639, "Failure retrying initial sync update", - applyCommand_inlock(txn, op).isOK()); + applyCommand_inlock(txn, op, true).isOK()); } } } catch (UserException& e) { diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index d98db32fd43..63858050014 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -644,7 +644,8 @@ std::map opsMap = { {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { BSONObjBuilder resultWeDontCareAbout; return collMod(txn, parseNs(ns, cmd), cmd, &resultWeDontCareAbout); - }}}, + }, + {ErrorCodes::IndexNotFound, ErrorCodes::NamespaceNotFound}}}, {"dropDatabase", {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { return dropDatabase(txn, NamespaceString(ns).db().toString()); }, @@ -712,7 +713,7 @@ std::map opsMap = { Status applyOperation_inlock(OperationContext* txn, Database* db, const BSONObj& op, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { LOG(3) << "applying op: " << op << endl; OpCounters* opCounters = txn->writesAreReplicated() ? &globalOpCounters : &replOpCounters; @@ -794,6 +795,7 @@ Status applyOperation_inlock(OperationContext* txn, // Wait for thread to start and register itself IndexBuilder::waitForBgIndexStarting(); } + txn->recoveryUnit()->abandonSnapshot(); } else { IndexBuilder builder(o); Status status = builder.buildInForeground(txn, db); @@ -866,7 +868,7 @@ Status applyOperation_inlock(OperationContext* txn, OpDebug debug; BSONObj updateCriteria = o2; - const bool upsert = valueB || convertUpdateToUpsert; + const bool upsert = valueB || inSteadyStateReplication; uassert(ErrorCodes::NoSuchKey, str::stream() << "Failed to apply update due to missing _id: " << op.toString(), @@ -949,7 +951,9 @@ Status applyOperation_inlock(OperationContext* txn, return Status::OK(); } -Status applyCommand_inlock(OperationContext* txn, const BSONObj& op) { +Status applyCommand_inlock(OperationContext* txn, + const BSONObj& op, + bool inSteadyStateReplication) { const char* names[] = {"o", "ns", "op"}; BSONElement fields[3]; op.getFields(3, names, fields); @@ -972,6 +976,14 @@ Status applyCommand_inlock(OperationContext* txn, const BSONObj& op) { const char* ns = fieldNs.valuestrsafe(); + // Applying renameCollection during initial sync might lead to data corruption, so we restart + // the initial sync. + if (!inSteadyStateReplication && o.firstElementFieldName() == std::string("renameCollection")) { + return Status(ErrorCodes::OplogOperationUnsupported, + str::stream() + << "Applying renameCollection not supported in initial sync: " << op); + } + // Applying commands in repl is done under Global W-lock, so it is safe to not // perform the current DB checks after reacquiring the lock. invariant(txn->lockState()->isW()); @@ -1190,8 +1202,7 @@ void SnapshotThread::run() { invariant(!opTimeOfSnapshot.isNull()); } - _manager->createSnapshot(txn.get(), name); - replCoord->onSnapshotCreate(opTimeOfSnapshot, name); + replCoord->createSnapshot(txn.get(), opTimeOfSnapshot, name); } catch (const WriteConflictException& wce) { log() << "skipping storage snapshot pass due to write conflict"; continue; diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h index b022aa0e545..e3251102fef 100644 --- a/src/mongo/db/repl/oplog.h +++ b/src/mongo/db/repl/oplog.h @@ -128,20 +128,22 @@ void oplogCheckCloseDatabase(OperationContext* txn, Database* db); /** * Take a non-command op and apply it locally * Used for applying from an oplog - * @param convertUpdateToUpsert convert some updates to upserts for idempotency reasons + * @param inSteadyStateReplication convert some updates to upserts for idempotency reasons * Returns failure status if the op was an update that could not be applied. */ Status applyOperation_inlock(OperationContext* txn, Database* db, const BSONObj& op, - bool convertUpdateToUpsert = false); + bool inSteadyStateReplication = false); /** * Take a command op and apply it locally * Used for applying from an oplog + * inSteadyStateReplication indicates whether we are in steady state replication, rather than + * initial sync. * Returns failure status if the op that could not be applied. */ -Status applyCommand_inlock(OperationContext* txn, const BSONObj& op); +Status applyCommand_inlock(OperationContext* txn, const BSONObj& op, bool inSteadyStateReplication); /** * Initializes the global Timestamp with the value from the timestamp of the last oplog entry. diff --git a/src/mongo/db/repl/oplogreader.cpp b/src/mongo/db/repl/oplogreader.cpp index 63832b201af..f00b95e147a 100644 --- a/src/mongo/db/repl/oplogreader.cpp +++ b/src/mongo/db/repl/oplogreader.cpp @@ -165,7 +165,8 @@ Status OplogReader::_compareRequiredOpTimeWithQueryResponse(const OpTime& requir void OplogReader::connectToSyncSource(OperationContext* txn, const OpTime& lastOpTimeFetched, const OpTime& requiredOpTime, - ReplicationCoordinator* replCoord) { + ReplicationCoordinator* replCoord, + int* rbidOut) { const Timestamp sentinelTimestamp(duration_cast(Milliseconds(curTimeMillis64())), 0); const OpTime sentinel(sentinelTimestamp, std::numeric_limits::max()); OpTime oldestOpTimeSeen = sentinel; @@ -209,6 +210,18 @@ void OplogReader::connectToSyncSource(OperationContext* txn, replCoord->blacklistSyncSource(candidate, Date_t::now() + Seconds(10)); continue; } + + if (rbidOut) { + BSONObj reply; + if (!conn()->runCommand("admin", BSON("replSetGetRBID" << 1), reply)) { + log() << "Error getting rbid from " << candidate << ": " << reply; + resetConnection(); + replCoord->blacklistSyncSource(candidate, Date_t::now() + Seconds(10)); + continue; + } + *rbidOut = reply["rbid"].Int(); + } + // Read the first (oldest) op and confirm that it's not newer than our last // fetched op. Otherwise, we have fallen off the back of that source's oplog. BSONObj remoteOldestOp(findOne(rsOplogName.c_str(), Query())); diff --git a/src/mongo/db/repl/oplogreader.h b/src/mongo/db/repl/oplogreader.h index 1434125697a..3c983b007fc 100644 --- a/src/mongo/db/repl/oplogreader.h +++ b/src/mongo/db/repl/oplogreader.h @@ -153,12 +153,17 @@ public: * is left unconnected, where this->conn() equals NULL. * In the process of connecting, this function may add items to the repl coordinator's * sync source blacklist. + * + * If the rbidOut param is non-null it will be set to the rbid of the server before any data is + * fetched from it. + * * This function may throw DB exceptions. */ void connectToSyncSource(OperationContext* txn, const OpTime& lastOpTimeFetched, const OpTime& requiredOpTime, - ReplicationCoordinator* replCoord); + ReplicationCoordinator* replCoord, + int* rbidOut = nullptr); private: /** diff --git a/src/mongo/db/repl/repl_set_request_votes_args.cpp b/src/mongo/db/repl/repl_set_request_votes_args.cpp index 0041e78c4b2..3d55af3f93c 100644 --- a/src/mongo/db/repl/repl_set_request_votes_args.cpp +++ b/src/mongo/db/repl/repl_set_request_votes_args.cpp @@ -40,7 +40,9 @@ const std::string kCandidateIndexFieldName = "candidateIndex"; const std::string kCommandName = "replSetRequestVotes"; const std::string kConfigVersionFieldName = "configVersion"; const std::string kDryRunFieldName = "dryRun"; -const std::string kLastCommittedOpFieldName = "lastCommittedOp"; +// The underlying field name is inaccurate, but changing it requires a fair amount of cross +// compatibility work for no real benefit. +const std::string kLastDurableOpTimeFieldName = "lastCommittedOp"; const std::string kOkFieldName = "ok"; const std::string kReasonFieldName = "reason"; const std::string kSetNameFieldName = "setName"; @@ -52,7 +54,7 @@ const std::string kLegalArgsFieldNames[] = { kCommandName, kConfigVersionFieldName, kDryRunFieldName, - kLastCommittedOpFieldName, + kLastDurableOpTimeFieldName, kSetNameFieldName, kTermFieldName, }; @@ -89,7 +91,7 @@ Status ReplSetRequestVotesArgs::initialize(const BSONObj& argsObj) { if (!status.isOK()) return status; - status = bsonExtractOpTimeField(argsObj, kLastCommittedOpFieldName, &_lastCommittedOp); + status = bsonExtractOpTimeField(argsObj, kLastDurableOpTimeFieldName, &_lastDurableOpTime); if (!status.isOK()) return status; @@ -112,8 +114,8 @@ long long ReplSetRequestVotesArgs::getConfigVersion() const { return _cfgver; } -OpTime ReplSetRequestVotesArgs::getLastCommittedOp() const { - return _lastCommittedOp; +OpTime ReplSetRequestVotesArgs::getLastDurableOpTime() const { + return _lastDurableOpTime; } bool ReplSetRequestVotesArgs::isADryRun() const { @@ -127,7 +129,7 @@ void ReplSetRequestVotesArgs::addToBSON(BSONObjBuilder* builder) const { builder->append(kTermFieldName, _term); builder->appendIntOrLL(kCandidateIndexFieldName, _candidateIndex); builder->appendIntOrLL(kConfigVersionFieldName, _cfgver); - _lastCommittedOp.append(builder, kLastCommittedOpFieldName); + _lastDurableOpTime.append(builder, kLastDurableOpTimeFieldName); } Status ReplSetRequestVotesResponse::initialize(const BSONObj& argsObj) { diff --git a/src/mongo/db/repl/repl_set_request_votes_args.h b/src/mongo/db/repl/repl_set_request_votes_args.h index 12541650798..7f1cf3bc72e 100644 --- a/src/mongo/db/repl/repl_set_request_votes_args.h +++ b/src/mongo/db/repl/repl_set_request_votes_args.h @@ -46,7 +46,7 @@ public: long long getTerm() const; long long getCandidateIndex() const; long long getConfigVersion() const; - OpTime getLastCommittedOp() const; + OpTime getLastDurableOpTime() const; bool isADryRun() const; void addToBSON(BSONObjBuilder* builder) const; @@ -56,9 +56,9 @@ private: long long _term = -1; // Current known term of the command issuer. // replSet config index of the member who sent the replSetRequestVotesCmd. long long _candidateIndex = -1; - long long _cfgver = -1; // replSet config version known to the command issuer. - OpTime _lastCommittedOp; // The last known committed op of the command issuer. - bool _dryRun = false; // Indicates this is a pre-election check when true. + long long _cfgver = -1; // replSet config version known to the command issuer. + OpTime _lastDurableOpTime; // The last known durable op of the command issuer. + bool _dryRun = false; // Indicates this is a pre-election check when true. }; class ReplSetRequestVotesResponse { diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h index dd55fcca799..d7eba5a493c 100644 --- a/src/mongo/db/repl/replication_coordinator.h +++ b/src/mongo/db/repl/replication_coordinator.h @@ -466,14 +466,13 @@ public: virtual void processReplSetGetConfig(BSONObjBuilder* result) = 0; /** - * Processes the ReplSetMetadata returned from a command run against another replica set - * member and updates protocol version 1 information (most recent optime that is committed, - * member id of the current PRIMARY, the current config version and the current term). - * - * TODO(dannenberg): Move this method to be testing only if it does not end up being used - * to process the find and getmore metadata responses from the DataReplicator. + * Processes the ReplSetMetadata returned from a command run against another + * replica set member and so long as the config version in the metadata matches the replica set + * config version this node currently has, updates the current term and optionally updates + * this node's notion of the commit point. */ - virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) = 0; + virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) = 0; /** * Elections under protocol version 1 are triggered by a timer. @@ -685,9 +684,8 @@ public: /** * Prepares a metadata object describing the current term, primary, and lastOp information. */ - virtual void prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) = 0; + virtual void prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const = 0; /** * Returns true if the V1 election protocol is being used and false otherwise. @@ -745,9 +743,12 @@ public: virtual void forceSnapshotCreation() = 0; /** - * Called when a new snapshot is created. + * Creates a new snapshot in the storage engine and registers it for use in the replication + * coordinator. */ - virtual void onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) = 0; + virtual void createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) = 0; /** * Blocks until either the current committed snapshot is at least as high as 'untilSnapshot', @@ -765,7 +766,7 @@ public: /** * Gets the latest OpTime of the currentCommittedSnapshot. */ - virtual OpTime getCurrentCommittedSnapshotOpTime() = 0; + virtual OpTime getCurrentCommittedSnapshotOpTime() const = 0; /** * Appends connection information to the provided BSONObjBuilder. diff --git a/src/mongo/db/repl/replication_coordinator_external_state.h b/src/mongo/db/repl/replication_coordinator_external_state.h index b595f606a61..de8ee0011e1 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state.h +++ b/src/mongo/db/repl/replication_coordinator_external_state.h @@ -233,6 +233,11 @@ public: */ virtual void updateCommittedSnapshot(SnapshotName newCommitPoint) = 0; + /** + * Creates a new snapshot. + */ + virtual void createSnapshot(OperationContext* txn, SnapshotName name) = 0; + /** * Signals the SnapshotThread, if running, to take a forced snapshot even if the global * timestamp hasn't changed. diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 128e18b9f53..f0e4cb4e0ee 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -297,9 +297,7 @@ StatusWith ReplicationCoordinatorExternalStateImpl::loadLocalLastVoteD << "Did not find replica set lastVote document in " << lastVoteCollectionName); } - LastVote lastVote; - lastVote.initialize(lastVoteObj); - return StatusWith(lastVote); + return LastVote::readFromLastVote(lastVoteObj); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END( txn, "load replica set lastVote", lastVoteCollectionName); @@ -315,12 +313,29 @@ Status ReplicationCoordinatorExternalStateImpl::storeLocalLastVoteDocument( MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { ScopedTransaction transaction(txn, MODE_IX); Lock::DBLock dbWriteLock(txn->lockState(), lastVoteDatabaseName, MODE_X); - Helpers::putSingleton(txn, lastVoteCollectionName, lastVoteObj); - return Status::OK(); + + // If there is no last vote document, we want to store one. Otherwise, we only want to + // replace it if the new last vote document would have a higher term. We both check + // the term of the current last vote document and insert the new document under the + // DBLock to synchronize the two operations. + BSONObj result; + bool exists = Helpers::getSingleton(txn, lastVoteCollectionName, result); + if (!exists) { + Helpers::putSingleton(txn, lastVoteCollectionName, lastVoteObj); + } else { + StatusWith oldLastVoteDoc = LastVote::readFromLastVote(result); + if (!oldLastVoteDoc.isOK()) { + return oldLastVoteDoc.getStatus(); + } + if (lastVote.getTerm() > oldLastVoteDoc.getValue().getTerm()) { + Helpers::putSingleton(txn, lastVoteCollectionName, lastVoteObj); + } + } } MONGO_WRITE_CONFLICT_RETRY_LOOP_END( txn, "save replica set lastVote", lastVoteCollectionName); - MONGO_UNREACHABLE; + txn->recoveryUnit()->waitUntilDurable(); + return Status::OK(); } catch (const DBException& ex) { return ex.toStatus(); } @@ -533,6 +548,13 @@ void ReplicationCoordinatorExternalStateImpl::updateCommittedSnapshot(SnapshotNa manager->setCommittedSnapshot(newCommitPoint); } +void ReplicationCoordinatorExternalStateImpl::createSnapshot(OperationContext* txn, + SnapshotName name) { + auto manager = getGlobalServiceContext()->getGlobalStorageEngine()->getSnapshotManager(); + invariant(manager); // This should never be called if there is no SnapshotManager. + manager->createSnapshot(txn, name); +} + void ReplicationCoordinatorExternalStateImpl::forceSnapshotCreation() { if (_snapshotThread) _snapshotThread->forceSnapshot(); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.h b/src/mongo/db/repl/replication_coordinator_external_state_impl.h index 26667fbba96..fb3135b8592 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.h @@ -79,6 +79,7 @@ public: virtual void dropAllTempCollections(OperationContext* txn); void dropAllSnapshots() final; void updateCommittedSnapshot(SnapshotName newCommitPoint) final; + void createSnapshot(OperationContext* txn, SnapshotName name) final; void forceSnapshotCreation() final; virtual bool snapshotsEnabled() const; virtual void notifyOplogMetadataWaiters(); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp index 2918d8cdcad..a05a7e7ae06 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp @@ -216,6 +216,9 @@ void ReplicationCoordinatorExternalStateMock::dropAllSnapshots() {} void ReplicationCoordinatorExternalStateMock::updateCommittedSnapshot(SnapshotName newCommitPoint) { } +void ReplicationCoordinatorExternalStateMock::createSnapshot(OperationContext* txn, + SnapshotName name) {} + void ReplicationCoordinatorExternalStateMock::forceSnapshotCreation() {} bool ReplicationCoordinatorExternalStateMock::snapshotsEnabled() const { diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.h b/src/mongo/db/repl/replication_coordinator_external_state_mock.h index 929772f5820..1434fc34bab 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.h @@ -80,6 +80,7 @@ public: virtual void dropAllTempCollections(OperationContext* txn); virtual void dropAllSnapshots(); virtual void updateCommittedSnapshot(SnapshotName newCommitPoint); + virtual void createSnapshot(OperationContext* txn, SnapshotName name); virtual void forceSnapshotCreation(); virtual bool snapshotsEnabled() const; virtual void notifyOplogMetadataWaiters(); diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 5dcf7b9b1c3..88e9310d141 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -329,7 +329,7 @@ Date_t ReplicationCoordinatorImpl::getPriorityTakeover_forTest() const { return _priorityTakeoverWhen; } -OpTime ReplicationCoordinatorImpl::getCurrentCommittedSnapshotOpTime() { +OpTime ReplicationCoordinatorImpl::getCurrentCommittedSnapshotOpTime() const { stdx::lock_guard lk(_mutex); if (_currentCommittedSnapshot) { return _currentCommittedSnapshot->opTime; @@ -348,7 +348,13 @@ void ReplicationCoordinatorImpl::_updateLastVote(const LastVote& lastVote) { bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* txn) { StatusWith lastVote = _externalState->loadLocalLastVoteDocument(txn); if (!lastVote.isOK()) { - log() << "Did not find local voted for document at startup; " << lastVote.getStatus(); + if (lastVote.getStatus() == ErrorCodes::NoMatchingDocument) { + log() << "Did not find local voted for document at startup."; + } else { + severe() << "Error loading local voted for document at startup; " + << lastVote.getStatus(); + fassertFailedNoTrace(40367); + } } else { LastVote vote = lastVote.getValue(); _replExecutor.scheduleWork( @@ -1439,10 +1445,38 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::_awaitRepl return StatusAndDuration(Status::OK(), Milliseconds(timer->millis())); } - if (replMode == modeReplSet && !_memberState.primary()) { - return StatusAndDuration( - Status(ErrorCodes::NotMaster, "Not master while waiting for replication"), - Milliseconds(timer->millis())); + auto checkForStepDown = [&]() -> Status { + if (replMode == modeReplSet && !_memberState.primary()) { + return {ErrorCodes::NotMaster, "Primary stepped down while waiting for replication"}; + } + + // Relax term checking under 3.2 because some commands (eg. createIndexes) might not return + // a term in the response metadata to mongos which may pass the no-term OpTime back to + // mongod eventually. + if (opTime.getTerm() != OpTime::kUninitializedTerm && + _cachedTerm != OpTime::kUninitializedTerm && opTime.getTerm() != _cachedTerm) { + return { + ErrorCodes::NotMaster, + str::stream() << "Term changed from " << opTime.getTerm() << " to " << _cachedTerm + << " while waiting for replication, indicating that this node must " + "have stepped down."}; + } + + if (_stepDownPending) { + return {ErrorCodes::NotMaster, + "Received stepdown request while waiting for replication"}; + } + return Status::OK(); + }; + + Status stepdownStatus = checkForStepDown(); + if (!stepdownStatus.isOK()) { + return StatusAndDuration(stepdownStatus, Milliseconds(timer->millis())); + } + + auto interruptStatus = txn->checkForInterruptNoAssert(); + if (!interruptStatus.isOK()) { + return StatusAndDuration(interruptStatus, Milliseconds(timer->millis())); } if (writeConcern.wMode.empty()) { @@ -1464,6 +1498,7 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::_awaitRepl return StatusAndDuration(interruptedStatus, elapsed); } + if (!waitInfo.master) { return StatusAndDuration(Status(ErrorCodes::NotMaster, "Not master anymore while waiting for replication" @@ -1500,6 +1535,11 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::_awaitRepl } else { condVar.wait_for(*lock, waitTime); } + + stepdownStatus = checkForStepDown(); + if (!stepdownStatus.isOK()) { + return StatusAndDuration(stepdownStatus, elapsed); + } } Status status = _checkIfWriteConcernCanBeSatisfied_inlock(writeConcern); @@ -1638,7 +1678,8 @@ void ReplicationCoordinatorImpl::_stepDownContinue( return; } bool forceNow = now >= waitUntil ? force : false; - if (_topCoord->stepDown(stepDownUntil, forceNow, getMyLastAppliedOpTime())) { + if (_topCoord->stepDown( + stepDownUntil, forceNow, getMyLastAppliedOpTime(), getLastCommittedOpTime())) { // Schedule work to (potentially) step back up once the stepdown period has ended. _replExecutor.scheduleWorkAt(stepDownUntil, stdx::bind(&ReplicationCoordinatorImpl::_handleTimePassing, @@ -1845,32 +1886,35 @@ int ReplicationCoordinatorImpl::_getMyId_inlock() const { } bool ReplicationCoordinatorImpl::prepareReplSetUpdatePositionCommand(BSONObjBuilder* cmdBuilder) { - stdx::lock_guard lock(_mutex); - invariant(_rsConfig.isInitialized()); - // Do not send updates if we have been removed from the config. - if (_selfIndex == -1) { - return false; - } - cmdBuilder->append("replSetUpdatePosition", 1); - // Create an array containing objects each live member connected to us and for ourself. - BSONArrayBuilder arrayBuilder(cmdBuilder->subarrayStart("optimes")); - for (SlaveInfoVector::iterator itr = _slaveInfo.begin(); itr != _slaveInfo.end(); ++itr) { - if (itr->lastAppliedOpTime.isNull()) { - // Don't include info on members we haven't heard from yet. - continue; - } - // Don't include members we think are down. - if (!itr->self && itr->down) { - continue; + { + stdx::lock_guard lock(_mutex); + invariant(_rsConfig.isInitialized()); + // Do not send updates if we have been removed from the config. + if (_selfIndex == -1) { + return false; } + cmdBuilder->append("replSetUpdatePosition", 1); + // Create an array containing objects each live member connected to us and for ourself. + BSONArrayBuilder arrayBuilder(cmdBuilder->subarrayStart("optimes")); + for (SlaveInfoVector::iterator itr = _slaveInfo.begin(); itr != _slaveInfo.end(); ++itr) { + if (itr->lastAppliedOpTime.isNull()) { + // Don't include info on members we haven't heard from yet. + continue; + } + // Don't include members we think are down. + if (!itr->self && itr->down) { + continue; + } - BSONObjBuilder entry(arrayBuilder.subobjStart()); - itr->lastDurableOpTime.append(&entry, "durableOpTime"); - itr->lastAppliedOpTime.append(&entry, "appliedOpTime"); - entry.append("memberId", itr->memberId); - entry.append("cfgver", _rsConfig.getConfigVersion()); + BSONObjBuilder entry(arrayBuilder.subobjStart()); + itr->lastDurableOpTime.append(&entry, "durableOpTime"); + itr->lastAppliedOpTime.append(&entry, "appliedOpTime"); + entry.append("memberId", itr->memberId); + entry.append("cfgver", _rsConfig.getConfigVersion()); + } } - + // Add metadata to command. Old style parsing logic will reject the metadata. + prepareReplMetadata(OpTime(), cmdBuilder); return true; } @@ -1991,10 +2035,11 @@ void ReplicationCoordinatorImpl::processReplSetGetConfig(BSONObjBuilder* result) result->append("config", _rsConfig.toBSON()); } -void ReplicationCoordinatorImpl::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) { +void ReplicationCoordinatorImpl::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) { EventHandle evh; - _scheduleWorkAndWaitForCompletion([this, &evh, &replMetadata](const CallbackArgs& args) { - evh = _processReplSetMetadata_incallback(replMetadata); + _scheduleWorkAndWaitForCompletion([&](const CallbackArgs& args) { + evh = _processReplSetMetadata_incallback(replMetadata, advanceCommitPoint); }); if (evh.isValid()) { _replExecutor.waitForEvent(evh); @@ -2007,11 +2052,13 @@ void ReplicationCoordinatorImpl::cancelAndRescheduleElectionTimeout() { } EventHandle ReplicationCoordinatorImpl::_processReplSetMetadata_incallback( - const rpc::ReplSetMetadata& replMetadata) { + const rpc::ReplSetMetadata& replMetadata, bool advanceCommitPoint) { if (replMetadata.getConfigVersion() != _rsConfig.getConfigVersion()) { return EventHandle(); } - _setLastCommittedOpTime(replMetadata.getLastOpCommitted()); + if (advanceCommitPoint) { + _setLastCommittedOpTime(replMetadata.getLastOpCommitted()); + } return _updateTerm_incallback(replMetadata.getTerm()); } @@ -2147,6 +2194,16 @@ void ReplicationCoordinatorImpl::_processReplSetFreeze_finish( return; } + if (_topCoord->getRole() != TopologyCoordinator::Role::follower) { + *result = Status(ErrorCodes::NotSecondary, + str::stream() + << "cannot freeze node when primary or running for election. state: " + << (_topCoord->getRole() == TopologyCoordinator::Role::leader + ? "Primary" + : "Running-Election")); + return; + } + _topCoord->prepareFreezeResponse(_replExecutor.now(), secs, response); if (_topCoord->getRole() == TopologyCoordinator::Role::candidate) { @@ -2321,12 +2378,14 @@ Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* txn, return status; } + auto finishEvh = uassertStatusOK(_replExecutor.makeEvent()); const stdx::function reconfigFinishFn( stdx::bind(&ReplicationCoordinatorImpl::_finishReplSetReconfig, this, stdx::placeholders::_1, newConfig, - myIndex.getValue())); + myIndex.getValue(), + finishEvh)); // If it's a force reconfig, the primary node may not be electable after the configuration // change. In case we are that primary node, finish the reconfig under the global lock, @@ -2338,14 +2397,15 @@ Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* txn, } fassert(18824, cbh.getStatus()); configStateGuard.Dismiss(); - _replExecutor.wait(cbh.getValue()); + _replExecutor.waitForEvent(finishEvh); return Status::OK(); } void ReplicationCoordinatorImpl::_finishReplSetReconfig( const ReplicationExecutor::CallbackArgs& cbData, const ReplicaSetConfig& newConfig, - int myIndex) { + int myIndex, + ReplicationExecutor::EventHandle finishedEvent) { stdx::unique_lock lk(_mutex); invariant(_rsConfigState == kConfigReconfiguring); invariant(_rsConfig.isInitialized()); @@ -2353,8 +2413,18 @@ void ReplicationCoordinatorImpl::_finishReplSetReconfig( const PostMemberStateUpdateAction action = _setCurrentRSConfig_inlock(cbData, newConfig, myIndex); lk.unlock(); - _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); - _performPostMemberStateUpdateAction(action); + auto evh = _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); + if (evh) { + _replExecutor.onEvent( + evh, + [this, action, finishedEvent](const ReplicationExecutor::CallbackArgs& cbArgs) { + _performPostMemberStateUpdateAction(action); + _replExecutor.signalEvent(finishedEvent); + }); + } else { + _performPostMemberStateUpdateAction(action); + _replExecutor.signalEvent(finishedEvent); + } } Status ReplicationCoordinatorImpl::processReplSetInitiate(OperationContext* txn, @@ -2475,11 +2545,8 @@ void ReplicationCoordinatorImpl::_finishReplSetInitiate( stdx::unique_lock lk(_mutex); invariant(_rsConfigState == kConfigInitiating); invariant(!_rsConfig.isInitialized()); - const ReplicaSetConfig oldConfig = _rsConfig; - const PostMemberStateUpdateAction action = - _setCurrentRSConfig_inlock(cbData, newConfig, myIndex); + auto action = _setCurrentRSConfig_inlock(cbData, newConfig, myIndex); lk.unlock(); - _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); _performPostMemberStateUpdateAction(action); } @@ -2517,6 +2584,7 @@ ReplicationCoordinatorImpl::_updateMemberStateFromTopologyCoordinator_inlock() { info->condVar->notify_all(); } _canAcceptNonLocalWrites = false; + _stepDownPending = false; result = kActionCloseAllConnections; } else { result = kActionFollowerModeStateChange; @@ -3093,7 +3161,7 @@ bool ReplicationCoordinatorImpl::shouldChangeSyncSource(const HostAndPort& curre } void ReplicationCoordinatorImpl::_updateLastCommittedOpTime_inlock() { - if (!_getMemberState_inlock().primary()) { + if (!_getMemberState_inlock().primary() || _stepDownPending) { return; } @@ -3200,10 +3268,8 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes( return cbh.getStatus(); } _replExecutor.wait(cbh.getValue()); - if (response->getVoteGranted()) { - LastVote lastVote; - lastVote.setTerm(args.getTerm()); - lastVote.setCandidateIndex(args.getCandidateIndex()); + if (!args.isADryRun() && response->getVoteGranted()) { + LastVote lastVote{args.getTerm(), args.getCandidateIndex()}; Status status = _externalState->storeLocalLastVoteDocument(txn, lastVote); if (!status.isOK()) { @@ -3266,37 +3332,35 @@ void ReplicationCoordinatorImpl::_processReplSetDeclareElectionWinner_finish( *result = _topCoord->processReplSetDeclareElectionWinner(args, responseTerm); } -void ReplicationCoordinatorImpl::prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) { - if (request.getMetadata().hasField(rpc::kReplSetMetadataFieldName)) { - rpc::ReplSetMetadata metadata; - CBHStatus cbh = _replExecutor.scheduleWork( - stdx::bind(&ReplicationCoordinatorImpl::_prepareReplResponseMetadata_finish, - this, - stdx::placeholders::_1, - lastOpTimeFromClient, - &metadata)); +void ReplicationCoordinatorImpl::prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const { + rpc::ReplSetMetadata metadata; - if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) { - return; - } - - fassert(28709, cbh.getStatus()); - _replExecutor.wait(cbh.getValue()); + CBHStatus cbh = _replExecutor.scheduleWork( + stdx::bind(&ReplicationCoordinatorImpl::_prepareReplResponseMetadata_finish, + this, + stdx::placeholders::_1, + lastOpTimeFromClient, + &metadata)); - metadata.writeToMetadata(builder); + if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) { + return; } + + fassert(28709, cbh.getStatus()); + _replExecutor.wait(cbh.getValue()); + + metadata.writeToMetadata(builder); } void ReplicationCoordinatorImpl::_prepareReplResponseMetadata_finish( const ReplicationExecutor::CallbackArgs& cbData, const OpTime& lastOpTimeFromClient, - rpc::ReplSetMetadata* metadata) { + rpc::ReplSetMetadata* metadata) const { OpTime lastReadableOpTime = getCurrentCommittedSnapshotOpTime(); OpTime lastVisibleOpTime = std::max(lastOpTimeFromClient, lastReadableOpTime); - _topCoord->prepareReplResponseMetadata(metadata, lastVisibleOpTime, _lastCommittedOpTime); + _topCoord->prepareReplMetadata(metadata, lastVisibleOpTime, _lastCommittedOpTime); } bool ReplicationCoordinatorImpl::isV1ElectionProtocol() { @@ -3503,7 +3567,7 @@ EventHandle ReplicationCoordinatorImpl::_updateTerm_incallback( if (localUpdateTermResult == TopologyCoordinator::UpdateTermResult::kTriggerStepDown) { log() << "stepping down from primary, because a new term has begun: " << term; _topCoord->prepareForStepDown(); - return _stepDownStart(); + return _stepDownStart(false); } return EventHandle(); } @@ -3541,9 +3605,11 @@ size_t ReplicationCoordinatorImpl::getNumUncommittedSnapshots() { return _uncommittedSnapshotsSize.load(); } -void ReplicationCoordinatorImpl::onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) { +void ReplicationCoordinatorImpl::createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) { stdx::lock_guard lock(_mutex); - + _externalState->createSnapshot(txn, name); auto snapshotInfo = SnapshotInfo{timeOfSnapshot, name}; if (timeOfSnapshot <= _lastCommittedOpTime) { @@ -3609,35 +3675,40 @@ void ReplicationCoordinatorImpl::waitForElectionDryRunFinish_forTest() { } } -void ReplicationCoordinatorImpl::_resetElectionInfoOnProtocolVersionUpgrade( +EventHandle ReplicationCoordinatorImpl::_resetElectionInfoOnProtocolVersionUpgrade( const ReplicaSetConfig& oldConfig, const ReplicaSetConfig& newConfig) { // On protocol version upgrade, reset last vote as if I just learned the term 0 from other // nodes. if (!oldConfig.isInitialized() || oldConfig.getProtocolVersion() >= newConfig.getProtocolVersion()) { - return; + return {}; } invariant(newConfig.getProtocolVersion() == 1); // Write last vote - auto cbStatus = _replExecutor.scheduleDBWork([this](const CallbackArgs& cbData) { + auto evhStatus = _replExecutor.makeEvent(); + if (evhStatus.getStatus() == ErrorCodes::ShutdownInProgress) { + return {}; + } + invariant(evhStatus.isOK()); + auto evh = evhStatus.getValue(); + + auto cbStatus = _replExecutor.scheduleDBWork([this, evh](const CallbackArgs& cbData) { if (cbData.status == ErrorCodes::CallbackCanceled) { return; } invariant(cbData.txn); - LastVote lastVote; - lastVote.setTerm(OpTime::kInitialTerm); - lastVote.setCandidateIndex(-1); + LastVote lastVote{OpTime::kInitialTerm, -1}; auto status = _externalState->storeLocalLastVoteDocument(cbData.txn, lastVote); invariant(status.isOK()); + _replExecutor.signalEvent(evh); }); if (cbStatus.getStatus() == ErrorCodes::ShutdownInProgress) { - return; + return {}; } - invariant(cbStatus.isOK()); - _replExecutor.wait(cbStatus.getValue()); + return evh; } CallbackHandle ReplicationCoordinatorImpl::_scheduleWork(const CallbackFn& work) { diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index 33b93a70027..5f22864311a 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -217,7 +217,8 @@ public: virtual void processReplSetGetConfig(BSONObjBuilder* result) override; - virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) override; + virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) override; virtual void cancelAndRescheduleElectionTimeout() override; @@ -290,9 +291,8 @@ public: virtual Status processReplSetDeclareElectionWinner(const ReplSetDeclareElectionWinnerArgs& args, long long* responseTerm) override; - void prepareReplResponseMetadata(const rpc::RequestInterface&, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) override; + virtual void prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const override; virtual Status processHeartbeatV1(const ReplSetHeartbeatArgsV1& args, ReplSetHeartbeatResponse* response) override; @@ -315,9 +315,11 @@ public: virtual void forceSnapshotCreation() override; - virtual void onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) override; + virtual void createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) override; - virtual OpTime getCurrentCommittedSnapshotOpTime() override; + virtual OpTime getCurrentCommittedSnapshotOpTime() const override; virtual void waitUntilSnapshotCommitted(OperationContext* txn, const SnapshotName& untilSnapshot) override; @@ -663,11 +665,11 @@ private: Status* result); /** - * Bottom half of prepareReplResponseMetadata. + * Bottom half of prepareReplMetadata. */ void _prepareReplResponseMetadata_finish(const ReplicationExecutor::CallbackArgs& cbData, const OpTime& lastOpTimeFromClient, - rpc::ReplSetMetadata* metadata); + rpc::ReplSetMetadata* metadata) const; /** * Scheduled to cause the ReplicationCoordinator to reconsider any state that might * need to change as a result of time passing - for instance becoming PRIMARY when a single @@ -891,7 +893,8 @@ private: */ void _finishReplSetReconfig(const ReplicationExecutor::CallbackArgs& cbData, const ReplicaSetConfig& newConfig, - int myIndex); + int myIndex, + ReplicationExecutor::EventHandle finishedEvent); /** * Changes _rsConfigState to newState, and notify any waiters. @@ -1025,7 +1028,10 @@ private: */ void _requestRemotePrimaryStepdown(const HostAndPort& target); - ReplicationExecutor::EventHandle _stepDownStart(); + /** + * Schedules stepdown to run with the global exclusive lock. + */ + ReplicationExecutor::EventHandle _stepDownStart(bool hasMutex); /** * Completes a step-down of the current node. Must be run with a global @@ -1064,9 +1070,11 @@ private: * Utility method that schedules or performs actions specified by a HeartbeatResponseAction * returned by a TopologyCoordinator::processHeartbeatResponse(V1) call with the given * value of "responseStatus". + * 'hasMutex' is true if the caller is holding _mutex. TODO(SERVER-27083): Remove this. */ void _handleHeartbeatResponseAction(const HeartbeatResponseAction& action, - const StatusWith& responseStatus); + const StatusWith& responseStatus, + bool hasMutex); /** * Bottom half of processHeartbeat(), which runs in the replication executor. @@ -1116,11 +1124,13 @@ private: /** * Callback that processes the ReplSetMetadata returned from a command run against another - * replica set member and updates protocol version 1 information (most recent optime that is - * committed, member id of the current PRIMARY, the current config version and the current term) + * replica set member and so long as the config version in the metadata matches the replica set + * config version this node currently has, updates the current term and optionally updates + * this node's notion of the commit point. * Returns the finish event which is invalid if the process has already finished. */ - EventHandle _processReplSetMetadata_incallback(const rpc::ReplSetMetadata& replMetadata); + EventHandle _processReplSetMetadata_incallback(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint); /** * Blesses a snapshot to be used for new committed reads. @@ -1176,11 +1186,11 @@ private: void _startElectSelfIfEligibleV1(bool isPriorityTakeover); /** - * Reset the term of last vote to 0 to prevent any node from voting for term 0. - * Blocking until last vote write finishes. Must be called without holding _mutex. + * Resets the term of last vote to 0 to prevent any node from voting for term 0. + * Returns the event handle that indicates when last vote write finishes. */ - void _resetElectionInfoOnProtocolVersionUpgrade(const ReplicaSetConfig& oldConfig, - const ReplicaSetConfig& newConfig); + EventHandle _resetElectionInfoOnProtocolVersionUpgrade(const ReplicaSetConfig& oldConfig, + const ReplicaSetConfig& newConfig); /** * Schedules work and returns handle to callback. @@ -1282,6 +1292,14 @@ private: // TODO: ideally this should only change on rollbacks NOT on mongod restarts also. int _rbid; // (M) + // Indicates that we've received a request to stepdown from PRIMARY (likely via a heartbeat) + // TODO(SERVER-27083): This bool is redundant of the same-named bool in TopologyCoordinatorImpl, + // but due to mutex ordering between _mutex and _topoMutex we can't inspect the + // TopologyCoordinator field in awaitReplication() where this bool is used. Once we get rid + // of topoMutex and start guarding access to the TopologyCoordinator via _mutex we should + // consolidate the two bools. + bool _stepDownPending = false; // (M) + // list of information about clients waiting on replication. Does *not* own the WaiterInfos. std::vector _replicationWaiterList; // (M) diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect.cpp index a2cd0eb386e..b8b413c7d79 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect.cpp @@ -202,7 +202,11 @@ void ReplicationCoordinatorImpl::_onFreshnessCheckComplete() { return; } - log() << "running for election"; + log() << "running for election" + << (abortReason == FreshnessChecker::FreshnessTie + ? "; slept last election, so running regardless of possible tie" + : ""); + // Secure our vote for ourself first if (!_topCoord->voteForMyself(now)) { return; diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp index 6dac4852d0f..1a3838b7c11 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp @@ -515,6 +515,7 @@ TEST_F(ReplCoordElectTest, StepsDownRemoteIfNodeHasHigherPriorityThanCurrentPrim auto&& request = noi->getRequest(); log() << request.target << " processing " << request.cmdObj; ASSERT_EQUALS("replSetStepDown", request.cmdObj.firstElement().fieldNameStringData()); + ASSERT_EQUALS(1LL, request.cmdObj["secondaryCatchUpPeriodSecs"].safeNumberLong()); auto target = request.target; ASSERT_EQUALS(HostAndPort("node2", 12345), target); auto response = makeResponseStatus(BSON("ok" << 1)); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp index 97b545a1292..b52c5a124e9 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp @@ -188,9 +188,7 @@ void ReplicationCoordinatorImpl::_onDryRunComplete(long long originalTerm) { _topCoord->voteForMyselfV1(); // Store the vote in persistent storage. - LastVote lastVote; - lastVote.setTerm(originalTerm + 1); - lastVote.setCandidateIndex(_selfIndex); + LastVote lastVote{originalTerm + 1, _selfIndex}; auto cbStatus = _replExecutor.scheduleDBWork( [this, lastVote](const ReplicationExecutor::CallbackArgs& cbData) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 4c23bab4f5b..aa3548335c1 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -64,6 +64,8 @@ typedef ReplicationExecutor::CallbackHandle CBHandle; using executor::RemoteCommandRequest; +MONGO_FP_DECLARE(blockHeartbeatStepdown); + void ReplicationCoordinatorImpl::_doMemberHeartbeat(ReplicationExecutor::CallbackArgs cbData, const HostAndPort& target, int targetIndex) { @@ -152,7 +154,9 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( if (replMetadata.isOK()) { // Asynchronous stepdown could happen, but it will be queued in executor after // this function, so we cannot and don't need to wait for it to finish. - _processReplSetMetadata_incallback(replMetadata.getValue()); + // Arbiters are the only nodes allowed to advance their commit point via heartbeats. + bool advanceCommitPoint = getMemberState().arbiter(); + _processReplSetMetadata_incallback(replMetadata.getValue(), advanceCommitPoint); } } const Date_t now = _replExecutor.now(); @@ -164,10 +168,11 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( networkTime = cbData.response.getValue().elapsedMillis; // TODO(sz) Because the term is duplicated in ReplSetMetaData, we can get rid of this // and update tests. - _updateTerm_incallback(hbStatusResponse.getValue().getTerm()); - // Postpone election timeout if we have a successful heartbeat response from the primary. const auto& hbResponse = hbStatusResponse.getValue(); - if (hbResponse.hasState() && hbResponse.getState().primary()) { + _updateTerm_incallback(hbResponse.getTerm()); + // Postpone election timeout if we have a successful heartbeat response from the primary. + if (hbResponse.hasState() && hbResponse.getState().primary() && + hbResponse.getTerm() == _topCoord->getTerm()) { cancelAndRescheduleElectionTimeout(); } } else { @@ -206,7 +211,7 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( _scheduleHeartbeatToTarget( target, targetIndex, std::max(now, action.getNextHeartbeatStartDate())); - _handleHeartbeatResponseAction(action, hbStatusResponse); + _handleHeartbeatResponseAction(action, hbStatusResponse, false /*we're not holding _mutex*/); } void ReplicationCoordinatorImpl::_updateOpTimesFromHeartbeat_inlock(int targetIndex, @@ -226,11 +231,13 @@ void ReplicationCoordinatorImpl::_updateOpTimesFromHeartbeat_inlock(int targetIn void ReplicationCoordinatorImpl::_handleHeartbeatResponseAction( const HeartbeatResponseAction& action, - const StatusWith& responseStatus) { + const StatusWith& responseStatus, + bool hasMutex) { switch (action.getAction()) { case HeartbeatResponseAction::NoAction: // Update the cached member state if different than the current topology member state if (_memberState != _topCoord->getMemberState()) { + invariant(!hasMutex); stdx::unique_lock lk(_mutex); const PostMemberStateUpdateAction postUpdateAction = _updateMemberStateFromTopologyCoordinator_inlock(); @@ -250,7 +257,7 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponseAction( log() << "Stepping down from primary in response to heartbeat"; _topCoord->prepareForStepDown(); // Don't need to wait for stepdown to finish. - _stepDownStart(); + _stepDownStart(hasMutex); break; case HeartbeatResponseAction::StepDownRemotePrimary: { invariant(action.getPrimaryConfigIndex() != _selfIndex); @@ -295,7 +302,13 @@ void remoteStepdownCallback(const ReplicationExecutor::RemoteCommandCallbackArgs } // namespace void ReplicationCoordinatorImpl::_requestRemotePrimaryStepdown(const HostAndPort& target) { - RemoteCommandRequest request(target, "admin", BSON("replSetStepDown" << 1)); + auto secondaryCatchUpPeriod(duration_cast(_rsConfig.getHeartbeatInterval() / 2)); + RemoteCommandRequest request( + target, + "admin", + BSON("replSetStepDown" << 20 << "secondaryCatchUpPeriodSecs" + << std::min(static_cast(secondaryCatchUpPeriod.count()), + 20LL))); log() << "Requesting " << target << " step down from primary"; CBHStatus cbh = _replExecutor.scheduleRemoteCommand(request, remoteStepdownCallback); @@ -304,11 +317,19 @@ void ReplicationCoordinatorImpl::_requestRemotePrimaryStepdown(const HostAndPort } } -ReplicationExecutor::EventHandle ReplicationCoordinatorImpl::_stepDownStart() { +ReplicationExecutor::EventHandle ReplicationCoordinatorImpl::_stepDownStart(bool hasMutex) { + { + boost::optional> lk; + if (!hasMutex) { + lk.emplace(_mutex); + } + _stepDownPending = true; + } auto finishEvent = _makeEvent(); if (!finishEvent) { return finishEvent; } + _replExecutor.scheduleWorkWithGlobalExclusiveLock(stdx::bind( &ReplicationCoordinatorImpl::_stepDownFinish, this, stdx::placeholders::_1, finishEvent)); return finishEvent; @@ -321,13 +342,28 @@ void ReplicationCoordinatorImpl::_stepDownFinish( return; } invariant(cbData.txn); + + if (MONGO_FAIL_POINT(blockHeartbeatStepdown)) { + // Must reschedule rather than block so we don't take up threads in the replication + // executor. + sleepmillis(10); + _replExecutor.scheduleWorkWithGlobalExclusiveLock( + stdx::bind(&ReplicationCoordinatorImpl::_stepDownFinish, + this, + stdx::placeholders::_1, + finishedEvent)); + + return; + } + // TODO Add invariant that we've got global shared or global exclusive lock, when supported // by lock manager. stdx::unique_lock lk(_mutex); - _topCoord->stepDownIfPending(); - const PostMemberStateUpdateAction action = _updateMemberStateFromTopologyCoordinator_inlock(); - lk.unlock(); - _performPostMemberStateUpdateAction(action); + if (_topCoord->stepDownIfPending()) { + const auto action = _updateMemberStateFromTopologyCoordinator_inlock(); + lk.unlock(); + _performPostMemberStateUpdateAction(action); + } _replExecutor.signalEvent(finishedEvent); } @@ -526,8 +562,15 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigFinish( const PostMemberStateUpdateAction action = _setCurrentRSConfig_inlock(cbData, newConfig, myIndexValue); lk.unlock(); - _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); - _performPostMemberStateUpdateAction(action); + auto evh = _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); + if (evh) { + _replExecutor.onEvent(evh, + [this, action](const ReplicationExecutor::CallbackArgs& cbArgs) { + _performPostMemberStateUpdateAction(action); + }); + } else { + _performPostMemberStateUpdateAction(action); + } } void ReplicationCoordinatorImpl::_trackHeartbeatHandle(const StatusWith& handle) { @@ -623,7 +666,9 @@ void ReplicationCoordinatorImpl::_handleLivenessTimeout( _topCoord->setMemberAsDown(now, memberIndex, _getMyLastDurableOpTime_inlock()); // Don't mind potential asynchronous stepdown as this is the last step of // liveness check. - _handleHeartbeatResponseAction(action, makeStatusWith()); + _handleHeartbeatResponseAction(action, + makeStatusWith(), + true /*we're holding _mutex*/); } } } diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp index 2233e21cc21..40a30c48e5a 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp @@ -351,7 +351,7 @@ TEST_F(ReplCoordHBV1Test, ArbiterRecordsCommittedOpTimeFromHeartbeatMetadata) { << 1 << "primaryIndex" << 1 << "term" << committedOpTime.getTerm() << "syncSourceIndex" << 1))); ASSERT_OK(metadata.getStatus()); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQ(getReplCoord()->getMyLastAppliedOpTime().getTimestamp(), expected.getTimestamp()); }; diff --git a/src/mongo/db/repl/replication_coordinator_impl_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_test.cpp index f730e4deccd..a9b1d297be0 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_test.cpp @@ -44,8 +44,10 @@ #include "mongo/db/repl/optime.h" #include "mongo/db/repl/read_concern_args.h" #include "mongo/db/repl/read_concern_response.h" +#include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/repl_set_heartbeat_args.h" #include "mongo/db/repl/repl_set_heartbeat_args_v1.h" +#include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/repl_settings.h" #include "mongo/db/repl/replica_set_config.h" #include "mongo/db/repl/replication_coordinator.h" // ReplSetReconfigArgs @@ -80,15 +82,15 @@ using executor::RemoteCommandResponse; typedef ReplicationCoordinator::ReplSetReconfigArgs ReplSetReconfigArgs; Status kInterruptedStatus(ErrorCodes::Interrupted, "operation was interrupted"); -// Helper class to wrap Timestamp as an OpTime with term 0. -struct OpTimeWithTermZero { - OpTimeWithTermZero(unsigned int sec, unsigned int i) : timestamp(sec, i) {} +// Helper class to wrap Timestamp as an OpTime with term 1. +struct OpTimeWithTermOne { + OpTimeWithTermOne(unsigned int sec, unsigned int i) : timestamp(sec, i) {} operator OpTime() const { - return OpTime(timestamp, 0); + return OpTime(timestamp, 1); } operator boost::optional() const { - return OpTime(timestamp, 0); + return OpTime(timestamp, 1); } OpTime asOpTime() const { @@ -673,7 +675,7 @@ TEST_F(ReplCoordTest, RollBackIDShouldIncreaseByOneWhenIncrementRollbackIDIsCall TEST_F(ReplCoordTest, NodeReturnsImmediatelyWhenAwaitReplicationIsRanAgainstAStandaloneNode) { init(""); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -691,7 +693,7 @@ TEST_F(ReplCoordTest, NodeReturnsImmediatelyWhenAwaitReplicationIsRanAgainstAMas settings.setMaster(true); init(settings); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -718,7 +720,7 @@ TEST_F(ReplCoordTest, NodeReturnsNotMasterWhenRunningAwaitReplicationAgainstASec HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -731,7 +733,7 @@ TEST_F(ReplCoordTest, NodeReturnsNotMasterWhenRunningAwaitReplicationAgainstASec ASSERT_EQUALS(ErrorCodes::NotMaster, statusAndDur.status); } -TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWithWZero) { +TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWithWTermOne) { assertStartSuccess(BSON("_id" << "mySet" << "version" << 2 << "members" @@ -746,7 +748,7 @@ TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWith HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -755,8 +757,8 @@ TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWith // Become primary. ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ASSERT(getReplCoord()->getMemberState().primary()); @@ -786,12 +788,12 @@ TEST_F(ReplCoordTest, << "_id" << 3))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -859,12 +861,12 @@ TEST_F(ReplCoordTest, NodeReturnsWriteConcernFailedUntilASufficientNumberOfNodes << "_id" << 3))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -1028,7 +1030,7 @@ TEST_F( getReplCoord()->setLastDurableOptime_forTest(2, 1, time1); getReplCoord()->setLastAppliedOptime_forTest(2, 2, time1); getReplCoord()->setLastDurableOptime_forTest(2, 2, time1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); statusAndDur = getReplCoord()->awaitReplication(&txn, time1, majorityWriteConcern); ASSERT_OK(statusAndDur.status); @@ -1055,6 +1057,8 @@ TEST_F( // another name if we didn't get a high enough one. } + auto zeroOpTimeInCurrentTerm = OpTime(Timestamp(0, 0), 1); + ReplClientInfo::forClient(txn.getClient()).setLastOp(zeroOpTimeInCurrentTerm); statusAndDur = getReplCoord()->awaitReplicationOfLastOpForClient(&txn, majorityWriteConcern); ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, statusAndDur.status); statusAndDur = getReplCoord()->awaitReplicationOfLastOpForClient(&txn, multiDCWriteConcern); @@ -1063,7 +1067,7 @@ TEST_F( ASSERT_OK(statusAndDur.status); // All modes satisfied - getReplCoord()->onSnapshotCreate(time1, getReplCoord()->reserveSnapshotName(nullptr)); + getReplCoord()->createSnapshot(&txn, time1, getReplCoord()->reserveSnapshotName(nullptr)); statusAndDur = getReplCoord()->awaitReplicationOfLastOpForClient(&txn, majorityWriteConcern); ASSERT_OK(statusAndDur.status); @@ -1157,14 +1161,14 @@ TEST_F(ReplCoordTest, NodeReturnsOkWhenAWriteConcernWithNoTimeoutHasBeenSatisfie << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1216,14 +1220,14 @@ TEST_F(ReplCoordTest, NodeReturnsWriteConcernFailedWhenAWriteConcernTimesOutBefo << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = 50; @@ -1257,14 +1261,14 @@ TEST_F(ReplCoordTest, << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1299,14 +1303,14 @@ TEST_F(ReplCoordTest, NodeReturnsNotMasterWhenSteppingDownBeforeSatisfyingAWrite << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1339,14 +1343,14 @@ TEST_F(ReplCoordTest, << "node3"))), HostAndPort("node1")); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1377,7 +1381,6 @@ private: virtual void setUp() { ReplCoordTest::setUp(); init("mySet/test1:1234,test2:1234,test3:1234"); - assertStartSuccess(BSON("_id" << "mySet" << "version" << 1 << "members" @@ -1393,6 +1396,7 @@ private: } }; + TEST_F(ReplCoordTest, NodeReturnsBadValueWhenUpdateTermIsRunAgainstANonReplNode) { init(ReplSettings()); ASSERT_TRUE(ReplicationCoordinator::modeNone == getReplCoord()->getReplicationMode()); @@ -1522,7 +1526,7 @@ TEST_F(ReplCoordTest, ConcurrentStepDownShouldNotSignalTheSameFinishEventMoreTha TEST_F(StepDownTest, NodeReturnsNotMasterWhenAskedToStepDownAsANonPrimaryNode) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); + OpTimeWithTermOne optime1(100, 1); // All nodes are caught up getReplCoord()->setMyLastAppliedOpTime(optime1); getReplCoord()->setMyLastDurableOpTime(optime1); @@ -1537,7 +1541,7 @@ TEST_F(StepDownTest, NodeReturnsNotMasterWhenAskedToStepDownAsANonPrimaryNode) { TEST_F(StepDownTest, NodeReturnsExceededTimeLimitWhenStepDownFailsToObtainTheGlobalLockWithinTheAllottedTime) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); + OpTimeWithTermOne optime1(100, 1); // All nodes are caught up getReplCoord()->setMyLastAppliedOpTime(optime1); getReplCoord()->setMyLastDurableOpTime(optime1); @@ -1554,43 +1558,170 @@ TEST_F(StepDownTest, ASSERT_TRUE(getReplCoord()->getMemberState().primary()); } -TEST_F(StepDownTest, - NodeTransitionsToSecondaryImmediatelyWhenStepDownIsRunAndAnUpToDateElectableNodeExists) { +/* Step Down Test for a 5-node replica set */ +class StepDownTestFiveNode : public StepDownTest { +protected: + /* + * Simulate a round of heartbeat requests from the primary by manually setting + * the heartbeat response messages from each node. 'numNodesCaughtUp' will + * determine how many nodes return an optime that is up to date with the + * primary's optime. Sets electability of all caught up nodes to 'caughtUpAreElectable' + */ + void simulateHeartbeatResponses(OpTime optimePrimary, + OpTime optimeLagged, + int numNodesCaughtUp, + bool caughtUpAreElectable) { + int hbNum = 1; + while (getNet()->hasReadyRequests()) { + NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); + RemoteCommandRequest request = noi->getRequest(); + + // Only process heartbeat requests. + ASSERT_EQ(request.cmdObj.firstElement().fieldNameStringData().toString(), + "replSetHeartbeat"); + + ReplSetHeartbeatArgsV1 hbArgs; + ASSERT_OK(hbArgs.initialize(request.cmdObj)); + + log() << request.target.toString() << " processing " << request.cmdObj; + + // Catch up 'numNodesCaughtUp' nodes out of 5. + OpTime optimeResponse = (hbNum <= numNodesCaughtUp) ? optimePrimary : optimeLagged; + bool isElectable = (hbNum <= numNodesCaughtUp) ? caughtUpAreElectable : true; + + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName(hbArgs.getSetName()); + hbResp.setState(MemberState::RS_SECONDARY); + hbResp.setConfigVersion(hbArgs.getConfigVersion()); + hbResp.setDurableOpTime(optimeResponse); + hbResp.setAppliedOpTime(optimeResponse); + hbResp.setElectable(isElectable); + BSONObjBuilder respObj; + respObj << "ok" << 1; + hbResp.addToBSON(&respObj, false); + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(respObj.obj())); + hbNum += 1; + } + } + +private: + virtual void setUp() { + ReplCoordTest::setUp(); + init("mySet/test1:1234,test2:1234,test3:1234,test4:1234,test5:1234"); + + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "test1:1234") + << BSON("_id" << 1 << "host" + << "test2:1234") + << BSON("_id" << 2 << "host" + << "test3:1234") + << BSON("_id" << 3 << "host" + << "test4:1234") + << BSON("_id" << 4 << "host" + << "test5:1234"))), + HostAndPort("test1", 1234)); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + myRid = getReplCoord()->getMyRID(); + } +}; + +TEST_F( + StepDownTestFiveNode, + NodeReturnsExceededTimeLimitWhenStepDownIsRunAndCaughtUpMajorityExistsButWithoutElectableNode) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); + OpTime optimeLagged(Timestamp(100, 1), 1); + OpTime optimePrimary(Timestamp(100, 2), 1); + // All nodes are caught up - getReplCoord()->setMyLastAppliedOpTime(optime1); - getReplCoord()->setMyLastDurableOpTime(optime1); - ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optime1)); - ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optime1)); + getReplCoord()->setMyLastAppliedOpTime(optimePrimary); + getReplCoord()->setMyLastDurableOpTime(optimePrimary); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 3, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 4, optimeLagged)); simulateSuccessfulV1Election(); enterNetwork(); getNet()->runUntil(getNet()->now() + Seconds(2)); ASSERT(getNet()->hasReadyRequests()); - NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); - RemoteCommandRequest request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - ReplSetHeartbeatArgsV1 hbArgs; - if (hbArgs.initialize(request.cmdObj).isOK()) { - ReplSetHeartbeatResponse hbResp; - hbResp.setSetName(hbArgs.getSetName()); - hbResp.setState(MemberState::RS_SECONDARY); - hbResp.setConfigVersion(hbArgs.getConfigVersion()); - hbResp.setDurableOpTime(optime1); - hbResp.setAppliedOpTime(optime1); - BSONObjBuilder respObj; - respObj << "ok" << 1; - hbResp.addToBSON(&respObj, false); - getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(respObj.obj())); - } - while (getNet()->hasReadyRequests()) { - getNet()->blackHole(getNet()->getNextReadyRequest()); - } + + // Make sure a majority are caught up (i.e. 3 out of 5). We catch up two secondaries since + // the primary counts as one towards majority + int numNodesCaughtUp = 2; + simulateHeartbeatResponses(optimePrimary, optimeLagged, numNodesCaughtUp, false); + getNet()->runReadyNetworkOperations(); + exitNetwork(); + + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); + auto status = getReplCoord()->stepDown(&txn, false, Milliseconds(0), Milliseconds(1000)); + ASSERT_EQUALS(ErrorCodes::ExceededTimeLimit, status); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); +} + +TEST_F(StepDownTestFiveNode, + NodeReturnsExceededTimeLimitWhenStepDownIsRunAndNoCaughtUpMajorityExists) { + OperationContextReplMock txn; + OpTime optimeLagged(Timestamp(100, 1), 1); + OpTime optimePrimary(Timestamp(100, 2), 1); + + // All nodes are caught up + getReplCoord()->setMyLastAppliedOpTime(optimePrimary); + getReplCoord()->setMyLastDurableOpTime(optimePrimary); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 3, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 4, optimeLagged)); + + simulateSuccessfulV1Election(); + + enterNetwork(); + getNet()->runUntil(getNet()->now() + Seconds(2)); + ASSERT(getNet()->hasReadyRequests()); + + // Make sure less than a majority are caught up (i.e. 2 out of 5) We catch up one secondary + // since the primary counts as one towards majority + int numNodesCaughtUp = 1; + simulateHeartbeatResponses(optimePrimary, optimeLagged, numNodesCaughtUp, true); getNet()->runReadyNetworkOperations(); exitNetwork(); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); + auto status = getReplCoord()->stepDown(&txn, false, Milliseconds(0), Milliseconds(1000)); + ASSERT_EQUALS(ErrorCodes::ExceededTimeLimit, status); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); +} + +TEST_F( + StepDownTestFiveNode, + NodeTransitionsToSecondaryImmediatelyWhenStepDownIsRunAndAnUpToDateMajorityWithElectableNodeExists) { + OperationContextReplMock txn; + OpTime optimeLagged(Timestamp(100, 1), 1); + OpTime optimePrimary(Timestamp(100, 2), 1); + + // All nodes are caught up + getReplCoord()->setMyLastAppliedOpTime(optimePrimary); + getReplCoord()->setMyLastDurableOpTime(optimePrimary); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 3, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 4, optimeLagged)); + + simulateSuccessfulV1Election(); + + enterNetwork(); + getNet()->runUntil(getNet()->now() + Seconds(2)); + ASSERT(getNet()->hasReadyRequests()); + + // Make sure a majority are caught up (i.e. 3 out of 5). We catch up two secondaries since + // the primary counts as one towards majority + int numNodesCaughtUp = 2; + simulateHeartbeatResponses(optimePrimary, optimeLagged, numNodesCaughtUp, true); + getNet()->runReadyNetworkOperations(); + exitNetwork(); ASSERT_TRUE(getReplCoord()->getMemberState().primary()); ASSERT_OK(getReplCoord()->stepDown(&txn, false, Milliseconds(0), Milliseconds(1000))); @@ -1631,8 +1762,8 @@ TEST_F(ReplCoordTest, NodeBecomesPrimaryAgainWhenStepDownTimeoutExpiresInASingle TEST_F(StepDownTest, NodeReturnsExceededTimeLimitWhenNoSecondaryIsCaughtUpWithinStepDownsSecondaryCatchUpPeriod) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTimeWithTermOne optime1(100, 1); + OpTimeWithTermOne optime2(100, 2); // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1666,8 +1797,9 @@ TEST_F(StepDownTest, TEST_F(StepDownTest, NodeTransitionsToSecondaryWhenASecondaryCatchesUpAfterTheFirstRoundOfHeartbeats) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTime optime1(Timestamp(100, 1), 1); + OpTime optime2(Timestamp(100, 2), 1); + // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1677,6 +1809,8 @@ TEST_F(StepDownTest, simulateSuccessfulV1Election(); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); + // Step down where the secondary actually has to catch up before the stepDown can succeed. // On entering the network, _stepDownContinue should cancel the heartbeats scheduled for // T + 2 seconds and send out a new round of heartbeats immediately. @@ -1713,6 +1847,7 @@ TEST_F(StepDownTest, log() << "Blackholing network request " << noi->getRequest().cmdObj; getNet()->blackHole(noi); } + getNet()->runReadyNetworkOperations(); exitNetwork(); @@ -1724,8 +1859,9 @@ TEST_F(StepDownTest, TEST_F(StepDownTest, NodeTransitionsToSecondaryWhenASecondaryCatchesUpDuringStepDownsSecondaryCatchupPeriod) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTime optime1(Timestamp(100, 1), 1); + OpTime optime2(Timestamp(100, 2), 1); + // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1808,8 +1944,8 @@ TEST_F(StepDownTest, TEST_F(StepDownTest, NodeReturnsInterruptedWhenInterruptedDuringStepDown) { const unsigned int opID = 100; OperationContextReplMock txn{opID}; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTimeWithTermOne optime1(100, 1); + OpTimeWithTermOne optime2(100, 2); // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1906,7 +2042,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInUpdatePositionCommand) { getReplCoord()->prepareReplSetUpdatePositionCommand(&cmdBuilder); BSONObj cmd = cmdBuilder.done(); - ASSERT_EQUALS(2, cmd.nFields()); + ASSERT_EQUALS(3, cmd.nFields()); ASSERT_EQUALS("replSetUpdatePosition", cmd.firstElement().fieldNameStringData()); std::set memberIds; @@ -1959,9 +2095,9 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInOldUpdatePositionCommand << "test2:1234") << BSON("_id" << 2 << "host" << "test3:1234"))), HostAndPort("test1", 1234)); - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); - OpTimeWithTermZero optime3(2, 1); + OpTimeWithTermOne optime1(100, 1); + OpTimeWithTermOne optime2(100, 2); + OpTimeWithTermOne optime3(2, 1); getReplCoord()->setMyLastAppliedOpTime(optime1); getReplCoord()->setMyLastDurableOpTime(optime1); ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optime2)); @@ -1991,7 +2127,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInOldUpdatePositionCommand ASSERT_EQUALS(2, memberId); ASSERT_EQUALS(optime3.timestamp, entry["optime"]["ts"].timestamp()); } - ASSERT_EQUALS(0, entry["optime"]["t"].Number()); + ASSERT_EQUALS(1, entry["optime"]["t"].Number()); } ASSERT_EQUALS(3U, memberIds.size()); // Make sure we saw all 3 nodes } @@ -2011,8 +2147,8 @@ TEST_F(ReplCoordTest, HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // Can't unset maintenance mode if it was never set to begin with. Status status = getReplCoord()->setMaintenanceMode(false); @@ -2035,8 +2171,8 @@ TEST_F(ReplCoordTest, HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // valid set ASSERT_OK(getReplCoord()->setMaintenanceMode(true)); ASSERT_TRUE(getReplCoord()->getMemberState().recovering()); @@ -2064,8 +2200,8 @@ TEST_F(ReplCoordTest, AllowAsManyUnsetMaintenanceModesAsThereHaveBeenSetMaintena HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // Can set multiple times ASSERT_OK(getReplCoord()->setMaintenanceMode(true)); ASSERT_OK(getReplCoord()->setMaintenanceMode(true)); @@ -2095,8 +2231,8 @@ TEST_F(ReplCoordTest, SettingAndUnsettingMaintenanceModeShouldNotAffectRollbackS HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // From rollback, entering and exiting maintenance mode doesn't change perceived // state. @@ -2134,8 +2270,8 @@ TEST_F(ReplCoordTest, DoNotAllowMaintenanceModeWhilePrimary) { HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // Can't modify maintenance mode when PRIMARY simulateSuccessfulV1Election(); @@ -2167,8 +2303,8 @@ TEST_F(ReplCoordTest, DoNotAllowSettingMaintenanceModeWhileConductingAnElection) HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // TODO this election shouldn't have to happen. simulateSuccessfulV1Election(); @@ -2227,8 +2363,8 @@ TEST_F(ReplCoordTest, HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); getReplCoord()->setMyLastAppliedOpTime(time2); getReplCoord()->setMyLastDurableOpTime(time2); @@ -2271,8 +2407,8 @@ TEST_F(ReplCoordTest, HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); getReplCoord()->setMyLastAppliedOpTime(time2); getReplCoord()->setMyLastDurableOpTime(time2); @@ -2301,8 +2437,8 @@ TEST_F(ReplCoordTest, NodeDoesNotIncludeItselfWhenRunningGetHostsWrittenToInMast OperationContextNoop txn; OID client = OID::gen(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); getExternalState()->setClientHostAndPort(clientHost); HandshakeArgs handshake; @@ -2451,12 +2587,12 @@ TEST_F(ReplCoordTest, DoNotProcessSelfWhenUpdatePositionContainsInfoAboutSelf) { << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTime time1({100, 1}, 2); - OpTime time2({100, 2}, 2); + OpTime time1({100, 1}, 1); + OpTime time2({100, 2}, 1); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2497,13 +2633,13 @@ TEST_F(ReplCoordTest, DoNotProcessSelfWhenOldUpdatePositionContainsInfoAboutSelf << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2541,12 +2677,12 @@ TEST_F(ReplCoordTest, DoNotProcessUpdatePositionWhenItsConfigVersionIsIncorrect) << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTime time1({100, 1}, 3); - OpTime time2({100, 2}, 3); + OpTime time1({100, 1}, 1); + OpTime time2({100, 2}, 1); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2586,13 +2722,13 @@ TEST_F(ReplCoordTest, DoNotProcessOldUpdatePositionWhenItsConfigVersionIsIncorre << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2629,12 +2765,12 @@ TEST_F(ReplCoordTest, DoNotProcessUpdatePositionOfMembersWhoseIdsAreNotInTheConf << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTime time1({100, 1}, 2); - OpTime time2({100, 2}, 2); + OpTime time1({100, 1}, 1); + OpTime time2({100, 2}, 1); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2672,13 +2808,13 @@ TEST_F(ReplCoordTest, DoNotProcessOldUpdatePositionOfMembersWhoseIdsAreNotInTheC << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2714,13 +2850,13 @@ TEST_F(ReplCoordTest, << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2779,11 +2915,11 @@ TEST_F(ReplCoordTest, AwaitReplicationShouldResolveAsNormalDuringAReconfig) { << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 2)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 2)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 2)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 2)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time(100, 2); + OpTimeWithTermOne time(100, 2); // 3 nodes waiting for time WriteConcernOptions writeConcern; @@ -2861,11 +2997,11 @@ TEST_F( << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 2)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 2)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 2)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 2)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time(100, 2); + OpTimeWithTermOne time(100, 2); // 3 nodes waiting for time WriteConcernOptions writeConcern; @@ -2922,15 +3058,15 @@ TEST_F(ReplCoordTest, << "_id" << 4))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 1)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 1)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 1)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 1)); simulateSuccessfulV1Election(); OpTime time(Timestamp(100, 2), 1); getReplCoord()->setMyLastAppliedOpTime(time); getReplCoord()->setMyLastDurableOpTime(time); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(2, 1, time)); @@ -3021,7 +3157,7 @@ TEST_F(ReplCoordTest, ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, getReplCoord()->awaitReplication(&txn, time, majorityWriteConcern).status); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); ASSERT_OK(getReplCoord()->awaitReplication(&txn, time, majorityWriteConcern).status); } @@ -3094,13 +3230,13 @@ TEST_F(ReplCoordTest, NodeReturnsShutdownInProgressWhenWaitingUntilAnOpTimeDurin << "_id" << 0))), HostAndPort("node1", 12345)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(10, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(10, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(10, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(10, 0)); shutdown(); auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_TRUE(result.didWait()); ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, result.getStatus()); @@ -3115,13 +3251,13 @@ TEST_F(ReplCoordTest, NodeReturnsInterruptedWhenWaitingUntilAnOpTimeIsInterrupte << "_id" << 0))), HostAndPort("node1", 12345)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(10, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(10, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(10, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(10, 0)); txn.setCheckForInterruptStatus(Status(ErrorCodes::Interrupted, "test")); auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_TRUE(result.didWait()); ASSERT_EQUALS(ErrorCodes::Interrupted, result.getStatus()); @@ -3151,10 +3287,10 @@ TEST_F(ReplCoordTest, NodeReturnsOkImmediatelyWhenWaitingUntilOpTimePassesAnOpTi << "_id" << 0))), HostAndPort("node1", 12345)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_TRUE(result.didWait()); ASSERT_OK(result.getStatus()); @@ -3170,7 +3306,7 @@ TEST_F(ReplCoordTest, NodeReturnsOkImmediatelyWhenWaitingUntilOpTimePassesAnOpTi HostAndPort("node1", 12345)); - OpTimeWithTermZero time(100, 0); + OpTimeWithTermOne time(100, 0); getReplCoord()->setMyLastAppliedOpTime(time); getReplCoord()->setMyLastDurableOpTime(time); auto result = getReplCoord()->waitUntilOpTime( @@ -3185,7 +3321,7 @@ TEST_F(ReplCoordTest, init(ReplSettings()); OperationContextNoop txn; auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_FALSE(result.didWait()); ASSERT_EQUALS(ErrorCodes::NotAReplicaSet, result.getStatus()); @@ -3261,7 +3397,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedGreaterOpTime) { getReplCoord()->setMyLastAppliedOpTime(OpTime(Timestamp(100, 0), 1)); getReplCoord()->setMyLastDurableOpTime(OpTime(Timestamp(100, 0), 1)); - getReplCoord()->onSnapshotCreate(OpTime(Timestamp(100, 0), 1), SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, OpTime(Timestamp(100, 0), 1), SnapshotName(1)); auto result = getReplCoord()->waitUntilOpTime( &txn, ReadConcernArgs(OpTime(Timestamp(50, 0), 1), ReadConcernLevel::kMajorityReadConcern)); @@ -3281,7 +3417,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedEqualOpTime) { OpTime time(Timestamp(100, 0), 1); getReplCoord()->setMyLastAppliedOpTime(time); getReplCoord()->setMyLastDurableOpTime(time); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); auto result = getReplCoord()->waitUntilOpTime( &txn, ReadConcernArgs(time, ReadConcernLevel::kMajorityReadConcern)); @@ -3307,7 +3443,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedDeferredGreaterOpTime) { // Not guaranteed to be scheduled after waitUntil blocks... getReplCoord()->setMyLastAppliedOpTime(committedOpTime); getReplCoord()->setMyLastDurableOpTime(committedOpTime); - getReplCoord()->onSnapshotCreate(committedOpTime, SnapshotName(1)); + getReplCoord()->createSnapshot(nullptr, committedOpTime, SnapshotName(1)); }); auto result = getReplCoord()->waitUntilOpTime( @@ -3339,7 +3475,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedDeferredEqualOpTime) { // Not guaranteed to be scheduled after waitUntil blocks... getReplCoord()->setMyLastAppliedOpTime(opTimeToWait); getReplCoord()->setMyLastDurableOpTime(opTimeToWait); - getReplCoord()->onSnapshotCreate(opTimeToWait, SnapshotName(1)); + getReplCoord()->createSnapshot(nullptr, opTimeToWait, SnapshotName(1)); }); auto result = getReplCoord()->waitUntilOpTime( @@ -3372,7 +3508,7 @@ TEST_F(ReplCoordTest, IgnoreTheContentsOfMetadataWhenItsConfigVersionDoesNotMatc "lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "configVersion" << 1 << "primaryIndex" << 2 << "term" << 2 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); // higher configVersion @@ -3382,7 +3518,7 @@ TEST_F(ReplCoordTest, IgnoreTheContentsOfMetadataWhenItsConfigVersionDoesNotMatc << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "configVersion" << 100 << "primaryIndex" << 2 << "term" << 2 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata2.getValue()); + getReplCoord()->processReplSetMetadata(metadata2.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); } @@ -3409,7 +3545,7 @@ TEST_F(ReplCoordTest, UpdateLastCommittedOpTimeWhenTheLastCommittedOpTimeFromMet ASSERT_EQUALS(1, getReplCoord()->getTerm()); OpTime time(Timestamp(10, 0), 1); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); // higher OpTime, should change StatusWith metadata = rpc::ReplSetMetadata::readFromMetadata(BSON( @@ -3417,7 +3553,7 @@ TEST_F(ReplCoordTest, UpdateLastCommittedOpTimeWhenTheLastCommittedOpTimeFromMet "lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 1) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 1) << "configVersion" << 2 << "primaryIndex" << 2 << "term" << 1 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 1), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 1), getReplCoord()->getCurrentCommittedSnapshotOpTime()); @@ -3427,7 +3563,7 @@ TEST_F(ReplCoordTest, UpdateLastCommittedOpTimeWhenTheLastCommittedOpTimeFromMet "lastOpCommitted" << BSON("ts" << Timestamp(9, 0) << "t" << 1) << "lastOpVisible" << BSON("ts" << Timestamp(9, 0) << "t" << 1) << "configVersion" << 2 << "primaryIndex" << 2 << "term" << 1 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata2.getValue()); + getReplCoord()->processReplSetMetadata(metadata2.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 1), getReplCoord()->getLastCommittedOpTime()); } @@ -3458,7 +3594,7 @@ TEST_F(ReplCoordTest, UpdateTermWhenTheTermFromMetadataIsNewerButNeverUpdateCurr "lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "configVersion" << 2 << "primaryIndex" << 2 << "term" << 3 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 3), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(3, getReplCoord()->getTerm()); ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); @@ -3469,7 +3605,7 @@ TEST_F(ReplCoordTest, UpdateTermWhenTheTermFromMetadataIsNewerButNeverUpdateCurr "lastOpCommitted" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "lastOpVisible" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "configVersion" << 2 << "primaryIndex" << 1 << "term" << 2 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata2.getValue()); + getReplCoord()->processReplSetMetadata(metadata2.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(11, 0), 3), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(3, getReplCoord()->getTerm()); ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); @@ -3480,14 +3616,14 @@ TEST_F(ReplCoordTest, UpdateTermWhenTheTermFromMetadataIsNewerButNeverUpdateCurr "lastOpCommitted" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "lastOpVisible" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "configVersion" << 2 << "primaryIndex" << 1 << "term" << 3 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata3.getValue()); + getReplCoord()->processReplSetMetadata(metadata3.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(11, 0), 3), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(3, getReplCoord()->getTerm()); ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); } TEST_F(ReplCoordTest, - TermAndLastCommittedOpTimeUpdateWhenHeartbeatResponseWithMetadataHasFresherValues) { + LastCommittedOpTimeNotUpdatedEvenWhenHeartbeatResponseWithMetadataHasFresherValues) { // Ensure that the metadata is processed if it is contained in a heartbeat response. assertStartSuccess(BSON("_id" << "mySet" @@ -3507,7 +3643,61 @@ TEST_F(ReplCoordTest, auto replCoord = getReplCoord(); auto config = replCoord->getConfig(); - // Higher term - should update term and lastCommittedOpTime. + // Higher term - should update term but not last committed optime. + StatusWith metadata = rpc::ReplSetMetadata::readFromMetadata(BSON( + rpc::kReplSetMetadataFieldName + << BSON("lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "lastOpVisible" + << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "configVersion" + << config.getConfigVersion() << "primaryIndex" << 1 << "term" << 3 + << "syncSourceIndex" << 1))); + BSONObjBuilder metadataBuilder; + ASSERT_OK(metadata.getValue().writeToMetadata(&metadataBuilder)); + auto metadataObj = metadataBuilder.obj(); + + auto net = getNet(); + net->enterNetwork(); + + ASSERT_TRUE(net->hasReadyRequests()); + auto noi = net->getNextReadyRequest(); + const auto& request = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); + ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); + + ReplSetHeartbeatResponse hbResp; + hbResp.setConfigVersion(config.getConfigVersion()); + hbResp.setSetName(config.getReplSetName()); + hbResp.setState(MemberState::RS_SECONDARY); + net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON(true), metadataObj)); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); + ASSERT_EQUALS(3, getReplCoord()->getTerm()); + ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); +} + +TEST_F(ReplCoordTest, TermAndLastCommittedOpTimeUpdatedFromHeartbeatWhenArbiter) { + // Ensure that the metadata is processed if it is contained in a heartbeat response. + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0 << "arbiterOnly" << true) + << BSON("host" + << "node2:12345" + << "_id" << 1)) << "protocolVersion" << 1), + HostAndPort("node1", 12345)); + ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); + OperationContextNoop txn; + getReplCoord()->updateTerm(&txn, 1); + ASSERT_EQUALS(1, getReplCoord()->getTerm()); + + auto replCoord = getReplCoord(); + auto config = replCoord->getConfig(); + + // Higher term - should update term and lastCommittedOpTime since arbiters learn of the + // commit point via heartbeats. StatusWith metadata = rpc::ReplSetMetadata::readFromMetadata(BSON( rpc::kReplSetMetadataFieldName << BSON("lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "lastOpVisible" @@ -3703,7 +3893,7 @@ TEST_F(ReplCoordTest, } TEST_F(ReplCoordTest, - CancelAndRescheduleElectionTimeoutWhenProcessingHeartbeatResponseFromPrimary) { + RescheduleElectionTimeoutWhenProcessingHeartbeatResponseFromPrimaryInSameTerm) { assertStartSuccess(BSON("_id" << "mySet" << "protocolVersion" << 1 << "version" << 2 << "members" @@ -3735,6 +3925,8 @@ TEST_F(ReplCoordTest, ReplSetHeartbeatResponse hbResp; hbResp.setSetName("mySet"); hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setTerm(replCoord->getTerm()); + // Heartbeat response is scheduled with a delay so that we can be sure that // the election was rescheduled due to the heartbeat response. auto heartbeatWhen = net->now() + Seconds(1); @@ -3748,6 +3940,54 @@ TEST_F(ReplCoordTest, replCoord->getElectionTimeout_forTest()); } +TEST_F(ReplCoordTest, + DontRescheduleElectionTimeoutWhenProcessingHeartbeatResponseFromPrimaryInDiffertTerm) { + assertStartSuccess(BSON("_id" + << "mySet" + << "protocolVersion" << 1 << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1))), + HostAndPort("node1", 12345)); + + ReplicationCoordinatorImpl* replCoord = getReplCoord(); + ASSERT_TRUE(replCoord->setFollowerMode(MemberState::RS_SECONDARY)); + + auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest(); + ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); + + auto net = getNet(); + net->enterNetwork(); + ASSERT_TRUE(net->hasReadyRequests()); + auto noi = net->getNextReadyRequest(); + auto&& request = noi->getRequest(); + log() << "processing " << request.cmdObj; + ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); + + ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); + + // Respond to node1's heartbeat command to indicate that node2 is PRIMARY. + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setTerm(replCoord->getTerm() - 1); + + // Heartbeat response is scheduled with a delay so that we can be sure that + // the election was rescheduled due to the heartbeat response. + auto heartbeatWhen = net->now() + Seconds(1); + net->scheduleResponse(noi, heartbeatWhen, makeResponseStatus(hbResp.toBSON(true))); + net->runUntil(heartbeatWhen); + ASSERT_EQUALS(heartbeatWhen, net->now()); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + ASSERT_GREATER_THAN(heartbeatWhen + replCoord->getConfig().getElectionTimeoutPeriod(), + replCoord->getElectionTimeout_forTest()); +} + TEST_F(ReplCoordTest, CancelAndRescheduleElectionTimeoutWhenProcessingHeartbeatResponseWithoutState) { assertStartSuccess(BSON("_id" @@ -3812,9 +4052,9 @@ TEST_F(ReplCoordTest, AdvanceCommittedSnapshotToMostRecentSnapshotPriorToOpTimeW OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); // ensure current snapshot follows price is right rules (closest but not greater than) getReplCoord()->setMyLastAppliedOpTime(time3); @@ -3844,9 +4084,9 @@ TEST_F(ReplCoordTest, DoNotAdvanceCommittedSnapshotWhenAnOpTimeIsNewerThanOurLat OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); // ensure current snapshot will not advance beyond existing snapshots getReplCoord()->setMyLastAppliedOpTime(time6); @@ -3874,16 +4114,16 @@ TEST_F(ReplCoordTest, OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); getReplCoord()->setMyLastAppliedOpTime(time6); getReplCoord()->setMyLastDurableOpTime(time6); ASSERT_EQUALS(time5, getReplCoord()->getCurrentCommittedSnapshotOpTime()); // ensure current snapshot updates on new snapshot if we are that far - getReplCoord()->onSnapshotCreate(time6, SnapshotName(4)); + getReplCoord()->createSnapshot(&txn, time6, SnapshotName(4)); ASSERT_EQUALS(time6, getReplCoord()->getCurrentCommittedSnapshotOpTime()); } @@ -3906,9 +4146,9 @@ TEST_F(ReplCoordTest, ZeroCommittedSnapshotWhenAllSnapshotsAreDropped) { OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); // ensure dropping all snapshots should reset the current committed snapshot getReplCoord()->dropAllSnapshots(); @@ -3930,7 +4170,7 @@ TEST_F(ReplCoordTest, DoNotAdvanceCommittedSnapshotWhenAppliedOpTimeChanges) { OpTime time1(Timestamp(100, 1), 1); OpTime time2(Timestamp(100, 2), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); getReplCoord()->setMyLastAppliedOpTime(time1); ASSERT_EQUALS(OpTime(), getReplCoord()->getCurrentCommittedSnapshotOpTime()); @@ -4089,6 +4329,37 @@ TEST_F(ReplCoordTest, OnlyForwardSyncProgressForOtherNodesWhenTheNodesAreBelieve ASSERT_EQUALS(1U, memberIds4.size()); } +TEST_F(ReplCoordTest, NewStyleUpdatePositionCmdHasMetadata) { + assertStartSuccess( + BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "test1:1234") + << BSON("_id" << 1 << "host" + << "test2:1234") << BSON("_id" << 2 << "host" + << "test3:1234")) + << "protocolVersion" << 1 << "settings" + << BSON("electionTimeoutMillis" << 2000 << "heartbeatIntervalMillis" << 40000)), + HostAndPort("test1", 1234)); + OpTime optime(Timestamp(100, 2), 0); + getReplCoord()->setMyLastAppliedOpTime(optime); + getReplCoord()->setMyLastDurableOpTime(optime); + + // Set last committed optime via metadata. + OperationContextReplMock txn; + rpc::ReplSetMetadata syncSourceMetadata(optime.getTerm(), optime, optime, 1, OID(), -1, 1); + getReplCoord()->processReplSetMetadata(syncSourceMetadata, true); + getReplCoord()->createSnapshot(&txn, optime, SnapshotName(1)); + + BSONObjBuilder cmdBuilder; + ASSERT_TRUE(getReplCoord()->prepareReplSetUpdatePositionCommand(&cmdBuilder)); + BSONObj cmd = cmdBuilder.obj(); + auto metadata = unittest::assertGet(rpc::ReplSetMetadata::readFromMetadata(cmd)); + ASSERT_EQUALS(metadata.getTerm(), getReplCoord()->getTerm()); + ASSERT_EQUALS(metadata.getLastOpVisible(), optime); +} + TEST_F(ReplCoordTest, StepDownWhenHandleLivenessTimeoutMarksAMajorityOfVotingNodesDown) { assertStartSuccess( BSON("_id" @@ -4417,6 +4688,89 @@ TEST_F(ReplCoordTest, PopulateUnsetWriteConcernOptionsSyncModeReturnsInputIfWMod getReplCoord()->populateUnsetWriteConcernOptionsSyncMode(wc).syncMode); } +TEST_F(ReplCoordTest, NodeStoresElectionVotes) { + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1) << BSON("host" + << "node3:12345" + << "_id" << 2))), + HostAndPort("node1", 12345)); + auto time = OpTime(Timestamp(100, 0), 1); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + getReplCoord()->setMyLastAppliedOpTime(time); + getReplCoord()->setMyLastDurableOpTime(time); + simulateSuccessfulV1Election(); + + OperationContextReplMock txn; + + ReplSetRequestVotesArgs args; + ASSERT_OK(args.initialize(BSON( + "replSetRequestVotes" << 1 << "setName" + << "mySet" + << "term" << 7LL << "candidateIndex" << 2LL << "configVersion" << 2LL + << "dryRun" << false << "lastCommittedOp" << time.toBSON()))); + ReplSetRequestVotesResponse response; + + ASSERT_OK(getReplCoord()->processReplSetRequestVotes(&txn, args, &response)); + ASSERT_EQUALS("", response.getReason()); + ASSERT_TRUE(response.getVoteGranted()); + + auto lastVote = getExternalState()->loadLocalLastVoteDocument(&txn); + ASSERT_OK(lastVote.getStatus()); + + // This is not a dry-run election so the last vote should include the new term and candidate. + ASSERT_EQUALS(lastVote.getValue().getTerm(), 7); + ASSERT_EQUALS(lastVote.getValue().getCandidateIndex(), 2); +} + +TEST_F(ReplCoordTest, NodeDoesNotStoreDryRunVotes) { + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1) << BSON("host" + << "node3:12345" + << "_id" << 2))), + HostAndPort("node1", 12345)); + auto time = OpTime(Timestamp(100, 0), 1); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + getReplCoord()->setMyLastAppliedOpTime(time); + getReplCoord()->setMyLastDurableOpTime(time); + simulateSuccessfulV1Election(); + + OperationContextReplMock txn; + + ReplSetRequestVotesArgs args; + ASSERT_OK(args.initialize(BSON( + "replSetRequestVotes" << 1 << "setName" + << "mySet" + << "term" << 7LL << "candidateIndex" << 2LL << "configVersion" << 2LL + << "dryRun" << true << "lastCommittedOp" << time.toBSON()))); + ReplSetRequestVotesResponse response; + + ASSERT_OK(getReplCoord()->processReplSetRequestVotes(&txn, args, &response)); + ASSERT_EQUALS("", response.getReason()); + ASSERT_TRUE(response.getVoteGranted()); + + auto lastVote = getExternalState()->loadLocalLastVoteDocument(&txn); + ASSERT_OK(lastVote.getStatus()); + + // This is a dry-run election so the last vote should not be updated with the new term and + // candidate. + ASSERT_EQUALS(lastVote.getValue().getTerm(), 1); + ASSERT_EQUALS(lastVote.getValue().getCandidateIndex(), 0); +} + // TODO(schwerin): Unit test election id updating } // namespace } // namespace repl diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp index 7d3f987878f..184cf75eda1 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_mock.cpp @@ -236,7 +236,8 @@ void ReplicationCoordinatorMock::processReplSetGetConfig(BSONObjBuilder* result) // TODO } -void ReplicationCoordinatorMock::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) {} +void ReplicationCoordinatorMock::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) {} void ReplicationCoordinatorMock::cancelAndRescheduleElectionTimeout() {} @@ -381,9 +382,8 @@ Status ReplicationCoordinatorMock::processReplSetDeclareElectionWinner( return Status::OK(); } -void ReplicationCoordinatorMock::prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) {} +void ReplicationCoordinatorMock::prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const {} Status ReplicationCoordinatorMock::processHeartbeatV1(const ReplSetHeartbeatArgsV1& args, ReplSetHeartbeatResponse* response) { @@ -414,11 +414,13 @@ SnapshotName ReplicationCoordinatorMock::reserveSnapshotName(OperationContext* t void ReplicationCoordinatorMock::forceSnapshotCreation() {} -void ReplicationCoordinatorMock::onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) {} +void ReplicationCoordinatorMock::createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name){}; void ReplicationCoordinatorMock::dropAllSnapshots() {} -OpTime ReplicationCoordinatorMock::getCurrentCommittedSnapshotOpTime() { +OpTime ReplicationCoordinatorMock::getCurrentCommittedSnapshotOpTime() const { return OpTime(); } diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h index 15315cd5a8f..b5500265045 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.h +++ b/src/mongo/db/repl/replication_coordinator_mock.h @@ -150,7 +150,8 @@ public: virtual void processReplSetGetConfig(BSONObjBuilder* result); - virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata); + void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) override; virtual void cancelAndRescheduleElectionTimeout() override; @@ -217,9 +218,8 @@ public: virtual Status processReplSetDeclareElectionWinner(const ReplSetDeclareElectionWinnerArgs& args, long long* responseTerm); - void prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) override; + void prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const override; virtual Status processHeartbeatV1(const ReplSetHeartbeatArgsV1& args, ReplSetHeartbeatResponse* response); @@ -238,11 +238,13 @@ public: virtual void forceSnapshotCreation() override; - virtual void onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name); + virtual void createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) override; virtual void dropAllSnapshots() override; - virtual OpTime getCurrentCommittedSnapshotOpTime() override; + virtual OpTime getCurrentCommittedSnapshotOpTime() const override; virtual void waitUntilSnapshotCommitted(OperationContext* txn, const SnapshotName& untilSnapshot) override; diff --git a/src/mongo/db/repl/replication_executor.cpp b/src/mongo/db/repl/replication_executor.cpp index 4081e3fb876..2b1c7c6124f 100644 --- a/src/mongo/db/repl/replication_executor.cpp +++ b/src/mongo/db/repl/replication_executor.cpp @@ -274,6 +274,7 @@ StatusWith ReplicationExecutor::onEvent( queue = &event->_waiters; } else { queue = &_readyQueue; + _networkInterface->signalWorkAvailable(); } return enqueueWork_inlock(queue, work); } @@ -382,6 +383,7 @@ StatusWith ReplicationExecutor::scheduleWor ++insertBefore; _sleepersQueue.splice(insertBefore, temp, temp.begin()); ++_counterScheduledWorkAts; + _networkInterface->signalWorkAvailable(); return cbHandle; } @@ -426,7 +428,9 @@ void ReplicationExecutor::_doOperation(OperationContext* txn, return; Callback* callback = _getCallbackFromHandle(cbHandle); const WorkQueue::iterator iter = callback->_iter; + callback->_isRemoved = true; iter->callback = CallbackHandle(); + iter->isNetworkOperation = false; _freeQueue.splice(_freeQueue.begin(), *workQueue, iter); lk.unlock(); { @@ -493,7 +497,10 @@ ReplicationExecutor::getWork() { } const WorkItem work = *_readyQueue.begin(); const CallbackHandle cbHandle = work.callback; + Callback* callback = _getCallbackFromHandle(cbHandle); + callback->_isRemoved = true; _readyQueue.begin()->callback = CallbackHandle(); + _readyQueue.begin()->isNetworkOperation = false; _freeQueue.splice(_freeQueue.begin(), _readyQueue, _readyQueue.begin()); return std::make_pair(work, cbHandle); } @@ -590,6 +597,7 @@ ReplicationExecutor::Callback::Callback(ReplicationExecutor* executor, _callbackFn(callbackFn), _isCanceled(false), _isSleeper(false), + _isRemoved(false), _iter(iter), _finishedEvent(finishedEvent) {} @@ -602,6 +610,11 @@ bool ReplicationExecutor::Callback::isCanceled() const { void ReplicationExecutor::Callback::cancel() { stdx::unique_lock lk(_executor->_mutex); + // If this element has already been removed from the queues, + // the cancel is too late and has no effect. + if (_isRemoved) + return; + _isCanceled = true; if (_isSleeper) { diff --git a/src/mongo/db/repl/replication_executor.h b/src/mongo/db/repl/replication_executor.h index c46ef4a659a..9402a66f364 100644 --- a/src/mongo/db/repl/replication_executor.h +++ b/src/mongo/db/repl/replication_executor.h @@ -363,6 +363,7 @@ private: CallbackFn _callbackFn; bool _isCanceled; bool _isSleeper; + bool _isRemoved; WorkQueue::iterator _iter; EventHandle _finishedEvent; }; diff --git a/src/mongo/db/repl/replication_executor_test.cpp b/src/mongo/db/repl/replication_executor_test.cpp index 998a80a549e..60259be0b52 100644 --- a/src/mongo/db/repl/replication_executor_test.cpp +++ b/src/mongo/db/repl/replication_executor_test.cpp @@ -52,6 +52,8 @@ namespace repl { namespace { using executor::NetworkInterfaceMock; +using executor::RemoteCommandResponse; +using unittest::assertGet; const int64_t prngSeed = 1; @@ -171,6 +173,164 @@ TEST_F(ReplicationExecutorTest, CancelBeforeRunningFutureWork) { ASSERT_EQUALS(1, executor.getDiagnosticBSON().getFieldDotted("queues.ready").Int()); } +// Equivalent to EventChainAndWaitingTest::onGo +TEST_F(ReplicationExecutorTest, ScheduleCallbackOnFutureEvent) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + // We signal this "ping" event and the executor will signal "pong" event in return. + auto ping = assertGet(executor.makeEvent()); + auto pong = assertGet(executor.makeEvent()); + auto fn = [&executor, pong](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(pong); + }; + + // Wait for a future event. + executor.onEvent(ping, fn); + ASSERT_EQUALS(0, executor.getDiagnosticBSON().getFieldDotted("queues.ready").Int()); + executor.signalEvent(ping); + executor.waitForEvent(pong); +} + +// Equivalent to EventChainAndWaitingTest::onGoAfterTriggered +TEST_F(ReplicationExecutorTest, ScheduleCallbackOnSignaledEvent) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + // We signal this "ping" event and the executor will signal "pong" event in return. + auto ping = assertGet(executor.makeEvent()); + auto pong = assertGet(executor.makeEvent()); + auto fn = [&executor, pong](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(pong); + }; + + // Wait for a signaled event. + executor.signalEvent(ping); + executor.onEvent(ping, fn); + executor.waitForEvent(pong); +} + +TEST_F(ReplicationExecutorTest, ScheduleCallbackAtNow) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + auto finishEvent = assertGet(executor.makeEvent()); + auto fn = [&executor, finishEvent](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(finishEvent); + }; + + auto cb = executor.scheduleWorkAt(getNet()->now(), fn); + executor.waitForEvent(finishEvent); +} + +TEST_F(ReplicationExecutorTest, ScheduleCallbackAtAFutureTime) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + auto finishEvent = assertGet(executor.makeEvent()); + auto fn = [&executor, finishEvent](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(finishEvent); + }; + + auto now = getNet()->now(); + now += Milliseconds(1000); + auto cb = executor.scheduleWorkAt(now, fn); + + getNet()->enterNetwork(); + getNet()->runUntil(now); + getNet()->exitNetwork(); + + executor.waitForEvent(finishEvent); +} + + +TEST_F(ReplicationExecutorTest, TestForCancelRace) { + launchExecutorThread(); + getNet()->exitNetwork(); + + unittest::Barrier enterCallback(2U), runCallback(2U); + + ReplicationExecutor& executor = getReplExecutor(); + bool firstEventDone = false; + bool firstEventCanceled = false; + auto fn = [&executor, &enterCallback, &runCallback, &firstEventDone, &firstEventCanceled]( + const executor::TaskExecutor::RemoteCommandCallbackArgs& cbData) { + // This barrier lets the test code wait until we're in the callback. + enterCallback.countDownAndWait(); + // This barrier lets the test code keep us in the callback until it has run the cancel. + runCallback.countDownAndWait(); + firstEventCanceled = !cbData.response.getStatus().isOK(); + firstEventDone = true; + }; + + // First, schedule a network event to run. + const executor::RemoteCommandRequest request( + HostAndPort("test1", 1234), "mydb", BSON("nothing" << 0)); + auto firstCallback = assertGet(executor.scheduleRemoteCommand(request, fn)); + + // Now let the request happen. + // We need to run the network on another thread, because the test + // fixture will hang waiting for the callbacks to complete. + auto timeThread = stdx::thread([this] { + getNet()->enterNetwork(); + ASSERT(getNet()->hasReadyRequests()); + auto noi = getNet()->getNextReadyRequest(); + getNet()->scheduleResponse(noi, getNet()->now(), RemoteCommandResponse()); + getNet()->runReadyNetworkOperations(); + getNet()->exitNetwork(); + }); + + // Wait until we're in the callback. + enterCallback.countDownAndWait(); + + // Schedule a different network event to run. + bool secondEventDone = false; + bool secondEventCanceled = false; + auto fn2 = [&executor, &secondEventDone, &secondEventCanceled]( + const executor::TaskExecutor::RemoteCommandCallbackArgs& cbData) { + secondEventCanceled = !cbData.response.getStatus().isOK(); + secondEventDone = true; + }; + auto secondCallback = assertGet(executor.scheduleRemoteCommand(request, fn2)); + ASSERT_FALSE(firstEventDone); // The first event should be stuck at runCallback barrier. + // Cancel the first callback. This cancel should have no effect as the callback has + // already been started. + executor.cancel(firstCallback); + + // Let the first callback continue to completion. + runCallback.countDownAndWait(); + + // Now the time thread can continue. + timeThread.join(); + + // The first event should be done, the second event should be pending. + ASSERT(firstEventDone); + ASSERT_FALSE(secondEventDone); + + // Run the network thread, which should run the second request. + { + getNet()->enterNetwork(); + // The second request should be ready. + ASSERT(getNet()->hasReadyRequests()) << "Second request is not ready (cancelled?)"; + auto noi = getNet()->getNextReadyRequest(); + getNet()->scheduleResponse(noi, getNet()->now(), RemoteCommandResponse()); + getNet()->runReadyNetworkOperations(); + getNet()->exitNetwork(); + } + + // The second callback should have run without being canceled. + ASSERT_TRUE(secondEventDone); + ASSERT_FALSE(secondEventCanceled); +} + } // namespace } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/replset_commands.cpp b/src/mongo/db/repl/replset_commands.cpp index ee0aa50a6f0..0c202e906a1 100644 --- a/src/mongo/db/repl/replset_commands.cpp +++ b/src/mongo/db/repl/replset_commands.cpp @@ -58,6 +58,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/executor/network_interface.h" +#include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -664,7 +665,9 @@ public: int, string& errmsg, BSONObjBuilder& result) { - Status status = getGlobalReplicationCoordinator()->checkReplEnabledForCommand(&result); + auto replCoord = repl::ReplicationCoordinator::get(txn->getClient()->getServiceContext()); + + Status status = replCoord->checkReplEnabledForCommand(&result); if (!status.isOK()) return appendCommandStatus(result, status); @@ -673,6 +676,14 @@ public: if (cmdObj.hasField("handshake")) return true; + auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(cmdObj); + if (metadataResult.isOK()) { + // New style update position command has metadata, which may inform the + // upstream of a higher term. + auto metadata = metadataResult.getValue(); + replCoord->processReplSetMetadata(metadata, false /*don't advance the commit point*/); + } + // In the case of an update from a member with an invalid replica set config, // we return our current config version. long long configVersion = -1; @@ -681,9 +692,8 @@ public: status = args.initialize(cmdObj); if (status.isOK()) { - // v3.2.2+ style replSetUpdatePosition command. - status = getGlobalReplicationCoordinator()->processReplSetUpdatePosition( - args, &configVersion); + // v3.2.4+ style replSetUpdatePosition command. + status = replCoord->processReplSetUpdatePosition(args, &configVersion); if (status == ErrorCodes::InvalidReplicaSetConfig) { result.append("configVersion", configVersion); @@ -696,8 +706,7 @@ public: if (!status.isOK()) return appendCommandStatus(result, status); - status = getGlobalReplicationCoordinator()->processReplSetUpdatePosition( - oldArgs, &configVersion); + status = replCoord->processReplSetUpdatePosition(oldArgs, &configVersion); if (status == ErrorCodes::InvalidReplicaSetConfig) { result.append("configVersion", configVersion); diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index c05c5a7d69c..f860b2e005f 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -62,8 +62,10 @@ #include "mongo/db/repl/roll_back_local_operations.h" #include "mongo/db/repl/rollback_source.h" #include "mongo/db/repl/rslog.h" +#include "mongo/util/exit.h" #include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" +#include "mongo/util/scopeguard.h" /* Scenarios * @@ -119,59 +121,50 @@ namespace repl { // Failpoint which causes rollback to hang before finishing. MONGO_FP_DECLARE(rollbackHangBeforeFinish); +MONGO_FP_DECLARE(rollbackHangThenFailAfterWritingMinValid); -namespace { - -class RSFatalException : public std::exception { -public: - RSFatalException(std::string m = "replica set fatal exception") : msg(m) {} - virtual ~RSFatalException() throw(){}; - virtual const char* what() const throw() { - return msg.c_str(); - } - -private: - std::string msg; -}; - -struct DocID { - // ns and _id both point into ownedObj's buffer - BSONObj ownedObj; - const char* ns; - BSONElement _id; - bool operator<(const DocID& other) const { - int comp = strcmp(ns, other.ns); - if (comp < 0) - return true; - if (comp > 0) - return false; - return _id < other._id; - } -}; - -struct FixUpInfo { - // note this is a set -- if there are many $inc's on a single document we need to rollback, - // we only need to refetch it once. - set toRefetch; +using namespace rollback_internal; - // collections to drop - set toDrop; +bool DocID::operator<(const DocID& other) const { + int comp = strcmp(ns, other.ns); + if (comp < 0) + return true; + if (comp > 0) + return false; - // Indexes to drop. - // Key is collection namespace. Value is name of index to drop. - multimap indexesToDrop; + return _id < other._id; +} - set collectionsToResyncData; - set collectionsToResyncMetadata; +bool DocID::operator==(const DocID& other) const { + // Since this is only used for tests, going with the simple impl that reuses operator< which is + // used in the real code. + return !(*this < other || other < *this); +} - OpTime commonPoint; - RecordId commonPointOurDiskloc; +void FixUpInfo::removeAllDocsToRefetchFor(const std::string& collection) { + docsToRefetch.erase(docsToRefetch.lower_bound(DocID::minFor(collection.c_str())), + docsToRefetch.upper_bound(DocID::maxFor(collection.c_str()))); +} - int rbid; // remote server's current rollback sequence # -}; +void FixUpInfo::removeRedundantOperations() { + // These loops and their bodies can be done in any order. The final result of the FixUpInfo + // members will be the same either way. + for (const auto& collection : collectionsToDrop) { + removeAllDocsToRefetchFor(collection); + indexesToDrop.erase(collection); + collectionsToResyncMetadata.erase(collection); + } + for (const auto& collection : collectionsToResyncData) { + removeAllDocsToRefetchFor(collection); + indexesToDrop.erase(collection); + collectionsToResyncMetadata.erase(collection); + collectionsToDrop.erase(collection); + } +} -Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { +Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(FixUpInfo& fixUpInfo, + const BSONObj& ourObj) { const char* op = ourObj.getStringField("op"); if (*op == 'n') return Status::OK(); @@ -183,14 +176,14 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { doc.ownedObj = ourObj.getOwned(); doc.ns = doc.ownedObj.getStringField("ns"); if (*doc.ns == '\0') { - warning() << "ignoring op on rollback no ns TODO : " << doc.ownedObj.toString(); - return Status::OK(); + throw RSFatalException(str::stream() + << "local op on rollback has no ns: " << doc.ownedObj.toString()); } BSONObj obj = doc.ownedObj.getObjectField(*op == 'u' ? "o2" : "o"); if (obj.isEmpty()) { - warning() << "ignoring op on rollback : " << doc.ownedObj.toString(); - return Status::OK(); + throw RSFatalException(str::stream() + << "local op on rollback has no object field: " << (doc.ownedObj)); } if (*op == 'c') { @@ -208,7 +201,7 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { // Create collection operation // { ts: ..., h: ..., op: "c", ns: "foo.$cmd", o: { create: "abc", ... } } string ns = nss.db().toString() + '.' + obj["create"].String(); // -> foo.abc - fixUpInfo.toDrop.insert(ns); + fixUpInfo.collectionsToDrop.insert(ns); return Status::OK(); } else if (cmdname == "drop") { string ns = nss.db().toString() + '.' + first.valuestr(); @@ -270,7 +263,7 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { severe() << message; return Status(ErrorCodes::UnrecoverableRollbackError, message); } - auto subStatus = refetch(fixUpInfo, subopElement.Obj()); + auto subStatus = updateFixUpInfoFromLocalOplogEntry(fixUpInfo, subopElement.Obj()); if (!subStatus.isOK()) { return subStatus; } @@ -328,13 +321,57 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { throw RSFatalException(); } - fixUpInfo.toRefetch.insert(doc); + fixUpInfo.docsToRefetch.insert(doc); return Status::OK(); } +namespace { + +/** + * This must be called before making any changes to our local data and after fetching any + * information from the upstream node. If any information is fetched from the upstream node after we + * have written locally, the function must be called again. + */ +void checkRbidAndUpdateMinValid(OperationContext* txn, + const int rbid, + const RollbackSource& rollbackSource) { + // It is important that the steps are performed in order to avoid racing with upstream rollbacks + // + // 1) Get the last doc in their oplog. + // 2) Get their RBID and fail if it has changed. + // 3) Set our minValid to the previously fetched OpTime of the top of their oplog. + + const auto newMinValidDoc = rollbackSource.getLastOperation(); + if (newMinValidDoc.isEmpty()) { + uasserted(40361, "rollback error newest oplog entry on source is missing or empty"); + } + if (rbid != rollbackSource.getRollbackId()) { + // Our source rolled back itself so the data we received isn't necessarily consistent. + uasserted(40365, "rollback rbid on source changed during rollback, canceling this attempt"); + } + + // we have items we are writing that aren't from a point-in-time. thus best not to come + // online until we get to that point in freshness. + OpTime minValid = fassertStatusOK(28774, OpTime::parseFromOplogEntry(newMinValidDoc)); + log() << "Setting minvalid to " << minValid; + setAppliedThrough(txn, {}); // Use top of oplog. + setMinValid(txn, minValid); + + if (MONGO_FAIL_POINT(rollbackHangThenFailAfterWritingMinValid)) { + // This log output is used in js tests so please leave it. + log() << "rollback - rollbackHangThenFailAfterWritingMinValid fail point " + "enabled. Blocking until fail point is disabled."; + while (MONGO_FAIL_POINT(rollbackHangThenFailAfterWritingMinValid)) { + invariant(!inShutdown()); // It is an error to shutdown while enabled. + mongo::sleepsecs(1); + } + uasserted(40378, + "failing rollback due to rollbackHangThenFailAfterWritingMinValid fail point"); + } +} void syncFixUp(OperationContext* txn, - FixUpInfo& fixUpInfo, + const FixUpInfo& fixUpInfo, const RollbackSource& rollbackSource, ReplicationCoordinator* replCoord) { // fetch all first so we needn't handle interruption in a fancy way @@ -344,73 +381,45 @@ void syncFixUp(OperationContext* txn, // namespace -> doc id -> doc map> goodVersions; - BSONObj newMinValid; - // fetch all the goodVersions of each document from current primary - DocID doc; unsigned long long numFetched = 0; - try { - for (set::iterator it = fixUpInfo.toRefetch.begin(); it != fixUpInfo.toRefetch.end(); - it++) { - doc = *it; - - verify(!doc._id.eoo()); + for (auto&& doc : fixUpInfo.docsToRefetch) { + invariant(!doc._id.eoo()); // This is checked when we insert to the set. - { - // TODO : slow. lots of round trips. - numFetched++; - BSONObj good = rollbackSource.findOne(NamespaceString(doc.ns), doc._id.wrap()); - totalSize += good.objsize(); - uassert(13410, "replSet too much data to roll back", totalSize < 300 * 1024 * 1024); - - // note good might be eoo, indicating we should delete it - goodVersions[doc.ns][doc] = good; + try { + // TODO : slow. lots of round trips. + numFetched++; + BSONObj good = rollbackSource.findOne(NamespaceString(doc.ns), doc._id.wrap()); + totalSize += good.objsize(); + if (totalSize >= 300 * 1024 * 1024) { + throw RSFatalException("replSet too much data to roll back"); } + + // Note good might be empty, indicating we should delete it. + goodVersions[doc.ns][doc] = good; + } catch (const DBException& ex) { + log() << "rollback couldn't re-get from ns: " << doc.ns << " _id: " << (doc._id) << ' ' + << numFetched << '/' << fixUpInfo.docsToRefetch.size() << ": " << (ex); + throw; } - newMinValid = rollbackSource.getLastOperation(); - if (newMinValid.isEmpty()) { - error() << "rollback error newMinValid empty?"; - return; - } - } catch (const DBException& e) { - LOG(1) << "rollback re-get objects: " << e.toString(); - error() << "rollback couldn't re-get ns:" << doc.ns << " _id:" << doc._id << ' ' - << numFetched << '/' << fixUpInfo.toRefetch.size(); - throw e; } log() << "rollback 3.5"; - if (fixUpInfo.rbid != rollbackSource.getRollbackId()) { - // Our source rolled back itself so the data we received isn't necessarily consistent. - warning() << "rollback rbid on source changed during rollback, " - << "cancelling this attempt"; - return; - } + checkRbidAndUpdateMinValid(txn, fixUpInfo.rbid, rollbackSource); // update them log() << "rollback 4 n:" << goodVersions.size(); - bool warn = false; - invariant(!fixUpInfo.commonPointOurDiskloc.isNull()); - // we have items we are writing that aren't from a point-in-time. thus best not to come - // online until we get to that point in freshness. - // TODO this is still wrong because we don't record that we are in rollback, and we can't really - // recover. - OpTime minValid = fassertStatusOK(28774, OpTime::parseFromOplogEntry(newMinValid)); - log() << "minvalid=" << minValid; - setAppliedThrough(txn, {}); // Use top of oplog. - setMinValid(txn, minValid); - // any full collection resyncs required? if (!fixUpInfo.collectionsToResyncData.empty() || !fixUpInfo.collectionsToResyncMetadata.empty()) { for (const string& ns : fixUpInfo.collectionsToResyncData) { log() << "rollback 4.1.1 coll resync " << ns; - fixUpInfo.indexesToDrop.erase(ns); - fixUpInfo.collectionsToResyncMetadata.erase(ns); + invariant(!fixUpInfo.indexesToDrop.count(ns)); + invariant(!fixUpInfo.collectionsToResyncMetadata.count(ns)); const NamespaceString nss(ns); @@ -421,7 +430,7 @@ void syncFixUp(OperationContext* txn, Database* db = dbHolder().openDb(txn, nss.db().toString()); invariant(db); WriteUnitOfWork wunit(txn); - db->dropCollection(txn, ns); + fassertStatusOK(40359, db->dropCollectionEvenIfSystem(txn, nss)); wunit.commit(); } @@ -443,9 +452,11 @@ void syncFixUp(OperationContext* txn, auto infoResult = rollbackSource.getCollectionInfo(nss); if (!infoResult.isOK()) { - // Collection dropped by "them" so we should drop it too. - log() << ns << " not found on remote host, dropping"; - fixUpInfo.toDrop.insert(ns); + // Collection dropped by "them" so we can't correctly change it here. If we get to + // the roll-forward phase, we will drop it then. If the drop is rolled-back upstream + // and we restart, we will be expected to still have the collection. + log() << ns << " not found on remote host, so not rolling back collmod operation." + " We will drop the collection soon."; continue; } @@ -496,56 +507,28 @@ void syncFixUp(OperationContext* txn, // we did more reading from primary, so check it again for a rollback (which would mess // us up), and make minValid newer. log() << "rollback 4.2"; - - string err; - try { - newMinValid = rollbackSource.getLastOperation(); - if (newMinValid.isEmpty()) { - err = "can't get minvalid from sync source"; - } else { - OpTime minValid = fassertStatusOK(28775, OpTime::parseFromOplogEntry(newMinValid)); - log() << "minvalid=" << minValid; - setMinValid(txn, minValid); - setAppliedThrough(txn, fixUpInfo.commonPoint); - } - } catch (const DBException& e) { - err = "can't get/set minvalid: "; - err += e.what(); - } - if (fixUpInfo.rbid != rollbackSource.getRollbackId()) { - // our source rolled back itself. so the data we received isn't necessarily - // consistent. however, we've now done writes. thus we have a problem. - err += "rbid at primary changed during resync/rollback"; - } - if (!err.empty()) { - severe() << "rolling back : " << err << ". A full resync will be necessary."; - // TODO: reset minvalid so that we are permanently in fatal state - // TODO: don't be fatal, but rather, get all the data first. - throw RSFatalException(); - } - log() << "rollback 4.3"; + checkRbidAndUpdateMinValid(txn, fixUpInfo.rbid, rollbackSource); } log() << "rollback 4.6"; - // drop collections to drop before doing individual fixups - that might make things faster - // below actually if there were subsequent inserts to rollback - for (set::iterator it = fixUpInfo.toDrop.begin(); it != fixUpInfo.toDrop.end(); it++) { + // drop collections to drop before doing individual fixups + for (set::iterator it = fixUpInfo.collectionsToDrop.begin(); + it != fixUpInfo.collectionsToDrop.end(); + it++) { log() << "rollback drop: " << *it; - fixUpInfo.indexesToDrop.erase(*it); + invariant(!fixUpInfo.indexesToDrop.count(*it)); ScopedTransaction transaction(txn, MODE_IX); const NamespaceString nss(*it); Lock::DBLock dbLock(txn->lockState(), nss.db(), MODE_X); Database* db = dbHolder().get(txn, nsToDatabaseSubstring(*it)); if (db) { - WriteUnitOfWork wunit(txn); - Helpers::RemoveSaver removeSaver("rollback", "", *it); // perform a collection scan and write all documents in the collection to disk std::unique_ptr exec(InternalPlanner::collectionScan( - txn, *it, db->getCollection(*it), PlanExecutor::YIELD_MANUAL)); + txn, *it, db->getCollection(*it), PlanExecutor::YIELD_AUTO)); BSONObj curObj; PlanExecutor::ExecState execState; while (PlanExecutor::ADVANCED == (execState = exec->getNext(&curObj, NULL))) { @@ -570,7 +553,8 @@ void syncFixUp(OperationContext* txn, throw RSFatalException(); } - db->dropCollection(txn, *it); + WriteUnitOfWork wunit(txn); + fassertStatusOK(40360, db->dropCollectionEvenIfSystem(txn, nss)); wunit.commit(); } } @@ -622,9 +606,8 @@ void syncFixUp(OperationContext* txn, // while rolling back createCollection operations. const auto& ns = nsAndGoodVersionsByDocID.first; unique_ptr removeSaver; - if (!fixUpInfo.toDrop.count(ns)) { - removeSaver.reset(new Helpers::RemoveSaver("rollback", "", ns)); - } + invariant(!fixUpInfo.collectionsToDrop.count(ns)); + removeSaver.reset(new Helpers::RemoveSaver("rollback", "", ns)); const auto& goodVersionsByDocID = nsAndGoodVersionsByDocID.second; for (const auto& idAndDoc : goodVersionsByDocID) { @@ -639,10 +622,7 @@ void syncFixUp(OperationContext* txn, BSONObj pattern = doc._id.wrap(); // { _id : ... } try { verify(doc.ns && *doc.ns); - if (fixUpInfo.collectionsToResyncData.count(doc.ns)) { - // We just synced this entire collection. - continue; - } + invariant(!fixUpInfo.collectionsToResyncData.count(doc.ns)); // TODO: Lots of overhead in context. This can be faster. const NamespaceString docNss(doc.ns); @@ -715,8 +695,13 @@ void syncFixUp(OperationContext* txn, } } } catch (const DBException& e) { - error() << "rolling back capped collection rec " << doc.ns << ' ' - << e.toString(); + // Replicated capped collections have many ways to become + // inconsistent. We rely on age-out to make these problems go away + // eventually. + warning() << "ignoring failure to roll back change to capped " + << "collection " << doc.ns << " with _id " + << idAndDoc.first._id.toString( + /*includeFieldName*/ false) << ": " << e; } } else { deleteObjects(txn, @@ -727,25 +712,6 @@ void syncFixUp(OperationContext* txn, true, // justone true); // god } - // did we just empty the collection? if so let's check if it even - // exists on the source. - if (collection->numRecords(txn) == 0) { - try { - NamespaceString nss(doc.ns); - auto infoResult = rollbackSource.getCollectionInfo(nss); - if (!infoResult.isOK()) { - // we should drop - WriteUnitOfWork wunit(txn); - ctx.db()->dropCollection(txn, doc.ns); - wunit.commit(); - } - } catch (const DBException& ex) { - // Failed to run listCollections command on sync source. - // This isn't *that* big a deal, but is bad. - warning() << "rollback error querying for existence of " << doc.ns - << " at the primary, ignoring: " << ex; - } - } } } else { // TODO faster... @@ -766,8 +732,8 @@ void syncFixUp(OperationContext* txn, } } catch (const DBException& e) { log() << "exception in rollback ns:" << doc.ns << ' ' << pattern.toString() << ' ' - << e.toString() << " ndeletes:" << deletes; - warn = true; + << e << " ndeletes:" << deletes; + throw; } } } @@ -795,113 +761,80 @@ void syncFixUp(OperationContext* txn, Status status = getGlobalAuthorizationManager()->initialize(txn); if (!status.isOK()) { - warning() << "Failed to reinitialize auth data after rollback: " << status; - warn = true; + severe() << "Failed to reinitialize auth data after rollback: " << status; + fassertFailedNoTrace(40366); } // Reload the lastAppliedOpTime and lastDurableOpTime value in the replcoord and the // lastAppliedHash value in bgsync to reflect our new last op. replCoord->resetLastOpTimesFromOplog(txn); - - // done - if (warn) - warning() << "issues during syncRollback, see log"; - else - log() << "rollback done"; + log() << "rollback done"; } Status _syncRollback(OperationContext* txn, const OplogInterface& localOplog, const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord, - const SleepSecondsFn& sleepSecondsFn) { + boost::optional requiredRBID, + ReplicationCoordinator* replCoord) { invariant(!txn->lockState()->isLocked()); - log() << "rollback 0"; - - /** by doing this, we will not service reads (return an error as we aren't in secondary - * state. that perhaps is moot because of the write lock above, but that write lock - * probably gets deferred or removed or yielded later anyway. - * - * also, this is better for status reporting - we know what is happening. - */ - { - Lock::GlobalWrite globalWrite(txn->lockState()); - if (!replCoord->setFollowerMode(MemberState::RS_ROLLBACK)) { - return Status(ErrorCodes::OperationFailed, - str::stream() << "Cannot transition from " - << replCoord->getMemberState().toString() << " to " - << MemberState(MemberState::RS_ROLLBACK).toString()); - } - } - FixUpInfo how; log() << "rollback 1"; how.rbid = rollbackSource.getRollbackId(); - { - log() << "rollback 2 FindCommonPoint"; - try { - auto processOperationForFixUp = - [&how](const BSONObj& operation) { return refetch(how, operation); }; - auto res = syncRollBackLocalOperations( - localOplog, rollbackSource.getOplog(), processOperationForFixUp); - if (!res.isOK()) { - const auto status = res.getStatus(); - switch (status.code()) { - case ErrorCodes::OplogStartMissing: - case ErrorCodes::UnrecoverableRollbackError: - sleepSecondsFn(Seconds(1)); - return status; - default: - throw RSFatalException(status.toString()); - } - } else { - how.commonPoint = res.getValue().first; - how.commonPointOurDiskloc = res.getValue().second; + if (requiredRBID) { + uassert(40362, + "Upstream node rolled back. Need to retry our rollback.", + how.rbid == *requiredRBID); + } + + log() << "rollback 2 FindCommonPoint"; + try { + auto processOperationForFixUp = [&how](const BSONObj& operation) { + return updateFixUpInfoFromLocalOplogEntry(how, operation); + }; + auto res = syncRollBackLocalOperations( + localOplog, rollbackSource.getOplog(), processOperationForFixUp); + if (!res.isOK()) { + const auto status = res.getStatus(); + switch (status.code()) { + case ErrorCodes::OplogStartMissing: + case ErrorCodes::UnrecoverableRollbackError: + return status; + default: + throw RSFatalException(status.toString()); } - } catch (const RSFatalException& e) { - error() << string(e.what()); - return Status(ErrorCodes::UnrecoverableRollbackError, - str::stream() - << "need to rollback, but unable to determine common point between" - " local and remote oplog: " << e.what(), - 18752); - } catch (const DBException& e) { - warning() << "rollback 2 exception " << e.toString() << "; sleeping 1 min"; - - sleepSecondsFn(Seconds(60)); - throw; } + + how.commonPoint = res.getValue().first; + how.commonPointOurDiskloc = res.getValue().second; + how.removeRedundantOperations(); + } catch (const RSFatalException& e) { + return Status(ErrorCodes::UnrecoverableRollbackError, + str::stream() + << "need to rollback, but unable to determine common point between" + " local and remote oplog: " << e.what(), + 18752); } + log() << "rollback common point is " << how.commonPoint; log() << "rollback 3 fixup"; - - replCoord->incrementRollbackID(); try { + ON_BLOCK_EXIT([&] { replCoord->incrementRollbackID(); }); syncFixUp(txn, how, rollbackSource, replCoord); } catch (const RSFatalException& e) { - error() << "exception during rollback: " << e.what(); - return Status(ErrorCodes::UnrecoverableRollbackError, - str::stream() << "exception during rollback: " << e.what(), - 18753); - } catch (...) { - replCoord->incrementRollbackID(); - - throw; + return Status(ErrorCodes::UnrecoverableRollbackError, e.what(), 18753); } - replCoord->incrementRollbackID(); if (MONGO_FAIL_POINT(rollbackHangBeforeFinish)) { // This log output is used in js tests so please leave it. log() << "rollback - rollbackHangBeforeFinish fail point " "enabled. Blocking until fail point is disabled."; while (MONGO_FAIL_POINT(rollbackHangBeforeFinish)) { + invariant(!inShutdown()); // It is an error to shutdown while enabled. mongo::sleepsecs(1); } } - // Success; leave "ROLLBACK" state intact until applier thread has reloaded the new minValid. - // Otherwise, the applier could transition the node to SECONDARY with an out-of-date minValid. return Status::OK(); } @@ -910,8 +843,8 @@ Status _syncRollback(OperationContext* txn, Status syncRollback(OperationContext* txn, const OplogInterface& localOplog, const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord, - const SleepSecondsFn& sleepSecondsFn) { + boost::optional requiredRBID, + ReplicationCoordinator* replCoord) { invariant(txn); invariant(replCoord); @@ -919,22 +852,11 @@ Status syncRollback(OperationContext* txn, DisableDocumentValidation validationDisabler(txn); txn->setReplicatedWrites(false); - Status status = _syncRollback(txn, localOplog, rollbackSource, replCoord, sleepSecondsFn); + Status status = _syncRollback(txn, localOplog, rollbackSource, requiredRBID, replCoord); log() << "rollback finished" << rsLog; return status; } -Status syncRollback(OperationContext* txn, - const OplogInterface& localOplog, - const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord) { - return syncRollback(txn, - localOplog, - rollbackSource, - replCoord, - [](Seconds seconds) { sleepsecs(durationCount(seconds)); }); -} - } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/rs_rollback.h b/src/mongo/db/repl/rs_rollback.h index c88e9fd27c0..8ee7dd04367 100644 --- a/src/mongo/db/repl/rs_rollback.h +++ b/src/mongo/db/repl/rs_rollback.h @@ -28,12 +28,10 @@ #pragma once -#include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" -#include "mongo/base/status_with.h" #include "mongo/db/jsobj.h" -#include "mongo/stdx/functional.h" -#include "mongo/util/time_support.h" +#include "mongo/db/record_id.h" +#include "mongo/db/repl/optime.h" namespace mongo { @@ -44,7 +42,6 @@ class OperationContext; namespace repl { class OplogInterface; -class OpTime; class ReplicationCoordinator; class RollbackSource; @@ -58,7 +55,7 @@ class RollbackSource; * - undo operations by fetching all documents affected, then replaying * the sync source's oplog until we reach the time in the oplog when we fetched the last * document. - * This function can throw std::exception on failures. + * This function can throw exceptions on failures. * This function runs a command on the sync source to detect if the sync source rolls back * while our rollback is in progress. * @@ -69,21 +66,77 @@ class RollbackSource; * supports fetching documents and copying collections. * @param replCoord Used to track the rollback ID and to change the follower state * - * Failures: Most failures are returned as a status but some failures throw an std::exception. + * If requiredRBID is supplied, we error if the upstream node has a different RBID (ie it rolled + * back) after fetching any information from it. + * + * Failures: If a Status with code UnrecoverableRollbackError is returned, the caller must exit + * fatally. All other errors should be considered recoverable regardless of whether reported as a + * status or exception. */ - -using SleepSecondsFn = stdx::function; - -Status syncRollback(OperationContext* txn, - const OplogInterface& localOplog, - const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord, - const SleepSecondsFn& sleepSecondsFn); - Status syncRollback(OperationContext* txn, const OplogInterface& localOplog, const RollbackSource& rollbackSource, + boost::optional requiredRBID, ReplicationCoordinator* replCoord); +/** + * This namespace contains internal details of the rollback system. It is only exposed in a header + * for unittesting. Nothing here should be used outside of rs_rollback.cpp or its unittest. + */ +namespace rollback_internal { + +struct DocID { + BSONObj ownedObj; + const char* ns; + BSONElement _id; + bool operator<(const DocID& other) const; + bool operator==(const DocID& other) const; + + static DocID minFor(const char* ns) { + auto obj = BSON("" << MINKEY); + return {obj, ns, obj.firstElement()}; + } + + static DocID maxFor(const char* ns) { + auto obj = BSON("" << MAXKEY); + return {obj, ns, obj.firstElement()}; + } +}; + +struct FixUpInfo { + // note this is a set -- if there are many $inc's on a single document we need to rollback, + // we only need to refetch it once. + std::set docsToRefetch; + + // Key is collection namespace. Value is name of index to drop. + std::multimap indexesToDrop; + + std::set collectionsToDrop; + std::set collectionsToResyncData; + std::set collectionsToResyncMetadata; + + OpTime commonPoint; + RecordId commonPointOurDiskloc; + + int rbid; // remote server's current rollback sequence # + + void removeAllDocsToRefetchFor(const std::string& collection); + void removeRedundantOperations(); +}; + +// Indicates that rollback cannot complete and the server must abort. +class RSFatalException : public std::exception { +public: + RSFatalException(std::string m = "replica set fatal exception") : msg(m) {} + virtual const char* what() const throw() { + return msg.c_str(); + } + +private: + std::string msg; +}; + +Status updateFixUpInfoFromLocalOplogEntry(FixUpInfo& fixUpInfo, const BSONObj& ourObj); +} // namespace rollback_internal } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/rs_rollback_test.cpp b/src/mongo/db/repl/rs_rollback_test.cpp index 3f768d79f4b..3a324aff179 100644 --- a/src/mongo/db/repl/rs_rollback_test.cpp +++ b/src/mongo/db/repl/rs_rollback_test.cpp @@ -33,6 +33,7 @@ #include #include +#include "mongo/bson/json.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" @@ -63,6 +64,7 @@ namespace { using namespace mongo; using namespace mongo::repl; +using namespace mongo::repl::rollback_internal; const OplogInterfaceMock::Operations kEmptyMockOperations; @@ -173,8 +175,6 @@ void RSRollbackTest::tearDown() { setGlobalReplicationCoordinator(nullptr); } -void noSleep(Seconds seconds) {} - TEST_F(RSRollbackTest, InconsistentMinValid) { repl::setAppliedThrough(_txn.get(), OpTime(Timestamp(Seconds(0), 0), 0)); repl::setMinValid(_txn.get(), OpTime(Timestamp(Seconds(1), 0), 0)); @@ -182,36 +182,12 @@ TEST_F(RSRollbackTest, InconsistentMinValid) { OplogInterfaceMock(kEmptyMockOperations), RollbackSourceMock(std::unique_ptr( new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); } -TEST_F(RSRollbackTest, SetFollowerModeFailed) { - class ReplicationCoordinatorSetFollowerModeMock : public ReplicationCoordinatorMock { - public: - ReplicationCoordinatorSetFollowerModeMock() - : ReplicationCoordinatorMock(createReplSettings()) {} - MemberState getMemberState() const override { - return MemberState::RS_DOWN; - } - bool setFollowerMode(const MemberState& newState) override { - return false; - } - }; - _coordinator = new ReplicationCoordinatorSetFollowerModeMock(); - setGlobalReplicationCoordinator(_coordinator); - - ASSERT_EQUALS(ErrorCodes::OperationFailed, - syncRollback(_txn.get(), - OplogInterfaceMock(kEmptyMockOperations), - RollbackSourceMock(std::unique_ptr( - new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep).code()); -} - TEST_F(RSRollbackTest, OplogStartMissing) { OpTime ts(Timestamp(Seconds(1), 0), 0); auto operation = @@ -223,8 +199,8 @@ TEST_F(RSRollbackTest, OplogStartMissing) { RollbackSourceMock(std::unique_ptr(new OplogInterfaceMock({ operation, }))), - _coordinator, - noSleep).code()); + {}, + _coordinator).code()); } TEST_F(RSRollbackTest, NoRemoteOpLog) { @@ -235,8 +211,8 @@ TEST_F(RSRollbackTest, NoRemoteOpLog) { OplogInterfaceMock({operation}), RollbackSourceMock(std::unique_ptr( new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); } @@ -257,12 +233,36 @@ TEST_F(RSRollbackTest, RemoteGetRollbackIdThrows) { OplogInterfaceMock({operation}), RollbackSourceLocal(std::unique_ptr( new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep), + {}, + _coordinator), UserException, ErrorCodes::UnknownError); } +TEST_F(RSRollbackTest, RemoteGetRollbackIdDiffersFromRequiredRBID) { + OpTime ts(Timestamp(Seconds(1), 0), 0); + auto operation = + std::make_pair(BSON("ts" << ts.getTimestamp() << "h" << ts.getTerm()), RecordId()); + + class RollbackSourceLocal : public RollbackSourceMock { + public: + RollbackSourceLocal(std::unique_ptr oplog) + : RollbackSourceMock(std::move(oplog)) {} + int getRollbackId() const override { + return 2; + } + }; + + ASSERT_THROWS_CODE(syncRollback(_txn.get(), + OplogInterfaceMock({operation}), + RollbackSourceLocal(std::unique_ptr( + new OplogInterfaceMock(kEmptyMockOperations))), + {1}, + _coordinator), + UserException, + ErrorCodes::Error(40362)); +} + TEST_F(RSRollbackTest, BothOplogsAtCommonPoint) { createOplog(_txn.get()); OpTime ts(Timestamp(Seconds(1), 0), 1); @@ -274,8 +274,8 @@ TEST_F(RSRollbackTest, BothOplogsAtCommonPoint) { RollbackSourceMock(std::unique_ptr(new OplogInterfaceMock({ operation, }))), - _coordinator, - noSleep)); + {}, + _coordinator)); } /** @@ -341,8 +341,8 @@ int _testRollbackDelete(OperationContext* txn, ASSERT_OK(syncRollback(txn, OplogInterfaceMock({deleteOperation, commonOperation}), rollbackSource, - coordinator, - noSleep)); + {}, + coordinator)); ASSERT_TRUE(rollbackSource.called); Lock::DBLock dbLock(txn->lockState(), "test", MODE_S); @@ -415,8 +415,8 @@ TEST_F(RSRollbackTest, RollbackInsertDocumentWithNoId) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -474,8 +474,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommand) { _txn.get(), OplogInterfaceMock({insertDocumentOperation, insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("rollback drop index: collection: test.t. index: a_1")); @@ -531,8 +531,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandIndexNotInCatalog) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("rollback drop index: collection: test.t. index: a_1")); @@ -578,8 +578,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandMissingNamespace) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -622,8 +622,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandInvalidNamespace) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -665,8 +665,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandMissingIndexName) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -700,8 +700,8 @@ TEST_F(RSRollbackTest, RollbackUnknownCommand) { RollbackSourceMock(std::unique_ptr(new OplogInterfaceMock({ commonOperation, }))), - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18751, status.location()); } @@ -735,11 +735,51 @@ TEST_F(RSRollbackTest, RollbackDropCollectionCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({dropCollectionOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); ASSERT_TRUE(rollbackSource.called); } +TEST_F(RSRollbackTest, RollbackDropCollectionCommandFailsIfRBIDChangesWhileSyncingCollection) { + createOplog(_txn.get()); + auto commonOperation = + std::make_pair(BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL), RecordId(1)); + auto dropCollectionOperation = + std::make_pair(BSON("ts" << Timestamp(Seconds(2), 0) << "h" << 1LL << "op" + << "c" + << "ns" + << "test.t" + << "o" << BSON("drop" + << "t")), + RecordId(2)); + class RollbackSourceLocal : public RollbackSourceMock { + public: + RollbackSourceLocal(std::unique_ptr oplog) + : RollbackSourceMock(std::move(oplog)), copyCollectionCalled(false) {} + int getRollbackId() const override { + return copyCollectionCalled ? 1 : 0; + } + void copyCollectionFromRemote(OperationContext* txn, + const NamespaceString& nss) const override { + copyCollectionCalled = true; + } + mutable bool copyCollectionCalled; + }; + RollbackSourceLocal rollbackSource(std::unique_ptr(new OplogInterfaceMock({ + commonOperation, + }))); + + _createCollection(_txn.get(), "test.t", CollectionOptions()); + ASSERT_THROWS_CODE(syncRollback(_txn.get(), + OplogInterfaceMock({dropCollectionOperation, commonOperation}), + rollbackSource, + 0, + _coordinator), + DBException, + 40365); + ASSERT(rollbackSource.copyCollectionCalled); +} + BSONObj makeApplyOpsOplogEntry(Timestamp ts, std::initializer_list ops) { BSONObjBuilder entry; entry << "ts" << ts << "h" << 1LL << "op" @@ -848,8 +888,8 @@ TEST_F(RSRollbackTest, RollbackApplyOpsCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({applyOpsOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); ASSERT_EQUALS(4U, rollbackSource.searchedIds.size()); ASSERT_EQUALS(1U, rollbackSource.searchedIds.count(1)); ASSERT_EQUALS(1U, rollbackSource.searchedIds.count(2)); @@ -887,8 +927,8 @@ TEST_F(RSRollbackTest, RollbackCreateCollectionCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({createCollectionOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); { Lock::DBLock dbLock(_txn->lockState(), "test", MODE_S); auto db = dbHolder().get(_txn.get(), "test"); @@ -928,8 +968,8 @@ TEST_F(RSRollbackTest, RollbackCollectionModificationCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({collectionModificationOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); stopCapturingLogMessages(); ASSERT_TRUE(rollbackSource.called); for (const auto& message : getCapturedLogMessages()) { @@ -967,10 +1007,89 @@ TEST_F(RSRollbackTest, RollbackCollectionModificationCommandInvalidCollectionOpt syncRollback(_txn.get(), OplogInterfaceMock({collectionModificationOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18753, status.location()); } +TEST(RSRollbackTest, LocalEntryWithoutNsIsFatal) { + const auto validOplogEntry = fromjson("{op: 'i', ns: 'test.t', o: {_id:1, a: 1}}"); + FixUpInfo fui; + ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry)); + ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry.removeField("ns")), + RSFatalException); +} + +TEST(RSRollbackTest, LocalEntryWithoutOIsFatal) { + const auto validOplogEntry = fromjson("{op: 'i', ns: 'test.t', o: {_id:1, a: 1}}"); + FixUpInfo fui; + ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry)); + ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry.removeField("o")), + RSFatalException); +} + +TEST(RSRollbackTest, LocalEntryWithoutO2IsFatal) { + const auto validOplogEntry = + fromjson("{op: 'u', ns: 'test.t', o2: {_id: 1}, o: {_id:1, a: 1}}"); + FixUpInfo fui; + ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry)); + ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry.removeField("o2")), + RSFatalException); +} + +// The testcases used here are trying to detect off-by-one errors in +// FixUpInfo::removeAllDocsToRefectchFor. +TEST(FixUpInfoTest, RemoveAllDocsToRefetchForWorks) { + const auto normalHolder = BSON("" << OID::gen()); + const auto normalKey = normalHolder.firstElement(); + + // Can't use ASSERT_EQ with this since it isn't ostream-able. Failures will at least give you + // the size. If that isn't enough, use GDB. + using DocSet = std::set; + + FixUpInfo fui; + fui.docsToRefetch = { + DocID::minFor("a"), + DocID{{}, "a", normalKey}, + DocID::maxFor("a"), + + DocID::minFor("b"), + DocID{{}, "b", normalKey}, + DocID::maxFor("b"), + + DocID::minFor("c"), + DocID{{}, "c", normalKey}, + DocID::maxFor("c"), + }; + + // Remove from the middle. + fui.removeAllDocsToRefetchFor("b"); + ASSERT((fui.docsToRefetch == + DocSet{ + DocID::minFor("a"), + DocID{{}, "a", normalKey}, + DocID::maxFor("a"), + + DocID::minFor("c"), + DocID{{}, "c", normalKey}, + DocID::maxFor("c"), + })) + << "remaining docs: " << fui.docsToRefetch.size(); + + // Remove from the end. + fui.removeAllDocsToRefetchFor("c"); + ASSERT((fui.docsToRefetch == + DocSet{ + DocID::minFor("a"), // This comment helps clang-format. + DocID{{}, "a", normalKey}, + DocID::maxFor("a"), + })) + << "remaining docs: " << fui.docsToRefetch.size(); + + // Everything else. + fui.removeAllDocsToRefetchFor("a"); + ASSERT((fui.docsToRefetch == DocSet{})) << "remaining docs: " << fui.docsToRefetch.size(); +} + } // namespace diff --git a/src/mongo/db/repl/sync_source_feedback.cpp b/src/mongo/db/repl/sync_source_feedback.cpp index 43b84f45e38..beb73bebbdd 100644 --- a/src/mongo/db/repl/sync_source_feedback.cpp +++ b/src/mongo/db/repl/sync_source_feedback.cpp @@ -113,17 +113,14 @@ Status SyncSourceFeedback::updateUpstream(OperationContext* txn, bool oldStyle) return Status::OK(); } BSONObjBuilder cmd; - { - stdx::unique_lock lock(_mtx); - // The command could not be created, likely because this node was removed from the set. - if (!oldStyle) { - if (!replCoord->prepareReplSetUpdatePositionCommand(&cmd)) { - return Status::OK(); - } - } else { - if (!replCoord->prepareOldReplSetUpdatePositionCommand(&cmd)) { - return Status::OK(); - } + // The command could not be created, likely because this node was removed from the set. + if (!oldStyle) { + if (!replCoord->prepareReplSetUpdatePositionCommand(&cmd)) { + return Status::OK(); + } + } else { + if (!replCoord->prepareOldReplSetUpdatePositionCommand(&cmd)) { + return Status::OK(); } } BSONObj res; diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index a336c8de68d..6c4d88132eb 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -255,7 +255,7 @@ bool SyncTail::peek(BSONObj* op) { // static Status SyncTail::syncApply(OperationContext* txn, const BSONObj& op, - bool convertUpdateToUpsert, + bool inSteadyStateReplication, ApplyOperationInLockFn applyOperationInLock, ApplyCommandInLockFn applyCommandInLock, IncrementOpsAppliedStatsFn incrementOpsAppliedStats) { @@ -291,7 +291,7 @@ Status SyncTail::syncApply(OperationContext* txn, Lock::GlobalWrite globalWriteLock(txn->lockState()); // special case apply for commands to avoid implicit database creation - Status status = applyCommandInLock(txn, op); + Status status = applyCommandInLock(txn, op, inSteadyStateReplication); incrementOpsAppliedStats(); return status; } @@ -304,7 +304,7 @@ Status SyncTail::syncApply(OperationContext* txn, txn->setReplicatedWrites(false); DisableDocumentValidation validationDisabler(txn); - Status status = applyOperationInLock(txn, db, op, convertUpdateToUpsert); + Status status = applyOperationInLock(txn, db, op, inSteadyStateReplication); if (!status.isOK() && status.code() == ErrorCodes::WriteConflict) { throw WriteConflictException(); } @@ -333,20 +333,24 @@ Status SyncTail::syncApply(OperationContext* txn, auto resetLocks = [&](LockMode mode) { collectionLock.reset(); + // Warning: We must reset the pointer to nullptr first, in order to ensure that we + // drop the DB lock before acquiring + // the upgraded one. + dbLock.reset(); dbLock.reset(new Lock::DBLock(txn->lockState(), dbName, mode)); collectionLock.reset(new Lock::CollectionLock(txn->lockState(), ns, mode)); }; resetLocks(MODE_IX); if (!dbHolder().get(txn, dbName)) { - // need to create database, try again + // Need to create database, so reset lock to stronger mode. resetLocks(MODE_X); ctx.reset(new OldClientContext(txn, ns)); } else { ctx.reset(new OldClientContext(txn, ns)); if (!ctx->db()->getCollection(ns)) { - // uh, oh, we need to create collection - // try again + // Need to implicitly create collection. This occurs for 'u' opTypes, + // but not for 'i' nor 'd'. ctx.reset(); resetLocks(MODE_X); ctx.reset(new OldClientContext(txn, ns)); @@ -365,10 +369,12 @@ Status SyncTail::syncApply(OperationContext* txn, return Status(ErrorCodes::BadValue, ss); } -Status SyncTail::syncApply(OperationContext* txn, const BSONObj& op, bool convertUpdateToUpsert) { +Status SyncTail::syncApply(OperationContext* txn, + const BSONObj& op, + bool inSteadyStateReplication) { return syncApply(txn, op, - convertUpdateToUpsert, + inSteadyStateReplication, applyOperation_inlock, applyCommand_inlock, stdx::bind(&Counter64::increment, &opsAppliedStats, 1ULL)); @@ -985,11 +991,12 @@ void multiSyncApply(const std::vector& ops, SyncTail* st) { // allow us to get through the magic barrier txn.lockState()->setIsBatchWriter(true); - bool convertUpdatesToUpserts = true; + // This function is only called in steady state replication. + bool inSteadyStateReplication = true; for (std::vector::const_iterator it = ops.begin(); it != ops.end(); ++it) { try { - const Status s = SyncTail::syncApply(&txn, *it, convertUpdatesToUpserts); + const Status s = SyncTail::syncApply(&txn, *it, inSteadyStateReplication); if (!s.isOK()) { severe() << "Error applying operation (" << it->toString() << "): " << s; fassertFailedNoTrace(16359); @@ -1012,23 +1019,39 @@ void multiInitialSyncApply(const std::vector& ops, SyncTail* st) { initializeWriterThread(); OperationContextImpl txn; - txn.setReplicatedWrites(false); - DisableDocumentValidation validationDisabler(&txn); + Status status = multiInitialSyncApply_noAbort(&txn, ops, st); + fassertNoTrace(15915, status); +} + +Status multiInitialSyncApply_noAbort(OperationContext* txn, + const std::vector& ops, + SyncTail* st) { + txn->setReplicatedWrites(false); + DisableDocumentValidation validationDisabler(txn); // allow us to get through the magic barrier - txn.lockState()->setIsBatchWriter(true); + txn->lockState()->setIsBatchWriter(true); - bool convertUpdatesToUpserts = false; + // This function is only called in initial sync, as its name suggests. + bool inSteadyStateReplication = false; for (std::vector::const_iterator it = ops.begin(); it != ops.end(); ++it) { try { - const Status s = SyncTail::syncApply(&txn, *it, convertUpdatesToUpserts); + const Status s = SyncTail::syncApply(txn, *it, inSteadyStateReplication); if (!s.isOK()) { - if (st->shouldRetry(&txn, *it)) { - const Status s2 = SyncTail::syncApply(&txn, *it, convertUpdatesToUpserts); + // Don't retry on commands. + SyncTail::OplogEntry entry(*it); + if (entry.opType[0] == 'c') { + error() << "Error applying command (" << it->toString() << "): " << s; + return s; + } + + // We might need to fetch the missing docs from the sync source. + if (st->shouldRetry(txn, *it)) { + const Status s2 = SyncTail::syncApply(txn, *it, inSteadyStateReplication); if (!s2.isOK()) { severe() << "Error applying operation (" << it->toString() << "): " << s2; - fassertFailedNoTrace(15915); + return s2; } } @@ -1037,16 +1060,24 @@ void multiInitialSyncApply(const std::vector& ops, SyncTail* st) { // subsequently got deleted and no longer exists on the Sync Target at all } } catch (const DBException& e) { + // SERVER-24927 If we have a NamespaceNotFound exception, then this document will be + // dropped before initial sync ends anyways and we should ignore it. + SyncTail::OplogEntry entry(*it); + if (e.getCode() == ErrorCodes::NamespaceNotFound && + isCrudOpType(entry.opType.rawData())) { + continue; + } + severe() << "writer worker caught exception: " << causedBy(e) << " on: " << it->toString(); if (inShutdown()) { - return; + return Status::OK(); } - - fassertFailedNoTrace(16361); + return e.toStatus(); } } + return Status::OK(); } } // namespace repl diff --git a/src/mongo/db/repl/sync_tail.h b/src/mongo/db/repl/sync_tail.h index 313a16bf48b..7493698b14c 100644 --- a/src/mongo/db/repl/sync_tail.h +++ b/src/mongo/db/repl/sync_tail.h @@ -59,7 +59,7 @@ public: /** * Type of function that takes a non-command op and applies it locally. * Used for applying from an oplog. - * Last boolean argument 'convertUpdateToUpsert' converts some updates to upserts for + * Last boolean argument 'inSteadyStateReplication' converts some updates to upserts for * idempotency reasons. * Returns failure status if the op was an update that could not be applied. */ @@ -71,7 +71,7 @@ public: * Used for applying from an oplog. * Returns failure status if the op that could not be applied. */ - using ApplyCommandInLockFn = stdx::function; + using ApplyCommandInLockFn = stdx::function; /** * Type of function to increment "repl.apply.ops" server status metric. @@ -88,12 +88,12 @@ public: */ static Status syncApply(OperationContext* txn, const BSONObj& o, - bool convertUpdateToUpsert, + bool inSteadyStateReplication, ApplyOperationInLockFn applyOperationInLock, ApplyCommandInLockFn applyCommandInLock, IncrementOpsAppliedStatsFn incrementOpsAppliedStats); - static Status syncApply(OperationContext* txn, const BSONObj& o, bool convertUpdateToUpsert); + static Status syncApply(OperationContext* txn, const BSONObj& o, bool inSteadyStateReplication); void oplogApplication(StorageInterface* storageInterface); bool peek(BSONObj* obj); @@ -191,9 +191,9 @@ public: static int replWriterThreadCount; protected: - // Cap the batches using the limit on journal commits. - // This works out to be 100 MB (64 bit) or 50 MB (32 bit) - static const unsigned int replBatchLimitBytes = dur::UncommittedBytesLimit; + // Cap the batches to 50 MB for 32-bit systems and 100 MB for 64-bit systems. + static const unsigned int replBatchLimitBytes = + (sizeof(void*) == 4) ? 50 * 1024 * 1024 : 100 * 1024 * 1024; static const unsigned int replBatchLimitOperations = 5000; // Apply a batch of operations, using multiple threads. @@ -220,6 +220,9 @@ private: // These free functions are used by the thread pool workers to write ops to the db. void multiSyncApply(const std::vector& ops, SyncTail* st); void multiInitialSyncApply(const std::vector& ops, SyncTail* st); +Status multiInitialSyncApply_noAbort(OperationContext* txn, + const std::vector& ops, + SyncTail* st); } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 29f81337e05..0d0250e4814 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -30,23 +30,32 @@ #include +#include "mongo/db/catalog/collection_options.h" +#include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/document_validation.h" +#include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/client.h" #include "mongo/db/curop.h" +#include "mongo/db/db_raii.h" #include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/bgsync.h" #include "mongo/db/repl/operation_context_repl_mock.h" +#include "mongo/db/repl/oplog_interface_local.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/repl/replication_coordinator_mock.h" #include "mongo/db/repl/sync_tail.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/unittest/unittest.h" #include "mongo/unittest/temp_dir.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/scopeguard.h" + namespace { @@ -88,7 +97,7 @@ void SyncTailTest::setUp() { // go away after the global storage engine is initialized. unittest::TempDir tempDir("sync_tail_test"); mongo::storageGlobalParams.dbpath = tempDir.path(); - mongo::storageGlobalParams.engine = "devnull"; + mongo::storageGlobalParams.engine = "ephemeralForTest"; mongo::storageGlobalParams.engineSetByUser = true; serviceContext->initializeGlobalStorageEngine(); } @@ -101,16 +110,29 @@ void SyncTailTest::setUp() { _txn.reset(new OperationContextReplMock(&cc(), 0)); _opsApplied = 0; _applyOp = - [](OperationContext* txn, Database* db, const BSONObj& op, bool convertUpdateToUpsert) { + [](OperationContext* txn, Database* db, const BSONObj& op, bool inSteadyStateReplication) { return Status::OK(); }; - _applyCmd = [](OperationContext* txn, const BSONObj& op) { return Status::OK(); }; + _applyCmd = [](OperationContext* txn, const BSONObj& op, bool) { return Status::OK(); }; _incOps = [this]() { _opsApplied++; }; } void SyncTailTest::tearDown() { + ON_BLOCK_EXIT([&] { Client::destroy(); }); + + dropAllDatabasesExceptLocal(_txn.get()); { + ScopedTransaction transaction(_txn.get(), MODE_X); Lock::GlobalWrite globalLock(_txn->lockState()); + AutoGetDb autoDBLocal(_txn.get(), "local", MODE_X); + auto localDB = autoDBLocal.getDb(); + if (localDB) { + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + // Do not wrap in a WriteUnitOfWork until SERVER-17103 is addressed. + dropDatabase(_txn.get(), localDB); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(_txn.get(), "_dropAllDBs", "local"); + } BSONObjBuilder unused; invariant(mongo::dbHolder().closeAll(_txn.get(), unused, false)); } @@ -118,6 +140,11 @@ void SyncTailTest::tearDown() { setGlobalReplicationCoordinator(nullptr); } +Status failedApplyCommand(OperationContext* txn, const BSONObj& theOperation, bool) { + FAIL("applyCommand unexpectedly invoked."); + return Status::OK(); +} + TEST_F(SyncTailTest, Peek) { BackgroundSyncMock bgsync; SyncTail syncTail(&bgsync, [](const std::vector& ops, SyncTail* st) {}); @@ -159,7 +186,7 @@ TEST_F(SyncTailTest, SyncApplyNoOp) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isDbLockedForMode("test", MODE_X)); @@ -167,17 +194,12 @@ TEST_F(SyncTailTest, SyncApplyNoOp) { ASSERT_TRUE(documentValidationDisabled(txn)); ASSERT_TRUE(db); ASSERT_EQUALS(op, theOperation); - ASSERT_FALSE(convertUpdateToUpsert); + ASSERT_FALSE(inSteadyStateReplication); return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; ASSERT_TRUE(_txn->writesAreReplicated()); ASSERT_FALSE(documentValidationDisabled(_txn.get())); - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, failedApplyCommand, _incOps)); ASSERT_TRUE(applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -191,19 +213,14 @@ TEST_F(SyncTailTest, SyncApplyNoOpApplyOpThrowsException) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled++; if (applyOpCalled < 5) { throw WriteConflictException(); } return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, failedApplyCommand, _incOps)); ASSERT_EQUALS(5, applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -217,7 +234,7 @@ void SyncTailTest::_testSyncApplyInsertDocument(LockMode expectedMode) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isDbLockedForMode("test", expectedMode)); @@ -226,17 +243,12 @@ void SyncTailTest::_testSyncApplyInsertDocument(LockMode expectedMode) { ASSERT_TRUE(documentValidationDisabled(txn)); ASSERT_TRUE(db); ASSERT_EQUALS(op, theOperation); - ASSERT_TRUE(convertUpdateToUpsert); + ASSERT_TRUE(inSteadyStateReplication); return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; ASSERT_TRUE(_txn->writesAreReplicated()); ASSERT_FALSE(documentValidationDisabled(_txn.get())); - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, true, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, true, applyOp, failedApplyCommand, _incOps)); ASSERT_TRUE(applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -259,11 +271,13 @@ TEST_F(SyncTailTest, SyncApplyInsertDocumentCollectionMissing) { TEST_F(SyncTailTest, SyncApplyInsertDocumentCollectionExists) { { Lock::GlobalWrite globalLock(_txn->lockState()); + WriteUnitOfWork wunit(_txn.get()); bool justCreated = false; Database* db = dbHolder().openDb(_txn.get(), "test", &justCreated); ASSERT_TRUE(db); ASSERT_TRUE(justCreated); Collection* collection = db->createCollection(_txn.get(), "test.t"); + wunit.commit(); ASSERT_TRUE(collection); } _testSyncApplyInsertDocument(MODE_IX); @@ -278,7 +292,7 @@ TEST_F(SyncTailTest, SyncApplyIndexBuild) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isDbLockedForMode("test", MODE_X)); @@ -286,17 +300,12 @@ TEST_F(SyncTailTest, SyncApplyIndexBuild) { ASSERT_TRUE(documentValidationDisabled(txn)); ASSERT_TRUE(db); ASSERT_EQUALS(op, theOperation); - ASSERT_FALSE(convertUpdateToUpsert); + ASSERT_FALSE(inSteadyStateReplication); return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; ASSERT_TRUE(_txn->writesAreReplicated()); ASSERT_FALSE(documentValidationDisabled(_txn.get())); - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, failedApplyCommand, _incOps)); ASSERT_TRUE(applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -310,12 +319,12 @@ TEST_F(SyncTailTest, SyncApplyCommand) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { FAIL("applyOperation unexpectedly invoked."); return Status::OK(); }; SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { + [&](OperationContext* txn, const BSONObj& theOperation, bool) { applyCmdCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isW()); @@ -340,12 +349,12 @@ TEST_F(SyncTailTest, SyncApplyCommandThrowsException) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { FAIL("applyOperation unexpectedly invoked."); return Status::OK(); }; SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { + [&](OperationContext* txn, const BSONObj& theOperation, bool) { applyCmdCalled++; if (applyCmdCalled < 5) { throw WriteConflictException(); @@ -357,4 +366,199 @@ TEST_F(SyncTailTest, SyncApplyCommandThrowsException) { ASSERT_EQUALS(1U, _opsApplied); } +/** + * Creates a command oplog entry with given optime and namespace. + */ +BSONObj makeCommandOplogEntry(OpTime opTime, const NamespaceString& nss, const BSONObj& command) { + BSONObjBuilder bob; + bob.appendElements(opTime.toBSON()); + bob.append("h", 1LL); + bob.append("v", 2); + bob.append("op", "c"); + bob.append("ns", nss.getCommandNS()); + bob.append("o", command); + return bob.obj(); +} + +/** + * Creates a create collection oplog entry with given optime. + */ +BSONObj makeCreateCollectionOplogEntry(OpTime opTime, + const NamespaceString& nss = NamespaceString("test.foo"), + const BSONObj& options = BSONObj()) { + BSONObjBuilder bob; + bob.append("create", nss.coll()); + bob.appendElements(options); + return makeCommandOplogEntry(opTime, nss, bob.obj()); +} + +/** + * Creates an insert oplog entry with given optime and namespace. + */ +BSONObj makeInsertDocumentOplogEntry(OpTime opTime, + const NamespaceString& nss, + const BSONObj& documentToInsert) { + BSONObjBuilder bob; + bob.appendElements(opTime.toBSON()); + bob.append("h", 1LL); + bob.append("op", "i"); + bob.append("ns", nss.ns()); + bob.append("o", documentToInsert); + return bob.obj(); +} + +class IdempotencyTest : public SyncTailTest { +protected: + BSONObj createCollection(); + BSONObj buildIndex(const BSONObj& indexSpec, const BSONObj& options = BSONObj()); + BSONObj dropIndex(const std::string& indexName); + OpTime nextOpTime() { + static long long lastSecond = 1; + return OpTime(Timestamp(Seconds(lastSecond++), 0), 1LL); + } + Status runOp(const BSONObj& entry); + Status runOps(std::initializer_list ops); + // Validate data and indexes. Return the MD5 hash of the documents ordered by _id. + std::string validate(); + + NamespaceString nss{"test.foo"}; + NamespaceString nssIndex{"test.system.indexes"}; +}; + +Status IdempotencyTest::runOp(const BSONObj& op) { + return runOps({op}); +} + +Status IdempotencyTest::runOps(std::initializer_list ops) { + std::vector opsVector(ops); + SyncTail syncTail(nullptr, SyncTail::MultiSyncApplyFunc()); + return multiInitialSyncApply_noAbort(_txn.get(), opsVector, &syncTail); +} + +BSONObj IdempotencyTest::createCollection() { + return makeCreateCollectionOplogEntry(nextOpTime(), nss); +} + +BSONObj IdempotencyTest::buildIndex(const BSONObj& indexSpec, const BSONObj& options) { + BSONObjBuilder bob; + bob.append("v", 1); + bob.append("key", indexSpec); + bob.append("name", std::string(indexSpec.firstElementFieldName()) + "_index"); + bob.append("ns", nss.ns()); + bob.appendElementsUnique(options); + return makeInsertDocumentOplogEntry(nextOpTime(), nssIndex, bob.obj()); +} + +BSONObj IdempotencyTest::dropIndex(const std::string& indexName) { + auto cmd = BSON("deleteIndex" << nss.coll() << "index" << indexName); + return makeCommandOplogEntry(nextOpTime(), nss, cmd); +} + +std::string IdempotencyTest::validate() { + auto collection = AutoGetCollectionForRead(_txn.get(), nss).getCollection(); + if (!collection) { + return "CollectionNotFound"; + } + ValidateResults validateResults; + BSONObjBuilder bob; + + Lock::DBLock lk(_txn->lockState(), nss.db(), MODE_IS); + Lock::CollectionLock lock(_txn->lockState(), nss.ns(), MODE_IS); + ASSERT_OK(collection->validate(_txn.get(), true, true, &validateResults, &bob)); + ASSERT_TRUE(validateResults.valid); + + IndexDescriptor* desc = collection->getIndexCatalog()->findIdIndex(_txn.get()); + ASSERT_TRUE(desc); + auto exec = InternalPlanner::indexScan(_txn.get(), + collection, + desc, + BSONObj(), + BSONObj(), + false, + PlanExecutor::YIELD_MANUAL, + InternalPlanner::FORWARD, + InternalPlanner::IXSCAN_FETCH); + ASSERT(NULL != exec.get()); + md5_state_t st; + md5_init(&st); + + PlanExecutor::ExecState state; + BSONObj c; + while (PlanExecutor::ADVANCED == (state = exec->getNext(&c, NULL))) { + md5_append(&st, (const md5_byte_t*)c.objdata(), c.objsize()); + } + ASSERT_EQUALS(PlanExecutor::IS_EOF, state); + md5digest d; + md5_finish(&st, d); + return digestToString(d); +} + +TEST_F(IdempotencyTest, CollModNamespaceNotFound) { + getGlobalReplicationCoordinator()->setFollowerMode(MemberState::RS_RECOVERING); + + ASSERT_OK(runOp(createCollection())); + ASSERT_OK(runOp(buildIndex(BSON("createdAt" << 1), BSON("expireAfterSeconds" << 3600)))); + + auto indexChange = fromjson("{keyPattern: {createdAt:1}, expireAfterSeconds:4000}}"); + auto collModCmd = BSON("collMod" << nss.coll() << "index" << indexChange); + auto collModOp = makeCommandOplogEntry(nextOpTime(), nss, collModCmd); + auto dropCollOp = makeCommandOplogEntry(nextOpTime(), nss, BSON("drop" << nss.coll())); + + auto ops = {collModOp, dropCollOp}; + + ASSERT_OK(runOps(ops)); + auto hash = validate(); + ASSERT_OK(runOps(ops)); + ASSERT_EQUALS(hash, validate()); +} + +TEST_F(IdempotencyTest, CollModIndexNotFound) { + getGlobalReplicationCoordinator()->setFollowerMode(MemberState::RS_RECOVERING); + + ASSERT_OK(runOp(createCollection())); + ASSERT_OK(runOp(buildIndex(BSON("createdAt" << 1), BSON("expireAfterSeconds" << 3600)))); + + auto indexChange = fromjson("{keyPattern: {createdAt:1}, expireAfterSeconds:4000}}"); + auto collModCmd = BSON("collMod" << nss.coll() << "index" << indexChange); + auto collModOp = makeCommandOplogEntry(nextOpTime(), nss, collModCmd); + auto dropIndexOp = dropIndex("createdAt_index"); + + auto ops = {collModOp, dropIndexOp}; + + ASSERT_OK(runOps(ops)); + auto hash = validate(); + ASSERT_OK(runOps(ops)); + ASSERT_EQUALS(hash, validate()); +} + +TEST_F(IdempotencyTest, ResyncOnRenameCollection) { + ReplicationCoordinator::get(_txn.get())->setFollowerMode(MemberState::RS_RECOVERING); + + auto cmd = BSON("renameCollection" << nss.ns() << "to" + << "test.bar" + << "stayTemp" << false << "dropTarget" << false); + auto op = makeCommandOplogEntry(nextOpTime(), nss, cmd); + ASSERT_EQUALS(runOp(op), ErrorCodes::OplogOperationUnsupported); +} + +TEST_F(IdempotencyTest, MultiInitialSyncApplySkipsDocumentOnNamespaceNotFound) { + BSONObj emptyDoc; + NamespaceString nss("test", "foo"); + NamespaceString badNss("test", "bad"); + auto doc1 = BSON("_id" << 1); + auto doc2 = BSON("_id" << 2); + auto doc3 = BSON("_id" << 3); + auto op0 = makeCreateCollectionOplogEntry(nextOpTime(), nss); + auto op1 = makeInsertDocumentOplogEntry(nextOpTime(), nss, doc1); + auto op2 = makeInsertDocumentOplogEntry(nextOpTime(), badNss, doc2); + auto op3 = makeInsertDocumentOplogEntry(nextOpTime(), nss, doc3); + runOps({op0, op1, op2, op3}); + + OplogInterfaceLocal collectionReader(_txn.get(), nss.ns()); + auto iter = collectionReader.makeIterator(); + ASSERT_EQUALS(doc3, unittest::assertGet(iter->next()).first); + ASSERT_EQUALS(doc1, unittest::assertGet(iter->next()).first); + ASSERT_EQUALS(ErrorCodes::NoSuchKey, iter->next().getStatus()); +} + } // namespace diff --git a/src/mongo/db/repl/topology_coordinator.h b/src/mongo/db/repl/topology_coordinator.h index de01144f796..3a51485052a 100644 --- a/src/mongo/db/repl/topology_coordinator.h +++ b/src/mongo/db/repl/topology_coordinator.h @@ -379,12 +379,24 @@ public: /** * Tries to transition the coordinator from the leader role to the follower role. * - * Fails if "force" is not set and no follower is known to be up. It is illegal - * to call this method if the node is not leader. + * If force==true, step down this node and return true immediately. Else, a step down + * succeeds only if the following conditions are met: * - * Returns whether or not the step down succeeded. + * C1. A majority set of nodes, M, in the replica set have optimes greater than or + * equal to the last applied optime of the primary. + * + * C2. If C1 holds, then there must exist at least one electable secondary node in the + * majority set M. + * + * If C1 and C2 hold, a step down occurs and this method returns true. Else, the step down + * fails and this method returns false. + * + * NOTE: It is illegal to call this method if the node is not a primary. */ - virtual bool stepDown(Date_t until, bool force, const OpTime& lastOpApplied) = 0; + virtual bool stepDown(Date_t until, + bool force, + const OpTime& lastOpApplied, + const OpTime& lastOpCommitted) = 0; /** * Sometimes a request to step down comes in (like via a heartbeat), but we don't have the @@ -410,9 +422,9 @@ public: /** * Prepares a BSONObj describing the current term, primary, and lastOp information. */ - virtual void prepareReplResponseMetadata(rpc::ReplSetMetadata* metadata, - const OpTime& lastVisibleOpTime, - const OpTime& lastCommittedOpTime) const = 0; + virtual void prepareReplMetadata(rpc::ReplSetMetadata* metadata, + const OpTime& lastVisibleOpTime, + const OpTime& lastCommittedOpTime) const = 0; /** * Writes into 'output' all the information needed to generate a summary of the current diff --git a/src/mongo/db/repl/topology_coordinator_impl.cpp b/src/mongo/db/repl/topology_coordinator_impl.cpp index e245ddb3276..53c79c80b03 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl.cpp @@ -51,6 +51,7 @@ #include "mongo/db/server_parameters.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/s/catalog/catalog_manager.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" @@ -67,6 +68,11 @@ const Seconds TopologyCoordinatorImpl::VoteLease::leaseTime = Seconds(30); // must be before it will call for a priority takeover election. MONGO_EXPORT_STARTUP_SERVER_PARAMETER(priorityTakeoverFreshnessWindowSeconds, int, 2); +// If this fail point is enabled, TopologyCoordinatorImpl::shouldChangeSyncSource() will ignore +// the option TopologyCoordinatorImpl::Options::maxSyncSourceLagSecs. The sync source will not be +// re-evaluated if it lags behind another node by more than 'maxSyncSourceLagSecs' seconds. +MONGO_FP_DECLARE(disableMaxSyncSourceLagSecs); + namespace { template @@ -1361,6 +1367,23 @@ bool TopologyCoordinatorImpl::_aMajoritySeemsToBeUp() const { return vUp * 2 > _rsConfig.getTotalVotingMembers(); } +bool TopologyCoordinatorImpl::_canSeeHealthyPrimaryOfEqualOrGreaterPriority( + const int candidateIndex) const { + const double candidatePriority = _rsConfig.getMemberAt(candidateIndex).getPriority(); + for (auto it = _hbdata.begin(); it != _hbdata.end(); ++it) { + if (!it->up() || it->getState() != MemberState::RS_PRIMARY) { + continue; + } + const int itIndex = indexOfIterator(_hbdata, it); + const double priority = _rsConfig.getMemberAt(itIndex).getPriority(); + if (itIndex != candidateIndex && priority >= candidatePriority) { + return true; + } + } + + return false; +} + bool TopologyCoordinatorImpl::_isOpTimeCloseEnoughToLatestToElect( const OpTime& otherOpTime, const OpTime& ourLastOpApplied) const { const OpTime latestKnownOpTime = _latestKnownOpTime(ourLastOpApplied); @@ -2228,24 +2251,37 @@ void TopologyCoordinatorImpl::processLoseElection() { } } -bool TopologyCoordinatorImpl::stepDown(Date_t until, bool force, const OpTime& lastOpApplied) { - bool canStepDown = force; - for (int i = 0; !canStepDown && i < _rsConfig.getNumMembers(); ++i) { - if (i == _selfIndex) { - continue; - } - UnelectableReasonMask reason = _getUnelectableReason(i, lastOpApplied); - if (!reason && _hbdata[i].getAppliedOpTime() >= lastOpApplied) { - canStepDown = true; - } +bool TopologyCoordinatorImpl::stepDown(Date_t until, + bool force, + const OpTime& lastOpApplied, + const OpTime& lastOpCommitted) { + // force==true overrides all other checks. + if (force) { + _stepDownUntil = until; + _stepDownSelfAndReplaceWith(-1); + return true; } - if (!canStepDown) { + // Ensure a majority of caught up nodes. + if (lastOpCommitted < lastOpApplied) { return false; } - _stepDownUntil = until; - _stepDownSelfAndReplaceWith(-1); - return true; + + // Now make sure we also have at least one caught up node that is also electable. + for (int memberIndex = 0; memberIndex < _rsConfig.getNumMembers(); memberIndex++) { + // ignore your self + if (memberIndex == _selfIndex) { + continue; + } + UnelectableReasonMask reason = _getUnelectableReason(memberIndex, lastOpApplied); + if (!reason && _hbdata.at(memberIndex).getAppliedOpTime() >= lastOpApplied) { + // Found a caught up and electable node, succeed with step down. + _stepDownUntil = until; + _stepDownSelfAndReplaceWith(-1); + return true; + } + } + return false; } void TopologyCoordinatorImpl::setFollowerMode(MemberState::MS newMode) { @@ -2383,32 +2419,40 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource(const HostAndPort& currentS return true; } - unsigned int currentSecs = currentSourceOpTime.getSecs(); - unsigned int goalSecs = currentSecs + durationCount(_options.maxSyncSourceLagSecs); + if (MONGO_FAIL_POINT(disableMaxSyncSourceLagSecs)) { + log() << "disableMaxSyncSourceLagSecs fail point enabled - not checking the most recent " + "OpTime, " << currentSourceOpTime.toString() << ", of our current sync source, " + << currentSource << ", against the OpTimes of the other nodes in this replica set."; + } else { + unsigned int currentSecs = currentSourceOpTime.getSecs(); + unsigned int goalSecs = currentSecs + durationCount(_options.maxSyncSourceLagSecs); - for (std::vector::const_iterator it = _hbdata.begin(); it != _hbdata.end(); - ++it) { - const int itIndex = indexOfIterator(_hbdata, it); - const MemberConfig& candidateConfig = _rsConfig.getMemberAt(itIndex); - if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) && - (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) && - it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) && - goalSecs < it->getAppliedOpTime().getSecs()) { - log() << "re-evaluating sync source because our current sync source's most recent " - << "OpTime is " << currentSourceOpTime.toString() << " which is more than " - << _options.maxSyncSourceLagSecs << " behind member " - << candidateConfig.getHostAndPort().toString() << " whose most recent OpTime is " - << it->getAppliedOpTime().toString(); - invariant(itIndex != _selfIndex); - return true; + for (std::vector::const_iterator it = _hbdata.begin(); + it != _hbdata.end(); + ++it) { + const int itIndex = indexOfIterator(_hbdata, it); + const MemberConfig& candidateConfig = _rsConfig.getMemberAt(itIndex); + if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) && + (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) && + it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) && + goalSecs < it->getAppliedOpTime().getSecs()) { + log() << "re-evaluating sync source because our current sync source's most recent " + << "OpTime is " << currentSourceOpTime.toString() << " which is more than " + << _options.maxSyncSourceLagSecs << " behind member " + << candidateConfig.getHostAndPort().toString() + << " whose most recent OpTime is " << it->getAppliedOpTime().toString(); + invariant(itIndex != _selfIndex); + return true; + } } } + return false; } -void TopologyCoordinatorImpl::prepareReplResponseMetadata(rpc::ReplSetMetadata* metadata, - const OpTime& lastVisibleOpTime, - const OpTime& lastCommittedOpTime) const { +void TopologyCoordinatorImpl::prepareReplMetadata(rpc::ReplSetMetadata* metadata, + const OpTime& lastVisibleOpTime, + const OpTime& lastCommittedOpTime) const { *metadata = rpc::ReplSetMetadata(_term, lastCommittedOpTime, @@ -2442,12 +2486,16 @@ void TopologyCoordinatorImpl::processReplSetRequestVotes(const ReplSetRequestVot } else if (args.getSetName() != _rsConfig.getReplSetName()) { response->setVoteGranted(false); response->setReason("candidate's set name differs from mine"); - } else if (args.getLastCommittedOp() < lastAppliedOpTime) { + } else if (args.getLastDurableOpTime() < lastAppliedOpTime) { response->setVoteGranted(false); response->setReason("candidate's data is staler than mine"); } else if (!args.isADryRun() && _lastVote.getTerm() == args.getTerm()) { response->setVoteGranted(false); response->setReason("already voted for another candidate this term"); + } else if (_selfConfig().isArbiter() && + _canSeeHealthyPrimaryOfEqualOrGreaterPriority(args.getCandidateIndex())) { + response->setVoteGranted(false); + response->setReason("can see a healthy primary of equal or greater priority"); } else { if (!args.isADryRun()) { _lastVote.setTerm(args.getTerm()); diff --git a/src/mongo/db/repl/topology_coordinator_impl.h b/src/mongo/db/repl/topology_coordinator_impl.h index fb4b7786cb3..4361f27f3bc 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.h +++ b/src/mongo/db/repl/topology_coordinator_impl.h @@ -219,12 +219,15 @@ public: virtual void processLoseElection(); virtual Status checkShouldStandForElection(Date_t now, const OpTime& lastOpApplied) const; virtual void setMyHeartbeatMessage(const Date_t now, const std::string& message); - virtual bool stepDown(Date_t until, bool force, const OpTime& lastOpApplied); + virtual bool stepDown(Date_t until, + bool force, + const OpTime& lastOpApplied, + const OpTime& lastOpCommitted); virtual bool stepDownIfPending(); virtual Date_t getStepDownTime() const; - virtual void prepareReplResponseMetadata(rpc::ReplSetMetadata* metadata, - const OpTime& lastVisibleOpTime, - const OpTime& lastCommitttedOpTime) const; + virtual void prepareReplMetadata(rpc::ReplSetMetadata* metadata, + const OpTime& lastVisibleOpTime, + const OpTime& lastCommitttedOpTime) const; Status processReplSetDeclareElectionWinner(const ReplSetDeclareElectionWinnerArgs& args, long long* responseTerm); virtual void processReplSetRequestVotes(const ReplSetRequestVotesArgs& args, @@ -305,6 +308,10 @@ private: // Sees if a majority number of votes are held by members who are currently "up" bool _aMajoritySeemsToBeUp() const; + // Returns true if the node can see a healthy primary of equal or greater priority to the + // candidate. + bool _canSeeHealthyPrimaryOfEqualOrGreaterPriority(const int candidateIndex) const; + // Is otherOpTime close enough (within 10 seconds) to the latest known optime to qualify // for an election bool _isOpTimeCloseEnoughToLatestToElect(const OpTime& otherOpTime, @@ -457,7 +464,7 @@ private: } _voteLease; // V1 last vote info for elections - LastVote _lastVote; + LastVote _lastVote{OpTime::kInitialTerm, -1}; enum class ReadCommittedSupport { kUnknown, diff --git a/src/mongo/db/repl/topology_coordinator_impl_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_test.cpp index 8d3ae2b6516..e35499c7797 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_test.cpp @@ -5513,6 +5513,43 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) { ASSERT_FALSE(response.getVoteGranted()); } +TEST_F(TopoCoordTest, ArbiterDoesNotGrantVoteWhenItCanSeeAHealthyPrimaryOfEqualOrGreaterPriority) { + updateConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 10 << "host" + << "hself" + << "arbiterOnly" << true) + << BSON("_id" << 20 << "host" + << "h2" + << "priority" << 5) << BSON("_id" << 30 << "host" + << "h3"))), + 0); + heartbeatFromMember(HostAndPort("h2"), + "rs0", + MemberState::RS_PRIMARY, + OpTime(Timestamp(0, 0), 0), + Milliseconds(300)); + heartbeatFromMember(HostAndPort("h3"), + "rs0", + MemberState::RS_SECONDARY, + OpTime(Timestamp(0, 0), 0), + Milliseconds(300)); + + ReplSetRequestVotesArgs args; + args.initialize(BSON("replSetRequestVotes" << 1 << "setName" + << "rs0" + << "term" << 1LL << "candidateIndex" << 2LL + << "configVersion" << 1LL << "lastCommittedOp" + << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); + ReplSetRequestVotesResponse response; + OpTime lastAppliedOpTime; + + getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + ASSERT_EQUALS("can see a healthy primary of equal or greater priority", response.getReason()); + ASSERT_FALSE(response.getVoteGranted()); +} + TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) { updateConfig(BSON("_id" << "rs0" diff --git a/src/mongo/db/repl/vote_requester.cpp b/src/mongo/db/repl/vote_requester.cpp index 06df2e289c5..eafa68715e2 100644 --- a/src/mongo/db/repl/vote_requester.cpp +++ b/src/mongo/db/repl/vote_requester.cpp @@ -48,12 +48,12 @@ VoteRequester::Algorithm::Algorithm(const ReplicaSetConfig& rsConfig, long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry) + OpTime lastDurableOpTime) : _rsConfig(rsConfig), _candidateIndex(candidateIndex), _term(term), _dryRun(dryRun), - _lastOplogEntry(lastOplogEntry) { + _lastDurableOpTime(lastDurableOpTime) { // populate targets with all voting members that aren't this node long long index = 0; for (auto member = _rsConfig.membersBegin(); member != _rsConfig.membersEnd(); member++) { @@ -75,10 +75,7 @@ std::vector VoteRequester::Algorithm::getRequests() const requestVotesCmdBuilder.append("candidateIndex", _candidateIndex); requestVotesCmdBuilder.append("configVersion", _rsConfig.getConfigVersion()); - BSONObjBuilder lastCommittedOp(requestVotesCmdBuilder.subobjStart("lastCommittedOp")); - lastCommittedOp.append("ts", _lastOplogEntry.getTimestamp()); - lastCommittedOp.append("t", _lastOplogEntry.getTerm()); - lastCommittedOp.done(); + _lastDurableOpTime.append(&requestVotesCmdBuilder, "lastCommittedOp"); const BSONObj requestVotesCmd = requestVotesCmdBuilder.obj(); @@ -150,9 +147,9 @@ StatusWith VoteRequester::start( long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry, + OpTime lastDurableOpTime, const stdx::function& onCompletion) { - _algorithm.reset(new Algorithm(rsConfig, candidateIndex, term, dryRun, lastOplogEntry)); + _algorithm.reset(new Algorithm(rsConfig, candidateIndex, term, dryRun, lastDurableOpTime)); _runner.reset(new ScatterGatherRunner(_algorithm.get())); return _runner->start(executor, onCompletion); } diff --git a/src/mongo/db/repl/vote_requester.h b/src/mongo/db/repl/vote_requester.h index 8ee9b052d4d..433facf345c 100644 --- a/src/mongo/db/repl/vote_requester.h +++ b/src/mongo/db/repl/vote_requester.h @@ -65,7 +65,7 @@ public: long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry); + OpTime lastDurableOpTime); virtual ~Algorithm(); virtual std::vector getRequests() const; virtual void processResponse(const executor::RemoteCommandRequest& request, @@ -89,7 +89,7 @@ public: const long long _candidateIndex; const long long _term; bool _dryRun = false; // this bool indicates this is a mock election when true - const OpTime _lastOplogEntry; + const OpTime _lastDurableOpTime; std::vector _targets; unordered_set _responders; bool _staleTerm = false; @@ -115,7 +115,7 @@ public: long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry, + OpTime lastDurableOpTime, const stdx::function& onCompletion = stdx::function()); /** diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index 96dd8de9b68..d6ea4110beb 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -469,6 +469,29 @@ bool MigrationSourceManager::storeCurrentLocs(OperationContext* txn, return true; } +namespace { + +static bool stillSameSession(boost::optional const& memberSessionId, + MigrationSessionId const& argSessionId, + std::string& errmsg) { + if (!memberSessionId) { + errmsg = "not active"; + return false; + } + + // A mongod version < v3.2 will not have sessionId, in which case it is empty and + // ignored. + if (!argSessionId.isEmpty() && !memberSessionId->matches(argSessionId)) { + errmsg = str::stream() << "migration session id changed from " << argSessionId.toString() + << " to " << memberSessionId->toString() + << " while initial clone was active"; + return false; + } + return true; +} + +} // namespace + bool MigrationSourceManager::clone(OperationContext* txn, const MigrationSessionId& sessionId, string& errmsg, @@ -483,16 +506,7 @@ bool MigrationSourceManager::clone(OperationContext* txn, stdx::lock_guard sl(_mutex); - if (!_sessionId) { - errmsg = "not active"; - return false; - } - - // A mongod version < v3.2 will not have sessionId, in which case it is empty and ignored. - if (!sessionId.isEmpty() && !_sessionId->matches(sessionId)) { - errmsg = str::stream() << "requested migration session id " << sessionId.toString() - << " does not match active session id " - << _sessionId->toString(); + if (!stillSameSession(_sessionId, sessionId, errmsg)) { return false; } @@ -507,37 +521,46 @@ bool MigrationSourceManager::clone(OperationContext* txn, static_cast((12 + collection->averageObjectSize(txn)) * cloneLocsRemaining())); } - bool isBufferFilled = false; BSONArrayBuilder clonedDocsArrayBuilder(allocSize); + std::vector cloneLocsTemp; + + // We carve off a limited number of records to look up per externally-visible iteration + // so that, for very big collections, progress meters register activity. + cloneLocsTemp.reserve(1000); + + bool isBufferFilled = false; while (!isBufferFilled) { ScopedTransaction scopedXact(txn, MODE_IS); AutoGetCollection autoColl(txn, _getNS(), MODE_IS); + Collection* collection = autoColl.getCollection(); - stdx::lock_guard sl(_mutex); - - if (!_sessionId) { - errmsg = "not active"; + if (!collection) { + errmsg = str::stream() << "collection " << _getNS().toString() << " does not exist"; return false; } - // A mongod version < v3.2 will not have sessionId, in which case it is empty and ignored. - if (!sessionId.isEmpty() && !_sessionId->matches(sessionId)) { - errmsg = str::stream() << "migration session id changed from " << sessionId.toString() - << " to " << _sessionId->toString() - << " while initial clone was active"; - return false; - } + { + stdx::lock_guard sl(_mutex); - Collection* collection = autoColl.getCollection(); - if (!collection) { - errmsg = str::stream() << "collection " << _nss.toString() << " does not exist"; - return false; - } + if (!stillSameSession(_sessionId, sessionId, errmsg)) { + return false; + } - stdx::lock_guard lk(_cloneLocsMutex); + { + stdx::lock_guard lk(_cloneLocsMutex); + + for (RecordId const& cloneLoc : _cloneLocs) { + cloneLocsTemp.push_back(cloneLoc); + if (cloneLocsTemp.size() == cloneLocsTemp.capacity()) { + break; // enough for now + } + } + } + } - std::set::iterator cloneLocsIter = _cloneLocs.begin(); - for (; cloneLocsIter != _cloneLocs.end(); ++cloneLocsIter) { + // release locks during find ops + auto cloneLocsIter = cloneLocsTemp.begin(); + for (; cloneLocsIter != cloneLocsTemp.end(); ++cloneLocsIter) { if (tracker.intervalHasElapsed()) // should I yield? break; @@ -561,12 +584,23 @@ bool MigrationSourceManager::clone(OperationContext* txn, clonedDocsArrayBuilder.append(doc.value()); } - _cloneLocs.erase(_cloneLocs.begin(), cloneLocsIter); + // reclaim locks and record progress + + stdx::lock_guard sl(_mutex); + + if (!stillSameSession(_sessionId, sessionId, errmsg)) { + return false; + } + + stdx::lock_guard lk(_cloneLocsMutex); - // Note: must be holding _cloneLocsMutex, don't move this inside while condition! + std::for_each(cloneLocsTemp.begin(), + cloneLocsIter, + [&](RecordId const& loc) { _cloneLocs.erase(loc); }); if (_cloneLocs.empty()) { - break; + break; // and return } + cloneLocsTemp.clear(); } result.appendArray("objects", clonedDocsArrayBuilder.arr()); diff --git a/src/mongo/db/stats/SConscript b/src/mongo/db/stats/SConscript index 0dea68e45da..a0ca5534447 100644 --- a/src/mongo/db/stats/SConscript +++ b/src/mongo/db/stats/SConscript @@ -54,3 +54,24 @@ env.Library( '$BUILD_DIR/mongo/util/concurrency/spin_lock', ], ) + +env.Library( + target='fill_locker_info', + source=[ + 'fill_locker_info.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/concurrency/lock_manager' + ], +) + +env.CppUnitTest( + target='fill_locker_info_test', + source=[ + 'fill_locker_info_test.cpp', + ], + LIBDEPS=[ + 'fill_locker_info', + ], +) diff --git a/src/mongo/db/stats/fill_locker_info_test.cpp b/src/mongo/db/stats/fill_locker_info_test.cpp new file mode 100644 index 00000000000..40a58879ddf --- /dev/null +++ b/src/mongo/db/stats/fill_locker_info_test.cpp @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/bsontypes.h" +#include "mongo/db/concurrency/lock_manager_defs.h" +#include "mongo/db/stats/fill_locker_info.h" +#include "mongo/unittest/death_test.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +using LockerInfo = Locker::LockerInfo; +using OneLock = Locker::OneLock; + +const ResourceId kGlobalId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL); + +TEST(FillLockerInfo, DoesReportWaitingForLockIfWaiting) { + LockerInfo info; + info.waitingResource = kGlobalId; + ASSERT_TRUE(info.waitingResource.isValid()); + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT(infoObj["waitingForLock"].type() == BSONType::Bool); + ASSERT_TRUE(infoObj["waitingForLock"].Bool()); +} + +TEST(FillLockerInfo, DoesNotReportWaitingForLockIfNotWaiting) { + LockerInfo info; + info.waitingResource = ResourceId(); // This means it is not waiting for anything. + ASSERT_FALSE(info.waitingResource.isValid()); + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT(infoObj["waitingForLock"].type() == BSONType::Bool); + ASSERT_FALSE(infoObj["waitingForLock"].Bool()); +} + +TEST(FillLockerInfo, DoesReportLockStats) { + LockerInfo info; + SingleThreadedLockStats stats; + stats.recordAcquisition(kGlobalId, MODE_IX); + info.stats = stats; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT_EQ(infoObj["lockStats"].type(), BSONType::Object); +} + +DEATH_TEST(FillLockerInfo, ShouldFailIfLocksAreNotSortedAppropriately, "Invariant failure") { + LockerInfo info; + // The global lock is supposed to come before the database lock. + info.locks = {OneLock{ResourceId(RESOURCE_DATABASE, std::string("TestDB")), MODE_X}, + OneLock{kGlobalId, MODE_IX}}; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); +} + +TEST(FillLockerInfo, DoesReportLocksHeld) { + const ResourceId dbId(RESOURCE_DATABASE, std::string("TestDB")); + LockerInfo info; + info.locks = {OneLock{kGlobalId, MODE_IX}, OneLock{dbId, MODE_IX}}; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT_EQ(infoObj["locks"].type(), BSONType::Object); + ASSERT_EQ(infoObj["locks"][resourceTypeName(kGlobalId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(kGlobalId.getType())].String(), "w"); + ASSERT_EQ(infoObj["locks"][resourceTypeName(dbId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(dbId.getType())].String(), "w"); +} + +TEST(FillLockerInfo, ShouldReportMaxTypeHeldForResourceType) { + const ResourceId firstDbId(RESOURCE_DATABASE, std::string("FirstDB")); + const ResourceId secondDbId(RESOURCE_DATABASE, std::string("SecondDB")); + LockerInfo info; + info.locks = { + OneLock{kGlobalId, MODE_IX}, OneLock{firstDbId, MODE_IX}, OneLock{secondDbId, MODE_X}}; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + BSONObj infoObj = infoBuilder.done(); + + ASSERT_EQ(infoObj["locks"].type(), BSONType::Object); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].String(), + "W"); // One is held in IX, one in X, so X should win and be displayed as "W". + + // Ensure it still works if locks are reported in the opposite order. + info.locks = { + OneLock{kGlobalId, MODE_IX}, OneLock{secondDbId, MODE_X}, OneLock{firstDbId, MODE_IX}}; + + ASSERT_EQ(infoObj["locks"].type(), BSONType::Object); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].String(), "W"); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp b/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp index 6e2d19d6bce..ffec90efc04 100644 --- a/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp +++ b/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp @@ -2310,10 +2310,6 @@ bool BtreeLogic::locate(OperationContext* txn, *bucketLocOut = _locate(txn, getRootLoc(txn), owned, posOut, &found, recordLoc, direction); - if (!found) { - return false; - } - skipUnusedKeys(txn, bucketLocOut, posOut, direction); return found; diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp index 915e3a7e44d..b8139cf7e89 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp @@ -97,7 +97,6 @@ NamespaceDetails::Extra* NamespaceDetails::allocExtra(OperationContext* txn, Namespace fullns(ns); Namespace extrans(fullns.extraName(i)); // throws UserException if ns name too long - massert(10350, "allocExtra: base ns missing?", this); massert(10351, "allocExtra: extra already exists", ni.details(extrans) == 0); Extra temp; diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp index 5cfda64a2af..e24b668ac3e 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp @@ -32,6 +32,9 @@ #include "mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/ops/update.h" #include "mongo/db/record_id.h" @@ -251,6 +254,11 @@ Status NamespaceDetailsCollectionCatalogEntry::removeIndex(OperationContext* txn d->idx(getTotalIndexCount(txn)) = IndexDetails(); } + // Someone may be querying the system.indexes namespace directly, so we need to invalidate + // its cursors. + MMAPV1DatabaseCatalogEntry::invalidateSystemCollectionRecord( + txn, NamespaceString(_db->name(), "system.indexes"), infoLocation); + // remove from system.indexes _indexRecordStore->deleteRecord(txn, infoLocation); @@ -360,8 +368,20 @@ void NamespaceDetailsCollectionCatalogEntry::_updateSystemNamespaces(OperationCo RecordData entry = _namespacesRecordStore->dataFor(txn, _namespacesRecordId); const BSONObj newEntry = applyUpdateOperators(entry.releaseToBson(), update); - StatusWith result = _namespacesRecordStore->updateRecord( - txn, _namespacesRecordId, newEntry.objdata(), newEntry.objsize(), false, NULL); + + // Get update notifier + invariant(txn->lockState()->isDbLockedForMode(_db->name(), MODE_X)); + Database* db = dbHolder().get(txn, _db->name()); + Collection* systemCollection = + db->getCollection(NamespaceString(_db->name(), "system.namespaces")); + UpdateNotifier* namespacesNotifier = systemCollection->getUpdateNotifier(); + + StatusWith result = _namespacesRecordStore->updateRecord(txn, + _namespacesRecordId, + newEntry.objdata(), + newEntry.objsize(), + false, + namespacesNotifier); fassert(17486, result.getStatus()); setNamespacesRecordId(txn, result.getValue()); } diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp index 4b3c592a554..c29d7507c9b 100644 --- a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp @@ -34,6 +34,8 @@ #include +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/index/2d_access_method.h" #include "mongo/db/index/btree_access_method.h" @@ -300,6 +302,9 @@ Status MMAPV1DatabaseCatalogEntry::renameCollection(OperationContext* txn, if (!s.isOK()) return s; } + // Invalidate index record for the old collection. + invalidateSystemCollectionRecord( + txn, NamespaceString(name(), "system.indexes"), record->id); systemIndexRecordStore->deleteRecord(txn, record->id); } @@ -366,6 +371,10 @@ Status MMAPV1DatabaseCatalogEntry::_renameSingleNamespace(OperationContext* txn, RecordId rid = _addNamespaceToNamespaceCollection(txn, toNS, newSpec.isEmpty() ? 0 : &newSpec); + // Invalidate old namespace record + invalidateSystemCollectionRecord( + txn, NamespaceString(name(), "system.namespaces"), oldSpecLocation); + _getNamespaceRecordStore()->deleteRecord(txn, oldSpecLocation); Entry*& entry = _collections[toNS.toString()]; @@ -378,6 +387,17 @@ Status MMAPV1DatabaseCatalogEntry::_renameSingleNamespace(OperationContext* txn, return Status::OK(); } +void MMAPV1DatabaseCatalogEntry::invalidateSystemCollectionRecord( + OperationContext* txn, NamespaceString systemCollectionNamespace, RecordId record) { + // Having to go back up through the DatabaseHolder is a bit of a layering + // violation, but at this point we're not going to add more MMAPv1 specific interfaces. + StringData dbName = systemCollectionNamespace.db(); + invariant(txn->lockState()->isDbLockedForMode(dbName, MODE_X)); + Database* db = dbHolder().get(txn, dbName); + Collection* systemCollection = db->getCollection(systemCollectionNamespace); + systemCollection->getCursorManager()->invalidateDocument(txn, record, INVALIDATION_DELETION); +} + void MMAPV1DatabaseCatalogEntry::appendExtraStats(OperationContext* opCtx, BSONObjBuilder* output, double scale) const { @@ -789,7 +809,12 @@ void MMAPV1DatabaseCatalogEntry::_removeNamespaceFromNamespaceCollection(Operati RecordStoreV1Base* rs = _getNamespaceRecordStore(); invariant(rs); - rs->deleteRecord(txn, entry->second->catalogEntry->getNamespacesRecordId()); + // Invalidate old namespace record + RecordId oldSpecLocation = entry->second->catalogEntry->getNamespacesRecordId(); + invalidateSystemCollectionRecord( + txn, NamespaceString(name(), "system.namespaces"), oldSpecLocation); + + rs->deleteRecord(txn, oldSpecLocation); } CollectionOptions MMAPV1DatabaseCatalogEntry::getCollectionOptions(OperationContext* txn, diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h index e127eece268..bc1229db016 100644 --- a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h +++ b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h @@ -127,6 +127,9 @@ public: * exist. */ void createNamespaceForIndex(OperationContext* txn, StringData name); + static void invalidateSystemCollectionRecord(OperationContext* txn, + NamespaceString systemCollectionNamespace, + RecordId record); /** * Ensures data files are compatible, in case we are downgrading from a newer version. Returns diff --git a/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp b/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp index 6803e5a3bc7..df0e8edd675 100644 --- a/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp +++ b/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp @@ -452,7 +452,8 @@ Status SimpleRecordStoreV1::compact(OperationContext* txn, _details->setLastExtentSize(txn, 0); // create a new extent so new records go there - increaseStorageSize(txn, _details->lastExtentSize(txn), true); + const bool enforceQuota = false; + increaseStorageSize(txn, _details->lastExtentSize(txn), enforceQuota); wunit.commit(); } diff --git a/src/mongo/db/storage/snapshot.h b/src/mongo/db/storage/snapshot.h index 6ce5b57e51d..b622c14d863 100644 --- a/src/mongo/db/storage/snapshot.h +++ b/src/mongo/db/storage/snapshot.h @@ -57,6 +57,10 @@ public: return _id != other._id; } + std::string toString() const { + return std::to_string(_id); + } + private: uint64_t _id; }; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index e5c478932ac..27ab575387f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -37,22 +37,22 @@ #include #include "mongo/base/checked_cast.h" -#include "mongo/db/json.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_descriptor.h" +#include "mongo/db/json.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/key_string.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" -#include "mongo/db/storage/storage_options.h" #include "mongo/stdx/memory.h" #include "mongo/util/assert_util.h" -#include "mongo/util/hex.h" #include "mongo/util/fail_point.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" @@ -381,7 +381,7 @@ bool WiredTigerIndex::isEmpty(OperationContext* txn) { WT_CURSOR* c = curwrap.get(); if (!c) return true; - int ret = WT_OP_CHECK(c->next(c)); + int ret = WT_READ_CHECK(c->next(c)); if (ret == WT_NOTFOUND) return true; invariantWTOK(ret); @@ -442,7 +442,7 @@ bool WiredTigerIndex::isDup(WT_CURSOR* c, const BSONObj& key, const RecordId& id KeyString data(key, _ordering); WiredTigerItem item(data.getBuffer(), data.getSize()); c->set_key(c, item.Get()); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret == WT_NOTFOUND) { return false; } @@ -490,13 +490,17 @@ protected: // Open cursors can cause bulk open_cursor to fail with EBUSY. // TODO any other cases that could cause EBUSY? WiredTigerSession* outerSession = WiredTigerRecoveryUnit::get(_txn)->getSession(_txn); - outerSession->closeAllCursors(); + outerSession->closeAllCursors(idx->uri()); // Not using cursor cache since we need to set "bulk". WT_CURSOR* cursor; - // We use our own session to ensure we aren't in a transaction. + // Use a different session to ensure we don't hijack an existing transaction. + // Configure the bulk cursor open to fail quickly if it would wait on a checkpoint + // completing - since checkpoints can take a long time, and waiting can result in + // an unexpected pause in building an index. WT_SESSION* session = _session->getSession(); - int err = session->open_cursor(session, idx->uri().c_str(), NULL, "bulk", &cursor); + int err = session->open_cursor( + session, idx->uri().c_str(), NULL, "bulk,checkpoint_wait=false", &cursor); if (!err) return cursor; @@ -805,7 +809,7 @@ protected: void advanceWTCursor() { WT_CURSOR* c = _cursor->get(); - int ret = WT_OP_CHECK(_forward ? c->next(c) : c->prev(c)); + int ret = WT_READ_CHECK(_forward ? c->next(c) : c->prev(c)); if (ret == WT_NOTFOUND) { _cursorAtEof = true; return; @@ -822,7 +826,7 @@ protected: const WiredTigerItem keyItem(query.getBuffer(), query.getSize()); c->set_key(c, keyItem.Get()); - int ret = WT_OP_CHECK(c->search_near(c, &cmp)); + int ret = WT_READ_CHECK(c->search_near(c, &cmp)); if (ret == WT_NOTFOUND) { _cursorAtEof = true; TRACE_CURSOR << "\t not found"; @@ -974,7 +978,7 @@ public: c->set_key(c, keyItem.Get()); // Using search rather than search_near. - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret != WT_NOTFOUND) invariantWTOK(ret); _cursorAtEof = ret == WT_NOTFOUND; @@ -989,7 +993,7 @@ public: WiredTigerIndexUnique::WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc) - : WiredTigerIndex(ctx, uri, desc) {} + : WiredTigerIndex(ctx, uri, desc), _partial(desc->isPartial()) {} std::unique_ptr WiredTigerIndexUnique::newCursor(OperationContext* txn, bool forward) const { @@ -1025,7 +1029,7 @@ Status WiredTigerIndexUnique::_insert(WT_CURSOR* c, // we put them all in the "list" // Note that we can't omit AllZeros when there are multiple ids for a value. When we remove // down to a single value, it will be cleaned up. - ret = WT_OP_CHECK(c->search(c)); + ret = WT_READ_CHECK(c->search(c)); invariantWTOK(ret); WT_ITEM old; @@ -1073,8 +1077,37 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, WiredTigerItem keyItem(data.getBuffer(), data.getSize()); c->set_key(c, keyItem.Get()); + auto triggerWriteConflictAtPoint = [&keyItem](WT_CURSOR* point) { + // WT_NOTFOUND may occur during a background index build. Insert a dummy value and + // delete it again to trigger a write conflict in case this is being concurrently + // indexed by the background indexer. + point->set_key(point, keyItem.Get()); + point->set_value(point, emptyItem.Get()); + invariantWTOK(WT_OP_CHECK(point->insert(point))); + point->set_key(point, keyItem.Get()); + invariantWTOK(WT_OP_CHECK(point->remove(point))); + }; + if (!dupsAllowed) { - // nice and clear + if (_partial) { + // Check that the record id matches. We may be called to unindex records that are not + // present in the index due to the partial filter expression. + int ret = WT_OP_CHECK(c->search(c)); + if (ret == WT_NOTFOUND) { + triggerWriteConflictAtPoint(c); + return; + } + WT_ITEM value; + invariantWTOK(c->get_value(c, &value)); + BufReader br(value.data, value.size); + fassert(40416, br.remaining()); + if (KeyString::decodeRecordId(&br) != id) { + return; + } + // Ensure there aren't any other values in here. + KeyString::TypeBits::fromBuffer(&br); + fassert(40417, !br.remaining()); + } int ret = WT_OP_CHECK(c->remove(c)); if (ret == WT_NOTFOUND) { return; @@ -1085,16 +1118,9 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, // dups are allowed, so we have to deal with a vector of RecordIds. - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret == WT_NOTFOUND) { - // WT_NOTFOUND is only expected during a background index build. Insert a dummy value and - // delete it again to trigger a write conflict in case this is being concurrently indexed by - // the background indexer. - c->set_key(c, keyItem.Get()); - c->set_value(c, emptyItem.Get()); - invariantWTOK(WT_OP_CHECK(c->insert(c))); - c->set_key(c, keyItem.Get()); - invariantWTOK(WT_OP_CHECK(c->remove(c))); + triggerWriteConflictAtPoint(c); return; } invariantWTOK(ret); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 4ea5741e7a5..66284492e69 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -164,6 +164,9 @@ public: Status _insert(WT_CURSOR* c, const BSONObj& key, const RecordId& id, bool dupsAllowed) override; void _unindex(WT_CURSOR* c, const BSONObj& key, const RecordId& id, bool dupsAllowed) override; + +private: + bool _partial; }; class WiredTigerIndexStandard : public WiredTigerIndex { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index d0d28218770..a98e230c257 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -200,7 +200,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, ss << "create,"; ss << "cache_size=" << cacheSizeGB << "G,"; ss << "session_max=20000,"; - ss << "eviction=(threads_max=4),"; + ss << "eviction=(threads_min=4,threads_max=4),"; ss << "config_base=false,"; ss << "statistics=(fast),"; // The setting may have a later setting override it if not using the journal. We make it @@ -345,11 +345,12 @@ int64_t WiredTigerKVEngine::getIdentSize(OperationContext* opCtx, StringData ide Status WiredTigerKVEngine::repairIdent(OperationContext* opCtx, StringData ident) { WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession(opCtx); - session->closeAllCursors(); + string uri = _uri(ident); + session->closeAllCursors(uri); + _sessionCache->closeAllCursors(uri); if (isEphemeral()) { return Status::OK(); } - string uri = _uri(ident); return _salvageIfNeeded(uri.c_str()); } @@ -530,6 +531,8 @@ Status WiredTigerKVEngine::dropIdent(OperationContext* opCtx, StringData ident) bool WiredTigerKVEngine::_drop(StringData ident) { string uri = _uri(ident); + _sessionCache->closeAllCursors(uri); + WiredTigerSession session(_conn); int ret = session.getSession()->drop( @@ -545,9 +548,9 @@ bool WiredTigerKVEngine::_drop(StringData ident) { // this is expected, queue it up { stdx::lock_guard lk(_identToDropMutex); - _identToDrop.push(uri); + _identToDrop.push_front(uri); } - _sessionCache->closeAllCursors(); + _sessionCache->closeCursorsForQueuedDrops(); return false; } @@ -555,6 +558,28 @@ bool WiredTigerKVEngine::_drop(StringData ident) { return false; } +std::list WiredTigerKVEngine::filterCursorsWithQueuedDrops( + std::list* cache) { + std::list toDrop; + + stdx::lock_guard lk(_identToDropMutex); + if (_identToDrop.empty()) + return toDrop; + + for (auto i = cache->begin(); i != cache->end();) { + if (!i->_cursor || + std::find(_identToDrop.begin(), _identToDrop.end(), std::string(i->_cursor->uri)) == + _identToDrop.end()) { + ++i; + continue; + } + toDrop.push_back(*i); + i = cache->erase(i); + } + + return toDrop; +} + bool WiredTigerKVEngine::haveDropsQueued() const { Date_t now = Date_t::now(); Milliseconds delta = now - _previousCheckedDropsQueued; @@ -565,13 +590,14 @@ bool WiredTigerKVEngine::haveDropsQueued() const { } // We only want to check the queue max once per second or we'll thrash - // This is done in haveDropsQueued, not dropSomeQueuedIdents so we skip the mutex if (delta < Milliseconds(1000)) return false; _previousCheckedDropsQueued = now; - stdx::lock_guard lk(_identToDropMutex); - return !_identToDrop.empty(); + + // Don't wait for the mutex: if we can't get it, report that no drops are queued. + stdx::unique_lock lk(_identToDropMutex, stdx::defer_lock); + return lk.try_lock() && !_identToDrop.empty(); } void WiredTigerKVEngine::dropSomeQueuedIdents() { @@ -597,7 +623,7 @@ void WiredTigerKVEngine::dropSomeQueuedIdents() { if (_identToDrop.empty()) break; uri = _identToDrop.front(); - _identToDrop.pop(); + _identToDrop.pop_front(); } int ret = session.getSession()->drop( session.getSession(), uri.c_str(), "force,checkpoint_wait=false"); @@ -605,7 +631,7 @@ void WiredTigerKVEngine::dropSomeQueuedIdents() { if (ret == EBUSY) { stdx::lock_guard lk(_identToDropMutex); - _identToDrop.push(uri); + _identToDrop.push_back(uri); } else { invariantWTOK(ret); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 8c5a54f6bb4..0085c5d5cf7 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -31,7 +31,7 @@ #pragma once -#include +#include #include #include @@ -133,6 +133,8 @@ public: return _conn; } void dropSomeQueuedIdents(); + std::list filterCursorsWithQueuedDrops( + std::list* cache); bool haveDropsQueued() const; void syncSizeInfo(bool sync) const; @@ -177,7 +179,7 @@ private: mutable stdx::mutex _dropAllQueuesMutex; mutable stdx::mutex _identToDropMutex; - std::queue _identToDrop; + std::list _identToDrop; mutable Date_t _previousCheckedDropsQueued; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index bd6bdceb5e9..3ed0b96f6df 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -92,6 +92,7 @@ bool shouldUseOplogHack(OperationContext* opCtx, const std::string& uri) { } // namespace MONGO_FP_DECLARE(WTWriteConflictException); +MONGO_FP_DECLARE(WTWriteConflictExceptionForReads); MONGO_FP_DECLARE(WTPausePrimaryOplogDurabilityLoop); const std::string kWiredTigerEngineName = "wiredTiger"; @@ -450,7 +451,7 @@ public: // Nothing after the next line can throw WCEs. // Note that an unpositioned (or eof) WT_CURSOR returns the first/last entry in the // table when you call next/prev. - int advanceRet = WT_OP_CHECK(_forward ? c->next(c) : c->prev(c)); + int advanceRet = WT_READ_CHECK(_forward ? c->next(c) : c->prev(c)); if (advanceRet == WT_NOTFOUND) { _eof = true; return {}; @@ -489,7 +490,7 @@ public: WT_CURSOR* c = _cursor->get(); c->set_key(c, _makeKey(id)); // Nothing after the next line can throw WCEs. - int seekRet = WT_OP_CHECK(c->search(c)); + int seekRet = WT_READ_CHECK(c->search(c)); if (seekRet == WT_NOTFOUND) { _eof = true; return {}; @@ -538,7 +539,7 @@ public: c->set_key(c, _makeKey(_lastReturnedId)); int cmp; - int ret = WT_OP_CHECK(c->search_near(c, &cmp)); + int ret = WT_READ_CHECK(c->search_near(c, &cmp)); if (ret == WT_NOTFOUND) { _eof = true; return !_rs._isCapped; @@ -641,7 +642,7 @@ public: } boost::optional next() final { - int advanceRet = WT_OP_CHECK(_cursor->next(_cursor)); + int advanceRet = WT_READ_CHECK(_cursor->next(_cursor)); if (advanceRet == WT_NOTFOUND) return {}; invariantWTOK(advanceRet); @@ -940,7 +941,7 @@ RecordData WiredTigerRecordStore::dataFor(OperationContext* txn, const RecordId& WT_CURSOR* c = curwrap.get(); invariant(c); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); massert(28556, "Didn't find RecordId in WiredTigerRecordStore", ret != WT_NOTFOUND); invariantWTOK(ret); return _getData(curwrap); @@ -953,7 +954,7 @@ bool WiredTigerRecordStore::findRecord(OperationContext* txn, WT_CURSOR* c = curwrap.get(); invariant(c); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret == WT_NOTFOUND) { return false; } @@ -971,7 +972,7 @@ void WiredTigerRecordStore::deleteRecord(OperationContext* txn, const RecordId& cursor.assertInActiveTxn(); WT_CURSOR* c = cursor.get(); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); invariantWTOK(ret); WT_ITEM old_value; @@ -1078,7 +1079,7 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* txn if (_cappedFirstRecord != RecordId()) { int64_t key = _makeKey(_cappedFirstRecord); truncateEnd->set_key(truncateEnd, key); - ret = WT_OP_CHECK(truncateEnd->search(truncateEnd)); + ret = WT_READ_CHECK(truncateEnd->search(truncateEnd)); if (ret == 0) { positioned = true; savedFirstKey = key; @@ -1087,7 +1088,7 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* txn // Advance the cursor truncateEnd until we find a suitable end point for our truncate while ((sizeSaved < sizeOverCap || docsRemoved < docsOverCap) && (docsRemoved < 20000) && - (positioned || (ret = WT_OP_CHECK(truncateEnd->next(truncateEnd))) == 0)) { + (positioned || (ret = WT_READ_CHECK(truncateEnd->next(truncateEnd))) == 0)) { positioned = false; int64_t key; invariantWTOK(truncateEnd->get_key(truncateEnd, &key)); @@ -1122,7 +1123,7 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* txn if (docsRemoved > 0) { // if we scanned to the end of the collection or past our insert, go back one if (ret == WT_NOTFOUND || newestIdToDelete >= justInserted) { - ret = WT_OP_CHECK(truncateEnd->prev(truncateEnd)); + ret = WT_READ_CHECK(truncateEnd->prev(truncateEnd)); } invariantWTOK(ret); @@ -1274,7 +1275,7 @@ Status WiredTigerRecordStore::insertRecords(OperationContext* txn, } else if (_isCapped) { stdx::lock_guard lk(_uncommittedRecordIdsMutex); record.id = _nextId(); - _addUncommitedRecordId_inlock(txn, record.id); + _addUncommittedRecordId_inlock(txn, record.id); } else { record.id = _nextId(); } @@ -1324,7 +1325,7 @@ StatusWith WiredTigerRecordStore::insertRecord(OperationContext* txn, } void WiredTigerRecordStore::_dealtWithCappedId(SortedRecordIds::iterator it, bool didCommit) { - invariant(&(*it) != NULL); + invariant(it->isNormal()); stdx::lock_guard lk(_uncommittedRecordIdsMutex); if (didCommit && _isOplog && *it != _oplog_highestSeen) { // Defer removal from _uncommittedRecordIds until it is durable. We don't need to wait for @@ -1379,7 +1380,7 @@ StatusWith WiredTigerRecordStore::updateRecord(OperationContext* txn, WT_CURSOR* c = curwrap.get(); invariant(c); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); invariantWTOK(ret); WT_ITEM old_value; @@ -1464,7 +1465,7 @@ std::vector> WiredTigerRecordStore::getManyCursors Status WiredTigerRecordStore::truncate(OperationContext* txn) { WiredTigerCursor startWrap(_uri, _tableId, true, txn); WT_CURSOR* start = startWrap.get(); - int ret = WT_OP_CHECK(start->next(start)); + int ret = WT_READ_CHECK(start->next(start)); // Empty collections don't have anything to truncate. if (ret == WT_NOTFOUND) { return Status::OK(); @@ -1621,7 +1622,7 @@ Status WiredTigerRecordStore::oplogDiskLocRegister(OperationContext* txn, const return id.getStatus(); stdx::lock_guard lk(_uncommittedRecordIdsMutex); - _addUncommitedRecordId_inlock(txn, id.getValue()); + _addUncommittedRecordId_inlock(txn, id.getValue()); return Status::OK(); } @@ -1703,11 +1704,10 @@ void WiredTigerRecordStore::waitForAllEarlierOplogWritesToBeVisible(OperationCon } } -void WiredTigerRecordStore::_addUncommitedRecordId_inlock(OperationContext* txn, - const RecordId& id) { - // todo: make this a dassert at some point - // invariant(_uncommittedRecordIds.empty() || _uncommittedRecordIds.back() < id); +void WiredTigerRecordStore::_addUncommittedRecordId_inlock(OperationContext* txn, RecordId id) { + dassert(_uncommittedRecordIds.empty() || _uncommittedRecordIds.back() < id); SortedRecordIds::iterator it = _uncommittedRecordIds.insert(_uncommittedRecordIds.end(), id); + invariant(it->isNormal()); txn->recoveryUnit()->registerChange(new CappedInsertChange(this, it)); _oplog_highestSeen = id; } @@ -1727,7 +1727,7 @@ boost::optional WiredTigerRecordStore::oplogStartHack( int cmp; c->set_key(c, _makeKey(startingPosition)); - int ret = WT_OP_CHECK(c->search_near(c, &cmp)); + int ret = WT_READ_CHECK(c->search_near(c, &cmp)); if (ret == 0 && cmp > 0) ret = c->prev(c); // landed one higher than startingPosition if (ret == WT_NOTFOUND) diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index 1fab25f8be4..6a439793ee0 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -47,10 +47,17 @@ /** * Either executes the specified operation and returns it's value or randomly throws a write - * conflict exception if the WTWriteConflictException failpoint is enabled. + * conflict exception if the WTWriteConflictException failpoint is enabled. This is only checked + * on cursor methods that make modifications. */ #define WT_OP_CHECK(x) (((MONGO_FAIL_POINT(WTWriteConflictException))) ? (WT_ROLLBACK) : (x)) +/** + * Identical to WT_OP_CHECK except this is checked on cursor seeks/advancement. + */ +#define WT_READ_CHECK(x) \ + (((MONGO_FAIL_POINT(WTWriteConflictExceptionForReads))) ? (WT_ROLLBACK) : (x)) + namespace mongo { class RecoveryUnit; @@ -264,7 +271,7 @@ private: static RecordId _fromKey(int64_t k); void _dealtWithCappedId(SortedRecordIds::iterator it, bool didCommit); - void _addUncommitedRecordId_inlock(OperationContext* txn, const RecordId& id); + void _addUncommittedRecordId_inlock(OperationContext* txn, RecordId id); RecordId _nextId(); void _setId(RecordId id); @@ -326,6 +333,7 @@ private: // WT failpoint to throw write conflict exceptions randomly MONGO_FP_FORWARD_DECLARE(WTWriteConflictException); +MONGO_FP_FORWARD_DECLARE(WTWriteConflictExceptionForReads); // Prevents oplog writes from being considered durable on the primary. Once activated, new writes // will not be considered durable until deactivated. It is unspecified whether writes that commit diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index 58fd74c2ee0..6e2d6643d92 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -206,10 +206,10 @@ void WiredTigerRecoveryUnit::_txnClose(bool commit) { WT_SESSION* s = _session->getSession(); if (commit) { invariantWTOK(s->commit_transaction(s, NULL)); - LOG(3) << "WT commit_transaction"; + LOG(3) << "WT commit_transaction for snapshot id " << _mySnapshotId; } else { invariantWTOK(s->rollback_transaction(s, NULL)); - LOG(3) << "WT rollback_transaction"; + LOG(3) << "WT rollback_transaction for snapshot id " << _mySnapshotId; } _active = false; _mySnapshotId = nextSnapshotId.fetchAndAdd(1); @@ -252,7 +252,7 @@ void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { invariantWTOK(s->begin_transaction(s, NULL)); } - LOG(3) << "WT begin_transaction"; + LOG(3) << "WT begin_transaction for snapshot id " << _mySnapshotId; _timer.reset(); _active = true; } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp index 54fff70be4d..cb1282a6790 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp @@ -106,16 +106,31 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) { } } -void WiredTigerSession::closeAllCursors(uint64_t cursorEpoch) { +void WiredTigerSession::closeAllCursors(const std::string& uri) { invariant(_session); - for (CursorCache::iterator i = _cursors.begin(); i != _cursors.end(); ++i) { + + for (auto i = _cursors.begin(); i != _cursors.end();) { + WT_CURSOR* cursor = i->_cursor; + if (cursor && uri == cursor->uri) { + invariantWTOK(cursor->close(cursor)); + i = _cursors.erase(i); + } else + ++i; + } +} + +void WiredTigerSession::closeCursorsForQueuedDrops(uint64_t cursorEpoch, + WiredTigerKVEngine* engine) { + invariant(_session); + + auto toDrop = engine->filterCursorsWithQueuedDrops(&_cursors); + + for (auto i = toDrop.begin(); i != toDrop.end(); i++) { WT_CURSOR* cursor = i->_cursor; if (cursor) { invariantWTOK(cursor->close(cursor)); } } - _cursors.clear(); - _cursorEpoch = cursorEpoch; } namespace { @@ -221,13 +236,20 @@ void WiredTigerSessionCache::waitUntilDurable(bool forceCheckpoint) { _journalListener->onDurable(token); } -void WiredTigerSessionCache::closeAllCursors() { +void WiredTigerSessionCache::closeAllCursors(const std::string& uri) { + stdx::lock_guard lock(_cacheLock); + for (SessionCache::iterator i = _sessions.begin(); i != _sessions.end(); i++) { + (*i)->closeAllCursors(uri); + } +} + +void WiredTigerSessionCache::closeCursorsForQueuedDrops() { // Increment the cursor epoch so that all cursors from this epoch are closed. uint64_t cursorEpoch = _cursorEpoch.addAndFetch(1); stdx::lock_guard lock(_cacheLock); for (SessionCache::iterator i = _sessions.begin(); i != _sessions.end(); i++) { - (*i)->closeAllCursors(cursorEpoch); + (*i)->closeCursorsForQueuedDrops(cursorEpoch, _engine); } } @@ -288,19 +310,22 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) { return; } - // This checks that we are only caching idle sessions and not something which might hold - // locks or otherwise prevent truncation. { WT_SESSION* ss = session->getSession(); uint64_t range; + // This checks that we are only caching idle sessions and not something which might hold + // locks or otherwise prevent truncation. invariantWTOK(ss->transaction_pinned_range(ss, &range)); invariant(range == 0); + + // Release resources in the session we're about to cache. + invariantWTOK(ss->reset(ss)); } // If the cursor epoch has moved on, close all cursors in the session. uint64_t cursorEpoch = _cursorEpoch.load(); if (session->_getCursorEpoch() != cursorEpoch) - session->closeAllCursors(cursorEpoch); + session->closeCursorsForQueuedDrops(cursorEpoch, _engine); bool returnedToCache = false; uint64_t currentEpoch = _epoch.load(); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h index 4cd482f9772..b0030fe5f20 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h @@ -83,7 +83,9 @@ public: void releaseCursor(uint64_t id, WT_CURSOR* cursor); - void closeAllCursors(uint64_t cursorEpoch = 0); + void closeCursorsForQueuedDrops(uint64_t cursorEpoch, WiredTigerKVEngine* engine); + + void closeAllCursors(const std::string& uri); int cursorsOut() const { return _cursorsOut; @@ -149,11 +151,16 @@ public: */ void closeAll(); + /** + * Closes cached cursors for tables that are queued to be dropped. + */ + void closeCursorsForQueuedDrops(); + /** * Closes all cached cursors and ensures that previously opened cursors will be closed on * release. */ - void closeAllCursors(); + void closeAllCursors(const std::string& uri); /** * Transitions the cache to shutting down mode. Any already released sessions are freed and diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index b840815e3a5..64eb665f9e2 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -396,9 +396,9 @@ int WiredTigerUtil::verifyTable(OperationContext* txn, ErrorAccumulator eventHandler(errors); // Try to close as much as possible to avoid EBUSY errors. - WiredTigerRecoveryUnit::get(txn)->getSession(txn)->closeAllCursors(); + WiredTigerRecoveryUnit::get(txn)->getSession(txn)->closeAllCursors(uri); WiredTigerSessionCache* sessionCache = WiredTigerRecoveryUnit::get(txn)->getSessionCache(); - sessionCache->closeAll(); + sessionCache->closeAllCursors(uri); // Open a new session with custom error handlers. WT_CONNECTION* conn = WiredTigerRecoveryUnit::get(txn)->getSessionCache()->conn(); diff --git a/src/mongo/db/write_concern.cpp b/src/mongo/db/write_concern.cpp index 3c9086ca39a..9dc04670dc4 100644 --- a/src/mongo/db/write_concern.cpp +++ b/src/mongo/db/write_concern.cpp @@ -45,6 +45,7 @@ #include "mongo/db/storage/storage_engine.h" #include "mongo/db/write_concern_options.h" #include "mongo/rpc/protocol.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -72,6 +73,8 @@ namespace { const std::string kLocalDB = "local"; } // namespace +MONGO_FP_DECLARE(hangBeforeWaitingForWriteConcern); + StatusWith extractWriteConcern(OperationContext* txn, const BSONObj& cmdObj, const std::string& dbName) { @@ -234,6 +237,8 @@ Status waitForWriteConcern(OperationContext* txn, // This check does not hold for writes done through dbeval because it runs with a global X lock. dassert(!txn->lockState()->isLocked() || txn->getClient()->isInDirectClient()); + MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangBeforeWaitingForWriteConcern); + // Next handle blocking on disk Timer syncTimer; diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript index f56098bfa1f..6f4d062df7c 100644 --- a/src/mongo/dbtests/SConscript +++ b/src/mongo/dbtests/SConscript @@ -109,6 +109,7 @@ dbtest = env.Program( 'query_stage_update.cpp', 'querytests.cpp', 'replica_set_monitor_test.cpp', + 'replica_set_tests.cpp', 'repltests.cpp', 'rollbacktests.cpp', 'socktests.cpp', diff --git a/src/mongo/dbtests/replica_set_tests.cpp b/src/mongo/dbtests/replica_set_tests.cpp new file mode 100644 index 00000000000..0d9a7cc3a3b --- /dev/null +++ b/src/mongo/dbtests/replica_set_tests.cpp @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/client.h" +#include "mongo/db/dbdirectclient.h" +#include "mongo/db/repl/last_vote.h" +#include "mongo/db/repl/replication_coordinator_external_state_impl.h" +#include "mongo/db/service_context.h" +#include "mongo/stdx/memory.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +ServiceContext::UniqueOperationContext makeOpCtx() { + return cc().makeOperationContext(); +} + +class ReplicaSetTest : public mongo::unittest::Test { +protected: + void setUp() { + auto txn = makeOpCtx(); + _replCoordExternalState.reset(new repl::ReplicationCoordinatorExternalStateImpl()); + } + + void tearDown() { + auto txn = makeOpCtx(); + DBDirectClient client(txn.get()); + client.dropCollection("local.replset.election"); + + _replCoordExternalState.reset(); + } + + repl::ReplicationCoordinatorExternalStateImpl* getReplCoordExternalState() { + return _replCoordExternalState.get(); + } + +private: + std::unique_ptr _replCoordExternalState; +}; + +TEST_F(ReplicaSetTest, ReplCoordExternalStateStoresLastVoteWithNewTerm) { + auto txn = makeOpCtx(); + auto replCoordExternalState = getReplCoordExternalState(); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 1}); + + auto lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{3, 1}); + + lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 3); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); +} + +TEST_F(ReplicaSetTest, ReplCoordExternalStateDoesNotStoreLastVoteWithOldTerm) { + auto txn = makeOpCtx(); + auto replCoordExternalState = getReplCoordExternalState(); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 1}); + + auto lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{1, 1}); + + lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); +} + +TEST_F(ReplicaSetTest, ReplCoordExternalStateDoesNotStoreLastVoteWithEqualTerm) { + auto txn = makeOpCtx(); + auto replCoordExternalState = getReplCoordExternalState(); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 1}); + + auto lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 2}); + + lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/executor/SConscript b/src/mongo/executor/SConscript index c8ff0e69fd0..ec25fc15952 100644 --- a/src/mongo/executor/SConscript +++ b/src/mongo/executor/SConscript @@ -149,6 +149,7 @@ env.Library( '$BUILD_DIR/mongo/base/system_error', '$BUILD_DIR/mongo/db/auth/authcommon', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', + '$BUILD_DIR/mongo/rpc/command_status', '$BUILD_DIR/mongo/rpc/rpc', '$BUILD_DIR/third_party/shim_asio', 'async_stream', diff --git a/src/mongo/executor/async_mock_stream_factory.cpp b/src/mongo/executor/async_mock_stream_factory.cpp index 2a2d8322b59..9613f19cf91 100644 --- a/src/mongo/executor/async_mock_stream_factory.cpp +++ b/src/mongo/executor/async_mock_stream_factory.cpp @@ -212,7 +212,7 @@ void AsyncMockStreamFactory::MockStream::read(asio::mutable_buffer buf, void AsyncMockStreamFactory::MockStream::pushRead(std::vector toRead) { stdx::unique_lock lk(_mutex); - invariant(_state != kRunning && _state != kCanceled); + invariant(_state != kRunning); _readQueue.emplace(std::move(toRead)); } @@ -222,7 +222,7 @@ void AsyncMockStreamFactory::MockStream::setError(std::error_code ec) { std::vector AsyncMockStreamFactory::MockStream::popWrite() { stdx::unique_lock lk(_mutex); - invariant(_state != kRunning && _state != kCanceled); + invariant(_state != kRunning); auto nextWrite = std::move(_writeQueue.front()); _writeQueue.pop(); return nextWrite; diff --git a/src/mongo/executor/async_mock_stream_factory.h b/src/mongo/executor/async_mock_stream_factory.h index 11c10128065..f7897806281 100644 --- a/src/mongo/executor/async_mock_stream_factory.h +++ b/src/mongo/executor/async_mock_stream_factory.h @@ -50,6 +50,49 @@ namespace executor { class AsyncStreamInterface; +/** + * A factory that produces mock streams to allow for testing of NetworkInterfaceASIO. + * + * The streams produced by this factory simulate a flow of Events (ConnectEvent, + * ReadEvent, WriteEvent). The streams created by this factory will automatically + * pause themselves at each Event, and the caller must unblock them by destroying + * the Event object to continue. + * + * Example use of this factory: + * + * AsyncMockStreamFactory factory(); + * + * // NIA will then call makeStream(...) to create new streams from the + * // factory, or the caller can do this manually. + * + * // Wait for the desired stream to exist + * auto stream = streamFactory.blockUntilStreamExists(host); + * + * // If we do not care to inspect after a certain event, we can skip it: + * ConnectEvent{stream}.skip(); + * + * // To examine the stream at an Event, instantiate the event object. + * // When the Event object goes out of scope the stream will unblock. + * { + * WriteEvent write{stream}; + * + * // Inspect what NIA wrote to this stream: + * auto messageData = stream->popWrite(); + * ... + * } + * + * // The Event object will keep the stream blocked as long as it exists. + * // Use this window to perform operations on the stream or inspect it. + * { + * ReadEvent read{stream}; + * + * // Simulate data sent to this stream over the network + * stream->pushRead( ... ); + * + * // Or, simulate a networking error + * stream->setError( error_code ); + * } + */ class AsyncMockStreamFactory final : public AsyncStreamFactoryInterface { public: AsyncMockStreamFactory() = default; @@ -57,6 +100,22 @@ public: std::unique_ptr makeStream(asio::io_service::strand* strand, const HostAndPort& host) override; + /** + * A mock stream class for testing the egress networking layer. + * + * At the core of this class is an idea of deferring actions and allowing inspection + * of state of the stream before those actions happen. + * + * This class operates on the assumption that two threads are in use: a networking + * thread used by NIA to issue IO calls on the MockStream, and a test thread to + * wait on those calls and react. + * + * When the test thread creates an Event object, the constructor sends it to wait + * on a condition variable. When NIA issues an IO call on the stream, the MockStream + * load the proper handler into a placeholder, and then calls notify() on the + * condition variable. At that point the stream is paused and the test thread + * may operate on it. + */ class MockStream final : public AsyncStreamInterface { public: MockStream(asio::io_service::strand* strand, diff --git a/src/mongo/executor/async_stream_test.cpp b/src/mongo/executor/async_stream_test.cpp index 508e81aa5f5..a34c39f9b1f 100644 --- a/src/mongo/executor/async_stream_test.cpp +++ b/src/mongo/executor/async_stream_test.cpp @@ -134,7 +134,11 @@ TEST(AsyncStreamTest, IsOpen) { server.shutdown(); - ASSERT_FALSE(stream.isOpen()); + // There is nothing we can wait on to determinstically know when + // the socket will transition to closed. Busy wait for that. + while (stream.isOpen()) { + stdx::this_thread::sleep_for(Milliseconds(1)); + } } } // namespace diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp index ee243789587..79461a9a5c1 100644 --- a/src/mongo/executor/connection_pool.cpp +++ b/src/mongo/executor/connection_pool.cpp @@ -1,4 +1,5 @@ -/** * Copyright (C) 2015 MongoDB Inc. +/** + * Copyright (C) 2015 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -93,11 +94,23 @@ public: */ size_t availableConnections(const stdx::unique_lock& lk); + /** + * Returns the number of in progress connections in the pool. + */ + size_t refreshingConnections(const stdx::unique_lock& lk); + /** * Returns the total number of connections ever created in this pool. */ size_t createdConnections(const stdx::unique_lock& lk); + /** + * Returns the total number of connections currently open that belong to + * this pool. This is the sum of refreshingConnections, availableConnections, + * and inUseConnections. + */ + size_t openConnections(const stdx::unique_lock& lk); + private: using OwnedConnection = std::unique_ptr; using OwnershipPool = std::unordered_map; @@ -112,7 +125,7 @@ private: void fulfillRequests(stdx::unique_lock& lk); - void spawnConnections(stdx::unique_lock& lk, const HostAndPort& hostAndPort); + void spawnConnections(stdx::unique_lock& lk); void shutdown(); @@ -137,6 +150,7 @@ private: Date_t _requestTimerExpiration; size_t _generation; bool _inFulfillRequests; + bool _inSpawnConnections; size_t _created; @@ -171,12 +185,15 @@ size_t const ConnectionPool::kDefaultMaxConns = std::numeric_limits::max size_t const ConnectionPool::kDefaultMinConns = 1; const Milliseconds ConnectionPool::kDefaultRefreshRequirement = Minutes(1); const Milliseconds ConnectionPool::kDefaultRefreshTimeout = Seconds(20); +size_t const ConnectionPool::kDefaultMaxConnecting = std::numeric_limits::max(); const Status ConnectionPool::kConnectionStateUnknown = Status(ErrorCodes::InternalError, "Connection is in an unknown state"); -ConnectionPool::ConnectionPool(std::unique_ptr impl, Options options) - : _options(std::move(options)), _factory(std::move(impl)) {} +ConnectionPool::ConnectionPool(std::unique_ptr impl, + std::string name, + Options options) + : _name(std::move(name)), _options(std::move(options)), _factory(std::move(impl)) {} ConnectionPool::~ConnectionPool() = default; @@ -221,13 +238,24 @@ void ConnectionPool::appendConnectionStats(ConnectionPoolStats* stats) const { HostAndPort host = kv.first; auto& pool = kv.second; - ConnectionStatsPerHost hostStats{pool->inUseConnections(lk), - pool->availableConnections(lk), - pool->createdConnections(lk)}; - stats->updateStatsForHost(host, hostStats); + ConnectionStatsPer hostStats{pool->inUseConnections(lk), + pool->availableConnections(lk), + pool->createdConnections(lk), + pool->refreshingConnections(lk)}; + stats->updateStatsForHost(_name, host, hostStats); } } +size_t ConnectionPool::getNumConnectionsPerHost(const HostAndPort& hostAndPort) const { + stdx::unique_lock lk(_mutex); + auto iter = _pools.find(hostAndPort); + if (iter != _pools.end()) { + return iter->second->openConnections(lk); + } + + return 0; +} + void ConnectionPool::returnConnection(ConnectionInterface* conn) { stdx::unique_lock lk(_mutex); @@ -244,6 +272,7 @@ ConnectionPool::SpecificPool::SpecificPool(ConnectionPool* parent, const HostAnd _requestTimer(parent->_factory->makeTimer()), _generation(0), _inFulfillRequests(false), + _inSpawnConnections(false), _created(0), _state(State::kRunning) {} @@ -260,26 +289,34 @@ size_t ConnectionPool::SpecificPool::availableConnections( return _readyPool.size(); } +size_t ConnectionPool::SpecificPool::refreshingConnections( + const stdx::unique_lock& lk) { + return _processingPool.size(); +} + size_t ConnectionPool::SpecificPool::createdConnections(const stdx::unique_lock& lk) { return _created; } +size_t ConnectionPool::SpecificPool::openConnections(const stdx::unique_lock& lk) { + return _checkedOutPool.size() + _readyPool.size() + _processingPool.size(); +} + void ConnectionPool::SpecificPool::getConnection(const HostAndPort& hostAndPort, Milliseconds timeout, stdx::unique_lock lk, GetConnectionCallback cb) { - // We need some logic here to handle kNoTimeout, which is defined as -1 Milliseconds. If we just - // added the timeout, we would get a time 1MS in the past, which would immediately timeout - the - // exact opposite of what we want. - auto expiration = (timeout == RemoteCommandRequest::kNoTimeout) - ? RemoteCommandRequest::kNoExpirationDate - : _parent->_factory->now() + timeout; + if (timeout < Milliseconds(0) || timeout > _parent->_options.refreshTimeout) { + timeout = _parent->_options.refreshTimeout; + } + + const auto expiration = _parent->_factory->now() + timeout; _requests.push(make_pair(expiration, std::move(cb))); updateStateInLock(); - spawnConnections(lk, hostAndPort); + spawnConnections(lk); fulfillRequests(lk); } @@ -302,6 +339,8 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr if (!conn->getStatus().isOK()) { // TODO: alert via some callback if the host is bad + log() << "Ending connection to host " << _hostAndPort << " due to bad connection status; " + << openConnections(lk) << " connections to that host remain open"; return; } @@ -312,6 +351,9 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr if (_readyPool.size() + _processingPool.size() + _checkedOutPool.size() >= _parent->_options.minConnections) { // If we already have minConnections, just let the connection lapse + log() << "Ending idle connection to host " << _hostAndPort + << " because the pool meets constraints; " << openConnections(lk) + << " connections to that host remain open"; return; } @@ -328,8 +370,10 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr auto conn = takeFromProcessingPool(connPtr); // If the host and port were dropped, let this lapse - if (conn->getGeneration() != _generation) + if (conn->getGeneration() != _generation) { + spawnConnections(lk); return; + } // If we're in shutdown, we don't need refreshed connections if (_state == State::kInShutdown) @@ -339,6 +383,19 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr // pool if (status.isOK()) { addToReady(lk, std::move(conn)); + spawnConnections(lk); + return; + } + + // If we've exceeded the time limit, start a new connect, rather than + // failing all operations. We do this because the various callers have + // their own time limit which is unrelated to our internal one. + if (status.code() == ErrorCodes::ExceededTimeLimit) { + log() << "Pending connection to host " << _hostAndPort + << " did not complete within the connection timeout," + << " retrying with a new connection;" << openConnections(lk) + << " connections to that host remain open"; + spawnConnections(lk); return; } @@ -402,6 +459,10 @@ void ConnectionPool::SpecificPool::processFailure(const Status& status, // Drop ready connections _readyPool.clear(); + // Log something helpful + log() << "Dropping all pooled connections to " << _hostAndPort + << " due to failed operation on a connection"; + // Migrate processing connections to the dropped pool for (auto&& x : _processingPool) { _droppedProcessingPool[x.first] = std::move(x.second); @@ -456,7 +517,7 @@ void ConnectionPool::SpecificPool::fulfillRequests(stdx::unique_lockgetHostAndPort()); + spawnConnections(lk); } // Drop the bad connection. @@ -486,8 +547,15 @@ void ConnectionPool::SpecificPool::fulfillRequests(stdx::unique_lock& lk, - const HostAndPort& hostAndPort) { +void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock& lk) { + // If some other thread (possibly this thread) is spawning connections, + // don't keep padding the callstack. + if (_inSpawnConnections) + return; + + _inSpawnConnections = true; + auto guard = MakeGuard([&] { _inSpawnConnections = false; }); + // We want minConnections <= outstanding requests <= maxConnections auto target = [&] { return std::max( @@ -496,11 +564,12 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock_options.maxConnecting)) { std::unique_ptr handle; try { // make a new connection and put it in processing - handle = _parent->_factory->makeConnection(hostAndPort, _generation); + handle = _parent->_factory->makeConnection(_hostAndPort, _generation); } catch (std::system_error& e) { severe() << "Failed to construct a new connection object: " << e.what(); fassertFailed(40336); @@ -524,8 +593,15 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lockgetGeneration() != _generation) { // If the host and port was dropped, let the // connection lapse + spawnConnections(lk); } else if (status.isOK()) { addToReady(lk, std::move(conn)); + spawnConnections(lk); + } else if (status.code() == ErrorCodes::ExceededTimeLimit) { + // If we've exceeded the time limit, restart the connect, rather than + // failing all operations. We do this because the various callers + // have their own time limit which is unrelated to our internal one. + spawnConnections(lk); } else { // If the setup failed, cascade the failure edge processFailure(status, std::move(lk)); diff --git a/src/mongo/executor/connection_pool.h b/src/mongo/executor/connection_pool.h index 83dbe54d240..ad5dec51a04 100644 --- a/src/mongo/executor/connection_pool.h +++ b/src/mongo/executor/connection_pool.h @@ -72,6 +72,7 @@ public: static const size_t kDefaultMinConns; static const Milliseconds kDefaultRefreshRequirement; static const Milliseconds kDefaultRefreshTimeout; + static const size_t kDefaultMaxConnecting; static const Status kConnectionStateUnknown; @@ -91,6 +92,13 @@ public: */ size_t maxConnections = kDefaultMaxConns; + /** + * The maximum number of processing connections for a host. This includes pending + * connections in setup/refresh. It's designed to rate limit connection storms rather than + * steady state processing (as maxConnections does). + */ + size_t maxConnecting = kDefaultMaxConnecting; + /** * Amount of time to wait before timing out a refresh attempt */ @@ -111,6 +119,7 @@ public: }; explicit ConnectionPool(std::unique_ptr impl, + std::string name, Options options = Options{}); ~ConnectionPool(); @@ -121,9 +130,13 @@ public: void appendConnectionStats(ConnectionPoolStats* stats) const; + size_t getNumConnectionsPerHost(const HostAndPort& hostAndPort) const; + private: void returnConnection(ConnectionInterface* connection); + std::string _name; + // Options are set at startup and never changed at run time, so these are // accessed outside the lock const Options _options; @@ -190,7 +203,6 @@ class ConnectionPool::ConnectionInterface : public TimerInterface { public: ConnectionInterface() = default; - virtual ~ConnectionInterface() = default; /** diff --git a/src/mongo/executor/connection_pool_stats.cpp b/src/mongo/executor/connection_pool_stats.cpp index 02ff34da6c1..64cc3bbc6eb 100644 --- a/src/mongo/executor/connection_pool_stats.cpp +++ b/src/mongo/executor/connection_pool_stats.cpp @@ -36,44 +36,73 @@ namespace mongo { namespace executor { -ConnectionStatsPerHost::ConnectionStatsPerHost(size_t nInUse, size_t nAvailable, size_t nCreated) - : inUse(nInUse), available(nAvailable), created(nCreated) {} +ConnectionStatsPer::ConnectionStatsPer(size_t nInUse, + size_t nAvailable, + size_t nCreated, + size_t nRefreshing) + : inUse(nInUse), available(nAvailable), created(nCreated), refreshing(nRefreshing) {} -ConnectionStatsPerHost::ConnectionStatsPerHost() = default; +ConnectionStatsPer::ConnectionStatsPer() = default; -ConnectionStatsPerHost& ConnectionStatsPerHost::operator+=(const ConnectionStatsPerHost& other) { +ConnectionStatsPer& ConnectionStatsPer::operator+=(const ConnectionStatsPer& other) { inUse += other.inUse; available += other.available; created += other.created; + refreshing += other.refreshing; return *this; } -void ConnectionPoolStats::updateStatsForHost(HostAndPort host, ConnectionStatsPerHost newStats) { +void ConnectionPoolStats::updateStatsForHost(std::string pool, + HostAndPort host, + ConnectionStatsPer newStats) { // Update stats for this host. - auto hostStats = mapFindWithDefault(statsByHost, host); - hostStats += newStats; - statsByHost.insert(std::make_pair(host, hostStats)); + statsByPool[pool] += newStats; + statsByHost[host] += newStats; + statsByPoolHost[pool][host] += newStats; // Update total connection stats. totalInUse += newStats.inUse; totalAvailable += newStats.available; totalCreated += newStats.created; + totalRefreshing += newStats.refreshing; } void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result) { result.appendNumber("totalInUse", totalInUse); result.appendNumber("totalAvailable", totalAvailable); result.appendNumber("totalCreated", totalCreated); + result.appendNumber("totalRefreshing", totalRefreshing); - BSONObjBuilder hostBuilder(result.subobjStart("hosts")); - for (auto&& host : statsByHost) { - BSONObjBuilder hostInfo(hostBuilder.subobjStart(host.first.toString())); - - auto hostStats = host.second; - hostInfo.appendNumber("inUse", hostStats.inUse); - hostInfo.appendNumber("available", hostStats.available); - hostInfo.appendNumber("created", hostStats.created); + { + BSONObjBuilder poolBuilder(result.subobjStart("pools")); + for (auto&& pool : statsByPool) { + BSONObjBuilder poolInfo(poolBuilder.subobjStart(pool.first)); + auto poolStats = pool.second; + poolInfo.appendNumber("poolInUse", poolStats.inUse); + poolInfo.appendNumber("poolAvailable", poolStats.available); + poolInfo.appendNumber("poolCreated", poolStats.created); + poolInfo.appendNumber("poolRefreshing", poolStats.refreshing); + for (auto&& host : statsByPoolHost[pool.first]) { + BSONObjBuilder hostInfo(poolInfo.subobjStart(host.first.toString())); + auto hostStats = host.second; + hostInfo.appendNumber("inUse", hostStats.inUse); + hostInfo.appendNumber("available", hostStats.available); + hostInfo.appendNumber("created", hostStats.created); + hostInfo.appendNumber("refreshing", hostStats.refreshing); + } + } + } + { + BSONObjBuilder hostBuilder(result.subobjStart("hosts")); + for (auto&& host : statsByHost) { + BSONObjBuilder hostInfo(hostBuilder.subobjStart(host.first.toString())); + auto hostStats = host.second; + hostInfo.appendNumber("inUse", hostStats.inUse); + hostInfo.appendNumber("available", hostStats.available); + hostInfo.appendNumber("created", hostStats.created); + hostInfo.appendNumber("refreshing", hostStats.refreshing); + } } } diff --git a/src/mongo/executor/connection_pool_stats.h b/src/mongo/executor/connection_pool_stats.h index 5e9bd0c7633..148dd89facf 100644 --- a/src/mongo/executor/connection_pool_stats.h +++ b/src/mongo/executor/connection_pool_stats.h @@ -36,19 +36,20 @@ namespace mongo { namespace executor { /** - * Holds connection information for a specific remote host. These objects are maintained by + * Holds connection information for a specific pool or remote host. These objects are maintained by * a parent ConnectionPoolStats object and should not need to be created directly. */ -struct ConnectionStatsPerHost { - ConnectionStatsPerHost(size_t nInUse, size_t nAvailable, size_t nCreated); +struct ConnectionStatsPer { + ConnectionStatsPer(size_t nInUse, size_t nAvailable, size_t nCreated, size_t nRefreshing); - ConnectionStatsPerHost(); + ConnectionStatsPer(); - ConnectionStatsPerHost& operator+=(const ConnectionStatsPerHost& other); + ConnectionStatsPer& operator+=(const ConnectionStatsPer& other); size_t inUse = 0u; size_t available = 0u; size_t created = 0u; + size_t refreshing = 0u; }; /** @@ -57,15 +58,19 @@ struct ConnectionStatsPerHost { * Total connection counts will then be updated accordingly. */ struct ConnectionPoolStats { - void updateStatsForHost(HostAndPort host, ConnectionStatsPerHost newStats); + void updateStatsForHost(std::string pool, HostAndPort host, ConnectionStatsPer newStats); void appendToBSON(mongo::BSONObjBuilder& result); size_t totalInUse = 0u; size_t totalAvailable = 0u; size_t totalCreated = 0u; + size_t totalRefreshing = 0u; - std::unordered_map statsByHost; + std::unordered_map statsByPool; + std::unordered_map statsByHost; + std::unordered_map> + statsByPoolHost; }; } // namespace executor diff --git a/src/mongo/executor/connection_pool_test.cpp b/src/mongo/executor/connection_pool_test.cpp index 24949a142c7..8bd79f6f445 100644 --- a/src/mongo/executor/connection_pool_test.cpp +++ b/src/mongo/executor/connection_pool_test.cpp @@ -66,7 +66,7 @@ private: * another. */ TEST_F(ConnectionPoolTest, SameConn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Grab and stash an id for the first request size_t conn1Id = 0; @@ -98,7 +98,7 @@ TEST_F(ConnectionPoolTest, SameConn) { * Verify that a failed connection isn't returned to the pool */ TEST_F(ConnectionPoolTest, FailedConnDifferentConn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Grab the first connection and indicate that it failed size_t conn1Id = 0; @@ -131,7 +131,7 @@ TEST_F(ConnectionPoolTest, FailedConnDifferentConn) { * connections. */ TEST_F(ConnectionPoolTest, DifferentHostDifferentConn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Conn 1 from port 30000 size_t conn1Id = 0; @@ -163,7 +163,7 @@ TEST_F(ConnectionPoolTest, DifferentHostDifferentConn) { * Verify that not returning handle's to the pool spins up new connections. */ TEST_F(ConnectionPoolTest, DifferentConnWithoutReturn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Get the first connection, move it out rather than letting it return ConnectionPool::ConnectionHandle conn1; @@ -199,7 +199,7 @@ TEST_F(ConnectionPoolTest, DifferentConnWithoutReturn) { * Note that the lack of pushSetup() calls delays the get. */ TEST_F(ConnectionPoolTest, TimeoutOnSetup) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); bool notOk = false; @@ -234,7 +234,7 @@ TEST_F(ConnectionPoolTest, refreshHappens) { ConnectionPool::Options options; options.refreshRequirement = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -270,7 +270,7 @@ TEST_F(ConnectionPoolTest, refreshTimeoutHappens) { ConnectionPool::Options options; options.refreshRequirement = Milliseconds(1000); options.refreshTimeout = Milliseconds(2000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -308,13 +308,14 @@ TEST_F(ConnectionPoolTest, refreshTimeoutHappens) { // see if that pans out. In this case, we'll get a failure on timeout. ConnectionImpl::pushSetup(Status::OK()); pool.get(HostAndPort(), - Milliseconds(10000), + Milliseconds(1000), [&](StatusWith swConn) { ASSERT(!swConn.isOK()); reachedA = true; }); ASSERT(!reachedA); + PoolImpl::setNow(now + Milliseconds(3000)); // Let the refresh timeout PoolImpl::setNow(now + Milliseconds(4000)); @@ -338,7 +339,7 @@ TEST_F(ConnectionPoolTest, refreshTimeoutHappens) { * Verify that requests are served in expiration order, not insertion order */ TEST_F(ConnectionPoolTest, requestsServedByUrgency) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); bool reachedA = false; bool reachedB = false; @@ -385,7 +386,7 @@ TEST_F(ConnectionPoolTest, maxPoolRespected) { ConnectionPool::Options options; options.minConnections = 1; options.maxConnections = 2; - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); ConnectionPool::ConnectionHandle conn1; ConnectionPool::ConnectionHandle conn2; @@ -436,6 +437,202 @@ TEST_F(ConnectionPoolTest, maxPoolRespected) { doneWith(conn3); } +/** + * Verify that we respect maxConnecting + */ +TEST_F(ConnectionPoolTest, maxConnectingRespected) { + ConnectionPool::Options options; + options.minConnections = 1; + options.maxConnecting = 2; + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + ConnectionPool::ConnectionHandle conn1; + ConnectionPool::ConnectionHandle conn2; + ConnectionPool::ConnectionHandle conn3; + + // Make 3 requests, each which keep their connection (don't return it to + // the pool) + pool.get(HostAndPort(), + Milliseconds(3000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + + conn3 = std::move(swConn.getValue()); + }); + pool.get(HostAndPort(), + Milliseconds(2000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + + conn2 = std::move(swConn.getValue()); + }); + pool.get(HostAndPort(), + Milliseconds(1000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + + conn1 = std::move(swConn.getValue()); + }); + + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 1u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + + ASSERT(conn1); + ASSERT(conn2); + ASSERT(conn3); + + ASSERT_NE(conn1.get(), conn2.get()); + ASSERT_NE(conn2.get(), conn3.get()); + ASSERT_NE(conn1.get(), conn3.get()); + + doneWith(conn1); + doneWith(conn2); + doneWith(conn3); +} + +/** + * Verify that refresh callbacks block new connections, then trigger new connection spawns after + * they return + */ +TEST_F(ConnectionPoolTest, maxConnectingWithRefresh) { + ConnectionPool::Options options; + options.maxConnecting = 1; + options.refreshRequirement = Milliseconds(1000); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + + PoolImpl::setNow(now); + + // Get a connection + ConnectionImpl::pushSetup(Status::OK()); + pool.get(HostAndPort(), + Milliseconds(5000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + doneWith(swConn.getValue()); + }); + + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 0u); + + // After 1 second, one refresh has queued + PoolImpl::setNow(now + Milliseconds(1000)); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 1u); + + bool reachedA = false; + + // Try to get another connection + pool.get(HostAndPort(), + Milliseconds(5000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + doneWith(swConn.getValue()); + reachedA = true; + }); + + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + ASSERT(!reachedA); + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 0u); + ASSERT(reachedA); +} + +/** + * Verify that refreshes block new connects, but don't themselves respect maxConnecting + */ +TEST_F(ConnectionPoolTest, maxConnectingWithMultipleRefresh) { + ConnectionPool::Options options; + options.maxConnecting = 2; + options.minConnections = 3; + options.refreshRequirement = Milliseconds(1000); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + + PoolImpl::setNow(now); + + // Get us spun up to 3 connections in the pool + pool.get(HostAndPort(), + Milliseconds(5000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + doneWith(swConn.getValue()); + }); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + + // Force more than two connections into refresh + PoolImpl::setNow(now + Milliseconds(1500)); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 3u); + + std::array conns; + + // Start 5 new requests + for (size_t i = 0; i < conns.size(); ++i) { + pool.get(HostAndPort(), + Milliseconds(static_cast(1000 + i)), + [&conns, i](StatusWith swConn) { + ASSERT(swConn.isOK()); + conns[i] = std::move(swConn.getValue()); + }); + } + + auto firstNBound = [&](size_t n) { + for (size_t i = 0; i < n; ++i) { + ASSERT(conns[i]); + } + for (size_t i = n; i < conns.size(); ++i) { + ASSERT_FALSE(conns[i]); + } + }; + + // None have started connecting + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 3u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + firstNBound(0); + + // After one refresh, one refreshed connection gets handed out + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 2u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + firstNBound(1); + + // After two refresh, one enters the setup queue, one refreshed connection gets handed out + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 1u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 1u); + firstNBound(2); + + // After three refresh, we're done refreshing. Two queued in setup + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 0u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + firstNBound(3); + + // now pushing setup gets us a new connection + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 1u); + firstNBound(4); + + // and we're done + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + firstNBound(5); + + for (auto& conn : conns) { + doneWith(conn); + } +} + /** * Verify that minConnections is respected */ @@ -445,7 +642,7 @@ TEST_F(ConnectionPoolTest, minPoolRespected) { options.maxConnections = 3; options.refreshRequirement = Milliseconds(1000); options.refreshTimeout = Milliseconds(2000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -554,7 +751,7 @@ TEST_F(ConnectionPoolTest, hostTimeoutHappens) { options.refreshRequirement = Milliseconds(5000); options.refreshTimeout = Milliseconds(5000); options.hostTimeout = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -603,7 +800,7 @@ TEST_F(ConnectionPoolTest, hostTimeoutHappensMoreGetsDelay) { options.refreshRequirement = Milliseconds(5000); options.refreshTimeout = Milliseconds(5000); options.hostTimeout = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -665,7 +862,7 @@ TEST_F(ConnectionPoolTest, hostTimeoutHappensCheckoutDelays) { options.refreshRequirement = Milliseconds(5000); options.refreshTimeout = Milliseconds(5000); options.hostTimeout = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -745,7 +942,7 @@ TEST_F(ConnectionPoolTest, dropConnections) { options.maxConnections = 1; options.refreshRequirement = Seconds(1); options.refreshTimeout = Seconds(2); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); PoolImpl::setNow(now); @@ -830,6 +1027,106 @@ TEST_F(ConnectionPoolTest, dropConnections) { ASSERT(reachedB); } +/** + * Verify that timeouts during setup don't prematurely time out unrelated requests + */ +TEST_F(ConnectionPoolTest, SetupTimeoutsDontTimeoutUnrelatedRequests) { + ConnectionPool::Options options; + + options.maxConnections = 1; + options.refreshTimeout = Seconds(2); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + PoolImpl::setNow(now); + + boost::optional> conn1; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn1 = std::move(swConn); }); + + // initially we haven't called our callback + ASSERT(!conn1); + + PoolImpl::setNow(now + Seconds(1)); + + // Still haven't fired on conn1 + ASSERT(!conn1); + + // Get conn2 (which should have an extra second before the timeout) + boost::optional> conn2; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn2 = std::move(swConn); }); + + PoolImpl::setNow(now + Seconds(2)); + + ASSERT(conn1); + ASSERT(!conn1->isOK()); + ASSERT(conn1->getStatus().code() == ErrorCodes::ExceededTimeLimit); + + ASSERT(!conn2); +} + +/** + * Verify that timeouts during refresh don't prematurely time out unrelated requests + */ +TEST_F(ConnectionPoolTest, RefreshTimeoutsDontTimeoutRequests) { + ConnectionPool::Options options; + + options.maxConnections = 1; + options.refreshTimeout = Seconds(2); + options.refreshRequirement = Seconds(3); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + PoolImpl::setNow(now); + + // Successfully get a new connection + size_t conn1Id = 0; + ConnectionImpl::pushSetup(Status::OK()); + pool.get(HostAndPort(), + Seconds(1), + [&](StatusWith swConn) { + conn1Id = CONN2ID(swConn); + doneWith(swConn.getValue()); + }); + ASSERT(conn1Id); + + // Force it into refresh + PoolImpl::setNow(now + Seconds(3)); + + boost::optional> conn1; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn1 = std::move(swConn); }); + + // initially we haven't called our callback + ASSERT(!conn1); + + // 1 second later we've triggered a refresh and still haven't called the callback + PoolImpl::setNow(now + Seconds(4)); + ASSERT(!conn1); + + // Get conn2 (which should have an extra second before the timeout) + boost::optional> conn2; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn2 = std::move(swConn); }); + + PoolImpl::setNow(now + Seconds(5)); + + ASSERT(conn1); + ASSERT(!conn1->isOK()); + ASSERT(conn1->getStatus().code() == ErrorCodes::ExceededTimeLimit); + + ASSERT(!conn2); +} + } // namespace connection_pool_test_details } // namespace executor } // namespace mongo diff --git a/src/mongo/executor/connection_pool_test_fixture.cpp b/src/mongo/executor/connection_pool_test_fixture.cpp index 2d48ad0b5e2..e8aff615448 100644 --- a/src/mongo/executor/connection_pool_test_fixture.cpp +++ b/src/mongo/executor/connection_pool_test_fixture.cpp @@ -116,9 +116,13 @@ void ConnectionImpl::pushSetup(PushSetupCallback status) { _pushSetupQueue.push_back(status); if (_setupQueue.size()) { - _setupQueue.front()->_setupCallback(_setupQueue.front(), _pushSetupQueue.front()()); + auto connPtr = _setupQueue.front(); + auto callback = _pushSetupQueue.front(); _setupQueue.pop_front(); _pushSetupQueue.pop_front(); + + auto cb = connPtr->_setupCallback; + cb(connPtr, callback()); } } @@ -126,13 +130,22 @@ void ConnectionImpl::pushSetup(Status status) { pushSetup([status]() { return status; }); } +size_t ConnectionImpl::setupQueueDepth() { + return _setupQueue.size(); +} + void ConnectionImpl::pushRefresh(PushRefreshCallback status) { _pushRefreshQueue.push_back(status); if (_refreshQueue.size()) { - _refreshQueue.front()->_refreshCallback(_refreshQueue.front(), _pushRefreshQueue.front()()); + auto connPtr = _refreshQueue.front(); + auto callback = _pushRefreshQueue.front(); + _refreshQueue.pop_front(); _pushRefreshQueue.pop_front(); + + auto cb = connPtr->_refreshCallback; + cb(connPtr, callback()); } } @@ -140,6 +153,10 @@ void ConnectionImpl::pushRefresh(Status status) { pushRefresh([status]() { return status; }); } +size_t ConnectionImpl::refreshQueueDepth() { + return _refreshQueue.size(); +} + Date_t ConnectionImpl::getLastUsed() const { return _lastUsed; } @@ -166,9 +183,13 @@ void ConnectionImpl::setup(Milliseconds timeout, SetupCallback cb) { _setupQueue.push_back(this); if (_pushSetupQueue.size()) { - _setupQueue.front()->_setupCallback(_setupQueue.front(), _pushSetupQueue.front()()); + auto connPtr = _setupQueue.front(); + auto callback = _pushSetupQueue.front(); _setupQueue.pop_front(); _pushSetupQueue.pop_front(); + + auto cb = connPtr->_setupCallback; + cb(connPtr, callback()); } } @@ -182,9 +203,14 @@ void ConnectionImpl::refresh(Milliseconds timeout, RefreshCallback cb) { _refreshQueue.push_back(this); if (_pushRefreshQueue.size()) { - _refreshQueue.front()->_refreshCallback(_refreshQueue.front(), _pushRefreshQueue.front()()); + auto connPtr = _refreshQueue.front(); + auto callback = _pushRefreshQueue.front(); + _refreshQueue.pop_front(); _pushRefreshQueue.pop_front(); + + auto cb = connPtr->_refreshCallback; + cb(connPtr, callback()); } } diff --git a/src/mongo/executor/connection_pool_test_fixture.h b/src/mongo/executor/connection_pool_test_fixture.h index 74946ba5cf4..8509e03171c 100644 --- a/src/mongo/executor/connection_pool_test_fixture.h +++ b/src/mongo/executor/connection_pool_test_fixture.h @@ -95,10 +95,12 @@ public: // Push either a callback that returns the status for a setup, or just the Status static void pushSetup(PushSetupCallback status); static void pushSetup(Status status); + static size_t setupQueueDepth(); // Push either a callback that returns the status for a refresh, or just the Status static void pushRefresh(PushRefreshCallback status); static void pushRefresh(Status status); + static size_t refreshQueueDepth(); private: void indicateUsed() override; diff --git a/src/mongo/executor/network_interface.h b/src/mongo/executor/network_interface.h index e35f8ed9c52..3dfae564d91 100644 --- a/src/mongo/executor/network_interface.h +++ b/src/mongo/executor/network_interface.h @@ -151,6 +151,11 @@ public: */ virtual bool onNetworkThread() = 0; + /** + * Drops all connections to the given host in the connection pool. + */ + virtual void dropConnections(const HostAndPort& hostAndPort) = 0; + protected: NetworkInterface(); }; diff --git a/src/mongo/executor/network_interface_asio.cpp b/src/mongo/executor/network_interface_asio.cpp index 3900b92f4e5..60db2b52e95 100644 --- a/src/mongo/executor/network_interface_asio.cpp +++ b/src/mongo/executor/network_interface_asio.cpp @@ -86,6 +86,7 @@ NetworkInterfaceASIO::NetworkInterfaceASIO(Options options) _timerFactory(std::move(_options.timerFactory)), _streamFactory(std::move(_options.streamFactory)), _connectionPool(stdx::make_unique(this), + _options.instanceName, _options.connectionPoolOptions), _isExecutorRunnable(false), _strand(_io_service) {} @@ -392,5 +393,9 @@ bool NetworkInterfaceASIO::onNetworkThread() { [id](const stdx::thread& thread) { return id == thread.get_id(); }); } +void NetworkInterfaceASIO::dropConnections(const HostAndPort& hostAndPort) { + _connectionPool.dropConnections(hostAndPort); +} + } // namespace executor } // namespace mongo diff --git a/src/mongo/executor/network_interface_asio.h b/src/mongo/executor/network_interface_asio.h index b4af49ae602..426515f4ee5 100644 --- a/src/mongo/executor/network_interface_asio.h +++ b/src/mongo/executor/network_interface_asio.h @@ -121,6 +121,8 @@ public: bool inShutdown() const; + void dropConnections(const HostAndPort& hostAndPort) override; + private: using ResponseStatus = TaskExecutor::ResponseStatus; using NetworkInterface::RemoteCommandCompletionFn; diff --git a/src/mongo/executor/network_interface_asio_auth.cpp b/src/mongo/executor/network_interface_asio_auth.cpp index b9a341d13d6..dcf8dc8a5ce 100644 --- a/src/mongo/executor/network_interface_asio_auth.cpp +++ b/src/mongo/executor/network_interface_asio_auth.cpp @@ -40,6 +40,7 @@ #include "mongo/db/commands.h" #include "mongo/db/server_options.h" #include "mongo/rpc/factory.h" +#include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/legacy_request_builder.h" #include "mongo/rpc/reply_interface.h" #include "mongo/stdx/memory.h" @@ -90,6 +91,12 @@ void NetworkInterfaceASIO::_runIsMaster(AsyncOp* op) { auto commandReply = std::move(swCommandReply.getValue()); + // Ensure that the isMaster response is "ok:1". + auto commandStatus = getStatusFromCommandResult(commandReply.data); + if (!commandStatus.isOK()) { + return _completeOperation(op, commandStatus); + } + auto protocolSet = rpc::parseProtocolSetFromIsMasterReply(commandReply.data); if (!protocolSet.isOK()) return _completeOperation(op, protocolSet.getStatus()); @@ -175,8 +182,7 @@ void NetworkInterfaceASIO::_authenticate(AsyncOp* op) { }; auto params = getInternalUserAuthParamsWithFallback(); - auth::authenticateClient( - params, op->request().target.host(), clientName, runCommandHook, authHook); + auth::authenticateClient(params, op->request().target, clientName, runCommandHook, authHook); } } // namespace executor diff --git a/src/mongo/executor/network_interface_asio_command.cpp b/src/mongo/executor/network_interface_asio_command.cpp index 88daf7aafe4..57b77298b42 100644 --- a/src/mongo/executor/network_interface_asio_command.cpp +++ b/src/mongo/executor/network_interface_asio_command.cpp @@ -235,7 +235,11 @@ void NetworkInterfaceASIO::_beginCommunication(AsyncOp* op) { // codepath. if (op->_inSetup) { - log() << "Successfully connected to " << op->request().target.toString(); + auto host = op->request().target; + auto getConnectionDuration = now() - op->start(); + log() << "Successfully connected to " << host << ", took " << getConnectionDuration << " (" + << _connectionPool.getNumConnectionsPerHost(host) << " connections now open to " + << host << ")"; op->_inSetup = false; op->finish(RemoteCommandResponse()); return; diff --git a/src/mongo/executor/network_interface_asio_integration_test.cpp b/src/mongo/executor/network_interface_asio_integration_test.cpp index 8de7c71ebc4..972996304dc 100644 --- a/src/mongo/executor/network_interface_asio_integration_test.cpp +++ b/src/mongo/executor/network_interface_asio_integration_test.cpp @@ -63,7 +63,13 @@ public: void startNet(NetworkInterfaceASIO::Options options = NetworkInterfaceASIO::Options()) { options.streamFactory = stdx::make_unique(); options.timerFactory = stdx::make_unique(); +#ifdef _WIN32 + // Connections won't queue on windows, so attempting to open too many connections + // concurrently will result in refused connections and test failure. + options.connectionPoolOptions.maxConnections = 16u; +#else options.connectionPoolOptions.maxConnections = 256u; +#endif _net = stdx::make_unique(std::move(options)); _net->startup(); } @@ -232,7 +238,7 @@ private: }; TEST_F(NetworkInterfaceASIOIntegrationTest, StressTest) { - const std::size_t numOps = 1000; + const std::size_t numOps = 500; std::vector testResults(numOps, {ErrorCodes::InternalError, "uninitialized"}); ErrorCodes::Error expectedResults[numOps]; CountdownLatch cl(numOps); @@ -249,7 +255,7 @@ TEST_F(NetworkInterfaceASIOIntegrationTest, StressTest) { for (std::size_t i = 0; i < numOps; ++i) { // stagger operations slightly to mitigate connection pool contention - sleepmillis(rng.nextInt32(16)); + sleepmillis(rng.nextInt32(50)); auto r = rng.nextCanonicalDouble(); diff --git a/src/mongo/executor/network_interface_asio_operation.cpp b/src/mongo/executor/network_interface_asio_operation.cpp index 2a75fb2c233..0c3e29d029c 100644 --- a/src/mongo/executor/network_interface_asio_operation.cpp +++ b/src/mongo/executor/network_interface_asio_operation.cpp @@ -204,6 +204,9 @@ NetworkInterfaceASIO::AsyncCommand* NetworkInterfaceASIO::AsyncOp::command() { } void NetworkInterfaceASIO::AsyncOp::finish(const ResponseStatus& status) { + LOG(2) << "Request " << _request.id << " finished with response: " + << (status.getStatus().isOK() ? status.getValue().data.toString() + : status.getStatus().toString()); _onFinish(status); } diff --git a/src/mongo/executor/network_interface_asio_test.cpp b/src/mongo/executor/network_interface_asio_test.cpp index bbe75be5f49..144a0ad91d4 100644 --- a/src/mongo/executor/network_interface_asio_test.cpp +++ b/src/mongo/executor/network_interface_asio_test.cpp @@ -533,6 +533,46 @@ public: } }; +TEST_F(NetworkInterfaceASIOConnectionHookTest, InvalidIsMaster) { + auto validationFailedStatus = + Status(ErrorCodes::InterruptedDueToReplStateChange, "operation was interrupted"); + + start(makeTestHook( + [&](const HostAndPort& remoteHost, const RemoteCommandResponse& isMasterReply) { + return Status(ErrorCodes::UnknownError, "unused"); + }, + [&](const HostAndPort& remoteHost) + -> StatusWith> { return {boost::none}; }, + [&](const HostAndPort& remoteHost, RemoteCommandResponse&& response) { + return Status::OK(); + })); + + auto deferred = startCommand(makeCallbackHandle(), + {testHost, + "blah", + BSON("foo" + << "bar")}); + + auto stream = streamFactory().blockUntilStreamExists(testHost); + + ConnectEvent{stream}.skip(); + + // simulate isMaster reply. + stream->simulateServer(rpc::Protocol::kOpQuery, + [](RemoteCommandRequest request) -> RemoteCommandResponse { + RemoteCommandResponse response; + response.data = BSON("ok" << 0.0 << "errmsg" + << "operation was interrupted" + << "code" << 11602); + return response; + }); + + // we should stop here. + auto& res = deferred.get(); + + ASSERT(res == validationFailedStatus); +} + TEST_F(NetworkInterfaceASIOConnectionHookTest, ValidateHostInvalid) { bool validateCalled = false; bool hostCorrect = false; diff --git a/src/mongo/executor/network_interface_impl.h b/src/mongo/executor/network_interface_impl.h index 5915d2d99d6..76759dd454b 100644 --- a/src/mongo/executor/network_interface_impl.h +++ b/src/mongo/executor/network_interface_impl.h @@ -96,6 +96,7 @@ public: void cancelAllCommands() override {} void setAlarm(Date_t when, const stdx::function& action) override; bool onNetworkThread() override; + void dropConnections(const HostAndPort& hostAndPort) override {} private: /** diff --git a/src/mongo/executor/network_interface_mock.h b/src/mongo/executor/network_interface_mock.h index 439b33e5caf..4096b8f68ac 100644 --- a/src/mongo/executor/network_interface_mock.h +++ b/src/mongo/executor/network_interface_mock.h @@ -104,6 +104,8 @@ public: virtual bool onNetworkThread(); + void dropConnections(const HostAndPort&) override {} + //////////////////////////////////////////////////////////////////////////////// // diff --git a/src/mongo/executor/thread_pool_task_executor.cpp b/src/mongo/executor/thread_pool_task_executor.cpp index 570ac69826e..e33e2bcc487 100644 --- a/src/mongo/executor/thread_pool_task_executor.cpp +++ b/src/mongo/executor/thread_pool_task_executor.cpp @@ -273,6 +273,9 @@ StatusWith ThreadPoolTaskExecutor::scheduleWorkAt( } invariant(now() >= when); stdx::unique_lock lk(_mutex); + if (cbState->canceled.load()) { + return; + } scheduleIntoPool_inlock(&_sleepersQueue, cbState->iter, std::move(lk)); }); @@ -469,22 +472,26 @@ void ThreadPoolTaskExecutor::scheduleIntoPool_inlock(WorkQueue* fromQueue, } void ThreadPoolTaskExecutor::runCallback(std::shared_ptr cbStateArg) { - auto cbStatePtr = cbStateArg.get(); CallbackHandle cbHandle; - setCallbackForHandle(&cbHandle, std::move(cbStateArg)); + setCallbackForHandle(&cbHandle, cbStateArg); CallbackArgs args(this, std::move(cbHandle), - cbStatePtr->canceled.load() + cbStateArg->canceled.load() ? Status({ErrorCodes::CallbackCanceled, "Callback canceled"}) : Status::OK()); - cbStatePtr->callback(std::move(args)); - cbStatePtr->isFinished.store(true); + invariant(!cbStateArg->isFinished.load()); + cbStateArg->callback(std::move(args)); + cbStateArg->isFinished.store(true); stdx::lock_guard lk(_mutex); - _poolInProgressQueue.erase(cbStatePtr->iter); - if (cbStatePtr->finishedCondition) { - cbStatePtr->finishedCondition->notify_all(); + _poolInProgressQueue.erase(cbStateArg->iter); + if (cbStateArg->finishedCondition) { + cbStateArg->finishedCondition->notify_all(); } } +void ThreadPoolTaskExecutor::dropConnections(const HostAndPort& hostAndPort) { + _net->dropConnections(hostAndPort); +} + } // namespace executor } // namespace mongo diff --git a/src/mongo/executor/thread_pool_task_executor.h b/src/mongo/executor/thread_pool_task_executor.h index f4afb7b58c9..ae4ce9bb40e 100644 --- a/src/mongo/executor/thread_pool_task_executor.h +++ b/src/mongo/executor/thread_pool_task_executor.h @@ -88,6 +88,11 @@ public: */ void cancelAllCommands(); + /** + * Drops all connections to the given host on the network interface. + */ + void dropConnections(const HostAndPort& hostAndPort); + private: class CallbackState; class EventState; diff --git a/src/mongo/installer/msi/SConscript b/src/mongo/installer/msi/SConscript index f17d3523464..f8fe85d7bc2 100644 --- a/src/mongo/installer/msi/SConscript +++ b/src/mongo/installer/msi/SConscript @@ -125,7 +125,7 @@ env.Command(objects, ' -dTargetFileName=${SERVER_ARCHIVE}' r' -dSaslSource=c:\sasl\bin' r' -dSnmpSource=c:\snmp\bin' - r' -dSslSource=c:\openssl\bin' + r' -dSslSource=' + env['WINDOWS_OPENSSL_BIN'] + ' -out ' + buildDir + r'\msi\\' ' -arch ' + msi_platform + ' -ext "$WIXUIEXT"' diff --git a/src/mongo/logger/logstream_builder.cpp b/src/mongo/logger/logstream_builder.cpp index 2577704aad0..199991c7dd7 100644 --- a/src/mongo/logger/logstream_builder.cpp +++ b/src/mongo/logger/logstream_builder.cpp @@ -81,12 +81,14 @@ LogstreamBuilder::LogstreamBuilder(MessageLogDomain* domain, LogstreamBuilder::LogstreamBuilder(MessageLogDomain* domain, std::string contextName, LogSeverity severity, - LogComponent component) + LogComponent component, + bool shouldCache) : _domain(domain), _contextName(std::move(contextName)), _severity(std::move(severity)), _component(std::move(component)), - _tee(nullptr) {} + _tee(nullptr), + _shouldCache(shouldCache) {} LogstreamBuilder::LogstreamBuilder(logger::MessageLogDomain* domain, const std::string& contextName, @@ -103,7 +105,8 @@ LogstreamBuilder::LogstreamBuilder(LogstreamBuilder&& other) _baseMessage(std::move(other._baseMessage)), _os(std::move(other._os)), _tee(std::move(other._tee)), - _isTruncatable(other._isTruncatable) {} + _isTruncatable(other._isTruncatable), + _shouldCache(other._shouldCache) {} LogstreamBuilder& LogstreamBuilder::operator=(LogstreamBuilder&& other) { _domain = std::move(other._domain); @@ -114,6 +117,7 @@ LogstreamBuilder& LogstreamBuilder::operator=(LogstreamBuilder&& other) { _os = std::move(other._os); _tee = std::move(other._tee); _isTruncatable = std::move(other._isTruncatable); + _shouldCache = other._shouldCache; return *this; } @@ -134,7 +138,8 @@ LogstreamBuilder::~LogstreamBuilder() { _tee->write(_os->str()); } _os->str(""); - if (isThreadOstreamCacheInitialized && !threadOstreamCache.getMake()->get()) { + if (_shouldCache && isThreadOstreamCacheInitialized && + !threadOstreamCache.getMake()->get()) { *threadOstreamCache.get() = std::move(_os); } } @@ -148,7 +153,8 @@ void LogstreamBuilder::operator<<(Tee* tee) { void LogstreamBuilder::makeStream() { if (!_os) { - if (isThreadOstreamCacheInitialized && threadOstreamCache.getMake()->get()) { + if (_shouldCache && isThreadOstreamCacheInitialized && + threadOstreamCache.getMake()->get()) { _os = std::move(*threadOstreamCache.get()); } else { _os = stdx::make_unique(); diff --git a/src/mongo/logger/logstream_builder.h b/src/mongo/logger/logstream_builder.h index 40f56ccf75f..f829621c8a7 100644 --- a/src/mongo/logger/logstream_builder.h +++ b/src/mongo/logger/logstream_builder.h @@ -72,11 +72,17 @@ public: * "contextName" is a short name of the thread or other context. * "severity" is the logging severity of the message. * "component" is the primary log component of the message. + * + * By default, this class will create one ostream per thread, and it + * will cache that object in a threadlocal and reuse it for subsequent + * logs messages. Set "shouldCache" to false to create a new ostream + * for each instance of this class rather than cacheing. */ LogstreamBuilder(MessageLogDomain* domain, std::string contextName, LogSeverity severity, - LogComponent component); + LogComponent component, + bool shouldCache = true); /** * Deprecated. @@ -236,6 +242,7 @@ private: std::unique_ptr _os; Tee* _tee; bool _isTruncatable = true; + bool _shouldCache; }; diff --git a/src/mongo/logger/message_event_utf8_encoder.cpp b/src/mongo/logger/message_event_utf8_encoder.cpp index 3483a113d1e..2a09e046736 100644 --- a/src/mongo/logger/message_event_utf8_encoder.cpp +++ b/src/mongo/logger/message_event_utf8_encoder.cpp @@ -46,6 +46,14 @@ MessageEventDetailsEncoder::DateFormatter MessageEventDetailsEncoder::getDateFor return _dateFormatter; } +namespace { +#ifdef _WIN32 +const auto kEOL = StringData("\r\n", StringData::LiteralTag()); +#else +const auto kEOL = StringData("\n", StringData::LiteralTag()); +#endif +} // namespace + MessageEventDetailsEncoder::~MessageEventDetailsEncoder() {} std::ostream& MessageEventDetailsEncoder::encode(const MessageEventEphemeral& event, std::ostream& os) { @@ -67,6 +75,26 @@ std::ostream& MessageEventDetailsEncoder::encode(const MessageEventEphemeral& ev } StringData msg = event.getMessage(); + +#ifdef _WIN32 + // We need to translate embedded Unix style line endings into Windows style endings. + std::string tempstr; + size_t embeddedNewLine = msg.find('\n'); + + if (embeddedNewLine != std::string::npos) { + tempstr = msg.toString().replace(embeddedNewLine, 1, "\r\n"); + + embeddedNewLine = tempstr.find('\n', embeddedNewLine + 2); + while (embeddedNewLine != std::string::npos) { + tempstr = tempstr.replace(embeddedNewLine, 1, "\r\n"); + + embeddedNewLine = tempstr.find('\n', embeddedNewLine + 2); + } + + msg = tempstr; + } +#endif + if (event.isTruncatable() && msg.size() > maxLogLine) { os << "warning: log line attempted (" << msg.size() / 1024 << "kB) over max size (" << maxLogLine / 1024 << "kB), printing beginning and end ... "; @@ -76,8 +104,10 @@ std::ostream& MessageEventDetailsEncoder::encode(const MessageEventEphemeral& ev } else { os << msg; } - if (!msg.endsWith(StringData("\n", StringData::LiteralTag()))) - os << '\n'; + + if (!msg.endsWith(kEOL)) + os << kEOL; + return os; } diff --git a/src/mongo/logger/ramlog.cpp b/src/mongo/logger/ramlog.cpp index e9d7685d49f..6ce15dba3a4 100644 --- a/src/mongo/logger/ramlog.cpp +++ b/src/mongo/logger/ramlog.cpp @@ -50,8 +50,7 @@ RM* _named = NULL; } // namespace RamLog::RamLog(const std::string& name) : _name(name), _totalLinesWritten(0), _lastWrite(0) { - h = 0; - n = 0; + clear(); for (int i = 0; i < N; i++) lines[i][C - 1] = 0; } @@ -84,6 +83,16 @@ void RamLog::write(const std::string& str) { h = (h + 1) % N; } +void RamLog::clear() { + stdx::lock_guard lk(_mutex); + _totalLinesWritten = 0; + _lastWrite = 0; + h = 0; + n = 0; + for (int i = 0; i < N; i++) + lines[i][0] = 0; +} + time_t RamLog::LineIterator::lastWrite() { return _ramlog->_lastWrite; } diff --git a/src/mongo/logger/ramlog.h b/src/mongo/logger/ramlog.h index c69fb1e4814..8e087c6d531 100644 --- a/src/mongo/logger/ramlog.h +++ b/src/mongo/logger/ramlog.h @@ -93,6 +93,11 @@ public: */ void write(const std::string& str); + /** + * Empties out the RamLog. + */ + void clear(); + /** * Writes an HTML representation of the log to "s". diff --git a/src/mongo/logger/rotatable_file_writer.cpp b/src/mongo/logger/rotatable_file_writer.cpp index 9b23952ac02..d1c2543f20f 100644 --- a/src/mongo/logger/rotatable_file_writer.cpp +++ b/src/mongo/logger/rotatable_file_writer.cpp @@ -97,8 +97,6 @@ private: virtual std::streamsize xsputn(const char* s, std::streamsize count); virtual int_type overflow(int_type ch = traits_type::eof()); - std::streamsize writeToFile(const char* s, std::streamsize count); - HANDLE _fileHandle; }; @@ -163,7 +161,8 @@ bool Win32FileStreambuf::open(StringData fileName, bool append) { return false; } -std::streamsize Win32FileStreambuf::writeToFile(const char* s, std::streamsize count) { +// Called when strings are written to ostream +std::streamsize Win32FileStreambuf::xsputn(const char* s, std::streamsize count) { DWORD totalBytesWritten = 0; while (count > totalBytesWritten) { @@ -171,32 +170,8 @@ std::streamsize Win32FileStreambuf::writeToFile(const char* s, std::streamsize c if (!WriteFile(_fileHandle, s, count - totalBytesWritten, &bytesWritten, NULL)) { break; } - totalBytesWritten += bytesWritten; - } - - return totalBytesWritten; -} - -// Called when strings are written to ostream -std::streamsize Win32FileStreambuf::xsputn(const char* s, std::streamsize count) { - DWORD totalBytesWritten = 0; - - // Scan for embedded newlines before end - // this should be rare since the newline should only be at the end - const char* startPos = s; - for (int i = 0; i < count; i++) { - if (s[i] == '\n') { - totalBytesWritten += writeToFile(startPos, i - (startPos - s)); - writeToFile("\r\n", 2); - totalBytesWritten += 1; // Caller expected we only wrote 1 char, so tell them so - startPos = &s[i + 1]; - } - } - // Did the string not end on "\n"? Write the remaining, no need for CRLF - // as upper layers are responsible for it - if ((startPos - s) != count) { - totalBytesWritten += writeToFile(startPos, count - (startPos - s)); + totalBytesWritten += bytesWritten; } return totalBytesWritten; diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 5400114f85d..743918fdabb 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -31,6 +31,7 @@ env.Library( 'cluster_find_and_modify_cmd.cpp', 'cluster_flush_router_config_cmd.cpp', 'cluster_fsync_cmd.cpp', + 'cluster_ftdc_commands.cpp', 'cluster_get_last_error_cmd.cpp', 'cluster_get_prev_error_cmd.cpp', 'cluster_get_shard_version_cmd.cpp', diff --git a/src/mongo/s/commands/cluster_ftdc_commands.cpp b/src/mongo/s/commands/cluster_ftdc_commands.cpp new file mode 100644 index 00000000000..29c80482c58 --- /dev/null +++ b/src/mongo/s/commands/cluster_ftdc_commands.cpp @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/init.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/operation_context.h" + +namespace mongo { +namespace { + +/** + * getDiagnosticData is a MongoD only command. We implement in MongoS to give users a better error + * message. + */ +class GetDiagnosticDataCommand final : public Command { +public: + GetDiagnosticDataCommand() : Command("getDiagnosticData") {} + + bool adminOnly() const override { + return true; + } + + void help(std::stringstream& help) const override { + help << "get latest diagnostic data collection snapshot"; + } + + bool slaveOk() const override { + return true; + } + + bool isWriteCommandForConfigServer() const override { + return false; + } + + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) override { + return Status::OK(); + } + + bool run(OperationContext* txn, + const std::string& db, + BSONObj& cmdObj, + int options, + std::string& errmsg, + BSONObjBuilder& result) override { + errmsg = "getDiagnosticData not allowed through mongos"; + + return false; + } +}; + +Command* ftdcCommand; + +MONGO_INITIALIZER(CreateDiagnosticDataCommand)(InitializerContext* context) { + ftdcCommand = new GetDiagnosticDataCommand(); + + return Status::OK(); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/s/commands/cluster_list_databases_cmd.cpp b/src/mongo/s/commands/cluster_list_databases_cmd.cpp index 23dad76f0a3..b55636965ef 100644 --- a/src/mongo/s/commands/cluster_list_databases_cmd.cpp +++ b/src/mongo/s/commands/cluster_list_databases_cmd.cpp @@ -87,6 +87,8 @@ public: int options, std::string& errmsg, BSONObjBuilder& result) { + const bool nameOnly = cmdObj["nameOnly"].trueValue(); + map sizes; map> dbShardInfo; @@ -152,9 +154,11 @@ public: BSONObjBuilder temp; temp.append("name", name); - temp.appendNumber("sizeOnDisk", size); - temp.appendBool("empty", size == 1); - temp.append("shards", dbShardInfo[name]->obj()); + if (!nameOnly) { + temp.appendNumber("sizeOnDisk", size); + temp.appendBool("empty", size == 1); + temp.append("shards", dbShardInfo[name]->obj()); + } dbListBuilder.append(temp.obj()); } @@ -168,6 +172,9 @@ public: dbListBuilder.done(); + if (nameOnly) + return true; + result.appendNumber("totalSize", totalSize); result.appendNumber("totalSizeMb", totalSize / (1024 * 1024)); diff --git a/src/mongo/s/commands/cluster_pipeline_cmd.cpp b/src/mongo/s/commands/cluster_pipeline_cmd.cpp index 48544a41c3d..16ef0b05a55 100644 --- a/src/mongo/s/commands/cluster_pipeline_cmd.cpp +++ b/src/mongo/s/commands/cluster_pipeline_cmd.cpp @@ -51,6 +51,7 @@ #include "mongo/s/commands/cluster_commands_common.h" #include "mongo/s/config.h" #include "mongo/s/grid.h" +#include "mongo/s/query/cluster_query_knobs.h" #include "mongo/s/query/store_possible_cursor.h" #include "mongo/s/stale_exception.h" #include "mongo/util/log.h" @@ -237,7 +238,8 @@ public: // Run merging command on random shard, unless a stage needs the primary shard. Need to use // ShardConnection so that the merging mongod is sent the config servers on connection init. auto& prng = txn->getClient()->getPrng(); - const auto& mergingShardId = needPrimaryShardMerger + const auto& mergingShardId = + (needPrimaryShardMerger || internalQueryAlwaysMergeOnPrimaryShard) ? conf->getPrimaryId() : shardResults[prng.nextInt32(shardResults.size())].shardTargetId; const auto mergingShard = grid.shardRegistry()->getShard(txn, mergingShardId); diff --git a/src/mongo/s/query/SConscript b/src/mongo/s/query/SConscript index 18022507aa2..fd27f991f01 100644 --- a/src/mongo/s/query/SConscript +++ b/src/mongo/s/query/SConscript @@ -6,6 +6,7 @@ env.Library( target="cluster_query", source=[ "cluster_find.cpp", + "cluster_query_knobs.cpp", ], LIBDEPS=[ '$BUILD_DIR/mongo/db/commands', diff --git a/src/mongo/s/query/async_results_merger.cpp b/src/mongo/s/query/async_results_merger.cpp index 984c2f85184..18f5b3a6c07 100644 --- a/src/mongo/s/query/async_results_merger.cpp +++ b/src/mongo/s/query/async_results_merger.cpp @@ -332,8 +332,9 @@ StatusWith AsyncResultsMerger::nextEvent() for (size_t i = 0; i < _remotes.size(); ++i) { auto& remote = _remotes[i]; - // It is illegal to call this method if there is an error received from any shard. - invariant(remote.status.isOK()); + if (!remote.status.isOK()) { + return remote.status; + } if (!remote.hasNext() && !remote.exhausted() && !remote.cbHandle.isValid()) { // If we already have established a cursor with this remote, and there is no outstanding @@ -474,6 +475,12 @@ void AsyncResultsMerger::handleBatchResponse( return; } else { remote.status = cursorResponseStatus.getStatus(); + if (remote.status == ErrorCodes::CallbackCanceled) { + // This callback should only be canceled as part of the shutdown sequence, so we + // promote a canceled callback error to an error that will make more sense to the + // client. + remote.status = Status(ErrorCodes::ShutdownInProgress, "shutdown in progress"); + } } // Unreachable host errors are swallowed if the 'allowPartialResults' option is set. We diff --git a/src/mongo/s/query/async_results_merger.h b/src/mongo/s/query/async_results_merger.h index 4f1d8e7ca58..eea7c330dc7 100644 --- a/src/mongo/s/query/async_results_merger.h +++ b/src/mongo/s/query/async_results_merger.h @@ -159,11 +159,12 @@ public: * Starts shutting down this ARM. Returns a handle to an event which is signaled when this * cursor is safe to destroy. * - * Returns an invalid handle if the underlying task executor is shutting down. In this case, it - * is legal to destroy the cursor only after the task executor shutdown process is complete. + * Returns an invalid handle if the underlying task executor is shutting down. In this case, + * killing is considered complete and the ARM may be destroyed immediately. * - * An ARM can only be destroyed if either 1) all its results have been exhausted or 2) the kill - * event returned by this method has been signaled. + * When the underlying task executor is *not* shutting down, an ARM can only be destroyed if + * either 1) all its results have been exhausted or 2) the kill event returned by this method + * has been signaled. * * May be called multiple times (idempotent). */ diff --git a/src/mongo/s/query/async_results_merger_test.cpp b/src/mongo/s/query/async_results_merger_test.cpp index 31cf4228bcf..2dd95b9708b 100644 --- a/src/mongo/s/query/async_results_merger_test.cpp +++ b/src/mongo/s/query/async_results_merger_test.cpp @@ -787,7 +787,7 @@ TEST_F(AsyncResultsMergerTest, NextEventAfterTaskExecutorShutdown) { BSONObj findCmd = fromjson("{find: 'testcoll'}"); makeCursorFromFindCmd(findCmd, kTestShardIds); executor->shutdown(); - ASSERT_NOT_OK(arm->nextEvent().getStatus()); + ASSERT_EQ(ErrorCodes::ShutdownInProgress, arm->nextEvent().getStatus()); auto killEvent = arm->kill(); ASSERT_FALSE(killEvent.isValid()); } @@ -1437,6 +1437,48 @@ TEST_F(AsyncResultsMergerTest, GetMoreRequestWithoutAwaitDataCantHaveMaxTime) { executor->waitForEvent(killEvent); } +TEST_F(AsyncResultsMergerTest, ShardCanErrorInBetweenReadyAndNextEvent) { + BSONObj findCmd = fromjson("{find: 'testcoll', tailable: true}"); + makeCursorFromFindCmd(findCmd, {kTestShardIds[0]}); + + ASSERT_FALSE(arm->ready()); + auto readyEvent = unittest::assertGet(arm->nextEvent()); + scheduleErrorResponse({ErrorCodes::BadValue, "bad thing happened"}); + + ASSERT_EQ(ErrorCodes::BadValue, arm->nextEvent().getStatus()); + + // Required to kill the 'arm' on error before destruction. + auto killEvent = arm->kill(); + executor->waitForEvent(killEvent); +} + +TEST_F(AsyncResultsMergerTest, RetryWhenShardHasRetriableErrorInBetweenReadyAndNextEvent) { + BSONObj findCmd = fromjson("{find: 'testcoll'}"); + makeCursorFromFindCmd(findCmd, {kTestShardIds[0]}); + + ASSERT_FALSE(arm->ready()); + + // First attempt returns a retriable error. + auto readyEvent = unittest::assertGet(arm->nextEvent()); + scheduleErrorResponse({ErrorCodes::NotMasterNoSlaveOk, "not master and not slave"}); + + // We expect to be able to retrieve another event, and be waiting on the retry to succeed. + readyEvent = unittest::assertGet(arm->nextEvent()); + std::vector responses; + std::vector batch = {fromjson("{_id: 1}"), fromjson("{_id: 2}")}; + responses.emplace_back(_nss, CursorId(0), batch); + scheduleNetworkResponses(std::move(responses), CursorResponse::ResponseType::InitialResponse); + + executor->waitForEvent(readyEvent); + ASSERT_TRUE(arm->ready()); + ASSERT_EQ(fromjson("{_id: 1}"), *unittest::assertGet(arm->nextReady())); + ASSERT_TRUE(arm->ready()); + ASSERT_EQ(fromjson("{_id: 2}"), *unittest::assertGet(arm->nextReady())); + ASSERT_TRUE(arm->ready()); + ASSERT_TRUE(arm->remotesExhausted()); + ASSERT(!unittest::assertGet(arm->nextReady())); +} + } // namespace } // namespace mongo diff --git a/src/mongo/s/query/cluster_query_knobs.cpp b/src/mongo/s/query/cluster_query_knobs.cpp new file mode 100644 index 00000000000..76e82b4f914 --- /dev/null +++ b/src/mongo/s/query/cluster_query_knobs.cpp @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/s/query/cluster_query_knobs.h" + +#include "mongo/db/server_parameters.h" + +namespace mongo { + +MONGO_EXPORT_SERVER_PARAMETER(internalQueryAlwaysMergeOnPrimaryShard, bool, false); + +} // namespace mongo diff --git a/src/mongo/s/query/cluster_query_knobs.h b/src/mongo/s/query/cluster_query_knobs.h new file mode 100644 index 00000000000..76df18fde4d --- /dev/null +++ b/src/mongo/s/query/cluster_query_knobs.h @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include + +namespace mongo { + +// If set to true on mongos, all aggregations delivered to the mongos which require a merging shard +// will select the primary shard as the merger. False by default, which means that the merging shard +// will be selected randomly amongst the shards participating in the query. +extern std::atomic internalQueryAlwaysMergeOnPrimaryShard; // NOLINT + +} // namespace mongo diff --git a/src/mongo/s/query/router_stage_merge.cpp b/src/mongo/s/query/router_stage_merge.cpp index 527bc0f0063..b4be2ae5b78 100644 --- a/src/mongo/s/query/router_stage_merge.cpp +++ b/src/mongo/s/query/router_stage_merge.cpp @@ -57,6 +57,10 @@ StatusWith> RouterStageMerge::next() { void RouterStageMerge::kill() { auto killEvent = _arm.kill(); + if (!killEvent) { + // Mongos is shutting down. + return; + } _executor->waitForEvent(killEvent); } diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp index 37257afe244..89348e61407 100644 --- a/src/mongo/s/sharding_initialization.cpp +++ b/src/mongo/s/sharding_initialization.cpp @@ -66,6 +66,7 @@ using executor::ConnectionPool; MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolHostTimeoutMS, int, -1); MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMaxSize, int, -1); +MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMaxConnecting, int, -1); MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMinSize, int, static_cast(ConnectionPool::kDefaultMinConns)); @@ -181,6 +182,9 @@ Status initializeGlobalShardingState(OperationContext* txn, connPoolOptions.maxConnections = (ShardingTaskExecutorPoolMaxSize != -1) ? ShardingTaskExecutorPoolMaxSize : ConnectionPool::kDefaultMaxConns; + connPoolOptions.maxConnecting = (ShardingTaskExecutorPoolMaxConnecting != -1) + ? ShardingTaskExecutorPoolMaxConnecting + : ConnectionPool::kDefaultMaxConnecting; connPoolOptions.minConnections = ShardingTaskExecutorPoolMinSize; connPoolOptions.refreshRequirement = (ShardingTaskExecutorPoolRefreshRequirementMS != -1) ? Milliseconds(ShardingTaskExecutorPoolRefreshRequirementMS) diff --git a/src/mongo/scripting/SConscript b/src/mongo/scripting/SConscript index 3f1e034fa40..ef524aa852e 100644 --- a/src/mongo/scripting/SConscript +++ b/src/mongo/scripting/SConscript @@ -10,6 +10,7 @@ Import([ env.Library( target='scripting_common', source=[ + 'deadline_monitor.cpp', 'engine.cpp', 'utils.cpp', ], @@ -183,5 +184,6 @@ env.CppUnitTest( 'deadline_monitor_test.cpp', ], LIBDEPS=[ + 'scripting_common', ], ) diff --git a/src/mongo/scripting/deadline_monitor.cpp b/src/mongo/scripting/deadline_monitor.cpp new file mode 100644 index 00000000000..5eb0f52e5de --- /dev/null +++ b/src/mongo/scripting/deadline_monitor.cpp @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/scripting/deadline_monitor.h" + +#include "mongo/db/server_parameters.h" + +namespace mongo { + +MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 1000); + +int getScriptingEngineInterruptInterval() { + return scriptingEngineInterruptIntervalMS.load(); +} + +} // namespace mongo diff --git a/src/mongo/scripting/deadline_monitor.h b/src/mongo/scripting/deadline_monitor.h index 89f7e9d1b84..b1c2855dd46 100644 --- a/src/mongo/scripting/deadline_monitor.h +++ b/src/mongo/scripting/deadline_monitor.h @@ -30,6 +30,7 @@ #include #include "mongo/base/disallow_copying.h" +#include "mongo/platform/atomic_word.h" #include "mongo/platform/unordered_map.h" #include "mongo/stdx/condition_variable.h" #include "mongo/stdx/thread.h" @@ -38,6 +39,9 @@ namespace mongo { +// Returns the current interrupt interval from the setParameter value +int getScriptingEngineInterruptInterval(); + /** * DeadlineMonitor * @@ -91,11 +95,18 @@ public: * @param task the task to kill() * @param timeoutMs number of milliseconds before the deadline expires */ - void startDeadline(_Task* const task, uint64_t timeoutMs) { - const auto deadline = Date_t::now() + Milliseconds(timeoutMs); + void startDeadline(_Task* const task, int64_t timeoutMs) { + Date_t deadline; + if (timeoutMs > 0) { + deadline = Date_t::now() + Milliseconds(timeoutMs); + } else { + deadline = Date_t::max(); + } stdx::lock_guard lk(_deadlineMutex); - _tasks[task] = deadline; + if (_tasks.find(task) == _tasks.end()) { + _tasks.emplace(task, deadline); + } if (deadline < _nearestDeadlineWallclock) { _nearestDeadlineWallclock = deadline; @@ -121,14 +132,25 @@ private: */ void deadlineMonitorThread() { stdx::unique_lock lk(_deadlineMutex); + Date_t lastInterruptCycle = Date_t::now(); while (!_inShutdown) { // get the next interval to wait const Date_t now = Date_t::now(); + const auto interruptInterval = Milliseconds{getScriptingEngineInterruptInterval()}; + + if (now - lastInterruptCycle > interruptInterval) { + for (const auto& task : _tasks) { + if (task.second > now) + task.first->interrupt(); + } + lastInterruptCycle = now; + } // wait for a task to be added or a deadline to expire if (_nearestDeadlineWallclock > now) { - if (_nearestDeadlineWallclock == Date_t::max()) { - _newDeadlineAvailable.wait(lk); + if (_nearestDeadlineWallclock == Date_t::max() || + _nearestDeadlineWallclock - now > interruptInterval) { + _newDeadlineAvailable.wait_for(lk, interruptInterval); } else { _newDeadlineAvailable.wait_until(lk, _nearestDeadlineWallclock.toSystemTimePoint()); @@ -138,7 +160,7 @@ private: // set the next interval to wait for deadline completion _nearestDeadlineWallclock = Date_t::max(); - typename TaskDeadlineMap::iterator i = _tasks.begin(); + auto i = _tasks.begin(); while (i != _tasks.end()) { if (i->second < now) { // deadline expired @@ -155,7 +177,7 @@ private: } } - typedef unordered_map<_Task*, Date_t> TaskDeadlineMap; + using TaskDeadlineMap = std::unordered_map<_Task*, Date_t>; TaskDeadlineMap _tasks; // map of running tasks with deadlines stdx::mutex _deadlineMutex; // protects all non-const members, except _monitorThread stdx::condition_variable _newDeadlineAvailable; // Signaled for timeout, start and stop diff --git a/src/mongo/scripting/deadline_monitor_test.cpp b/src/mongo/scripting/deadline_monitor_test.cpp index 98f4008b234..71daefbce8f 100644 --- a/src/mongo/scripting/deadline_monitor_test.cpp +++ b/src/mongo/scripting/deadline_monitor_test.cpp @@ -72,6 +72,7 @@ public: if (_group) _group->noteKill(); } + void interrupt() {} TaskGroup* _group; uint64_t _killed; }; diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index 1bee97838f8..7e6628344ab 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -267,13 +267,11 @@ ScriptingFunction Scope::createFunction(const char* code) { FunctionCacheMap::iterator i = _cachedFunctions.find(code); if (i != _cachedFunctions.end()) return i->second; - // NB: we calculate the function number for v8 so the cache can be utilized to - // lookup the source on an exception, but SpiderMonkey uses the value - // returned by JS_CompileFunction. - ScriptingFunction defaultFunctionNumber = getFunctionCache().size() + 1; - ScriptingFunction actualFunctionNumber = _createFunction(code, defaultFunctionNumber); - _cachedFunctions[code] = actualFunctionNumber; - return actualFunctionNumber; + + // Get a function number, so the cache can be utilized to lookup the source on an exception + ScriptingFunction functionNumber = _createFunction(code); + _cachedFunctions[code] = functionNumber; + return functionNumber; } namespace JSFiles { @@ -507,12 +505,8 @@ public: } protected: - FunctionCacheMap& getFunctionCache() { - return _real->getFunctionCache(); - } - - ScriptingFunction _createFunction(const char* code, ScriptingFunction functionNumber = 0) { - return _real->_createFunction(code, functionNumber); + ScriptingFunction _createFunction(const char* code) { + return _real->_createFunction(code); } private: diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index f06333e9a45..d31311f0114 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -211,11 +211,7 @@ protected: */ class StoredFuncModLogOpHandler; - virtual FunctionCacheMap& getFunctionCache() { - return _cachedFunctions; - } - virtual ScriptingFunction _createFunction(const char* code, - ScriptingFunction functionNumber = 0) = 0; + virtual ScriptingFunction _createFunction(const char* code) = 0; std::string _localDBName; int64_t _loadedVersion; diff --git a/src/mongo/scripting/mozjs/implscope.cpp b/src/mongo/scripting/mozjs/implscope.cpp index c5bd51ef5cd..3b260c1558c 100644 --- a/src/mongo/scripting/mozjs/implscope.cpp +++ b/src/mongo/scripting/mozjs/implscope.cpp @@ -169,6 +169,10 @@ void MozJSImplScope::kill() { JS_RequestInterruptCallback(_runtime); } +void MozJSImplScope::interrupt() { + JS_RequestInterruptCallback(_runtime); +} + bool MozJSImplScope::isKillPending() const { return _pendingKill.load(); } @@ -483,11 +487,20 @@ BSONObj MozJSImplScope::getObject(const char* field) { void MozJSImplScope::newFunction(StringData raw, JS::MutableHandleValue out) { MozJSEntry entry(this); - std::string code = str::stream() << "____MongoToSM_newFunction_temp = " << raw; + _MozJSCreateFunction(raw, std::move(out)); +} + +void MozJSImplScope::_MozJSCreateFunction(StringData raw, JS::MutableHandleValue fun) { + std::string code = str::stream() + << "(" << parseJSFunctionOrExpression(_context, StringData(raw)) << ")"; JS::CompileOptions co(_context); setCompileOptions(&co); - _checkErrorState(JS::Evaluate(_context, _global, co, code.c_str(), code.length(), out)); + + _checkErrorState(JS::Evaluate(_context, _global, co, code.c_str(), code.length(), fun)); + uassert(10232, + "not a function", + fun.isObject() && JS_ObjectIsFunction(_context, fun.toObjectOrNull())); } BSONObj MozJSImplScope::callThreadArgs(const BSONObj& args) { @@ -528,39 +541,20 @@ bool hasFunctionIdentifier(StringData code) { return code[8] == ' ' || code[8] == '('; } -void MozJSImplScope::_MozJSCreateFunction(const char* raw, - ScriptingFunction functionNumber, - JS::MutableHandleValue fun) { - std::string code = str::stream() << "_funcs" << functionNumber << " = " - << parseJSFunctionOrExpression(_context, StringData(raw)); - - JS::CompileOptions co(_context); - setCompileOptions(&co); - - _checkErrorState(JS::Evaluate(_context, _global, co, code.c_str(), code.length(), fun)); - uassert(10232, - "not a function", - fun.isObject() && JS_ObjectIsFunction(_context, fun.toObjectOrNull())); -} - -ScriptingFunction MozJSImplScope::_createFunction(const char* raw, - ScriptingFunction functionNumber) { +ScriptingFunction MozJSImplScope::_createFunction(const char* raw) { MozJSEntry entry(this); JS::RootedValue fun(_context); - _MozJSCreateFunction(raw, functionNumber, &fun); + _MozJSCreateFunction(raw, &fun); _funcs.emplace_back(_context, fun.get()); - - return functionNumber; + return _funcs.size(); } void MozJSImplScope::setFunction(const char* field, const char* code) { MozJSEntry entry(this); JS::RootedValue fun(_context); - - _MozJSCreateFunction(code, getFunctionCache().size() + 1, &fun); - + _MozJSCreateFunction(code, &fun); ObjectWrapper(_context, _global).setValue(field, fun); } @@ -606,14 +600,16 @@ int MozJSImplScope::invoke(ScriptingFunction func, if (timeoutMs) _engine->getDeadlineMonitor().startDeadline(this, timeoutMs); + else { + _engine->getDeadlineMonitor().startDeadline(this, -1); + } JS::RootedValue out(_context); JS::RootedObject obj(_context, smrecv.toObjectOrNull()); bool success = JS::Call(_context, obj, funcValue, args, &out); - if (timeoutMs) - _engine->getDeadlineMonitor().stopDeadline(this); + _engine->getDeadlineMonitor().stopDeadline(this); _checkErrorState(success); @@ -651,15 +647,17 @@ bool MozJSImplScope::exec(StringData code, if (_checkErrorState(success, reportError, assertOnError)) return false; - if (timeoutMs) + if (timeoutMs) { _engine->getDeadlineMonitor().startDeadline(this, timeoutMs); + } else { + _engine->getDeadlineMonitor().startDeadline(this, -1); + } JS::RootedValue out(_context); success = JS_ExecuteScript(_context, _global, script, &out); - if (timeoutMs) - _engine->getDeadlineMonitor().stopDeadline(this); + _engine->getDeadlineMonitor().stopDeadline(this); if (_checkErrorState(success, reportError, assertOnError)) return false; @@ -706,6 +704,11 @@ void MozJSImplScope::localConnectForDbEval(OperationContext* txn, const char* db // NOTE: order is important here. the following methods must be called after // the above conditional statements. + _connectState = ConnectState::Local; + _localDBName = dbName; + + loadStored(txn); + // install db access functions in the global object installDBAccess(); @@ -713,16 +716,11 @@ void MozJSImplScope::localConnectForDbEval(OperationContext* txn, const char* db _mongoLocalProto.install(_global); execCoreFiles(); - const char* const makeMongo = "_mongo = new Mongo()"; + const char* const makeMongo = "const _mongo = new Mongo()"; exec(makeMongo, "local connect 2", false, true, true, 0); - std::string makeDB = str::stream() << "db = _mongo.getDB(\"" << dbName << "\");"; + std::string makeDB = str::stream() << "const db = _mongo.getDB(\"" << dbName << "\");"; exec(makeDB, "local connect 3", false, true, true, 0); - - _connectState = ConnectState::Local; - _localDBName = dbName; - - loadStored(txn); } void MozJSImplScope::externalSetup() { diff --git a/src/mongo/scripting/mozjs/implscope.h b/src/mongo/scripting/mozjs/implscope.h index 3fac5e5bdef..bfff70a93b0 100644 --- a/src/mongo/scripting/mozjs/implscope.h +++ b/src/mongo/scripting/mozjs/implscope.h @@ -89,6 +89,8 @@ public: void kill(); + void interrupt(); + bool isKillPending() const override; OperationContext* getOpContext() const; @@ -145,8 +147,7 @@ public: void injectNative(const char* field, NativeFunction func, void* data = 0) override; - ScriptingFunction _createFunction(const char* code, - ScriptingFunction functionNumber = 0) override; + ScriptingFunction _createFunction(const char* code) override; void newFunction(StringData code, JS::MutableHandleValue out); @@ -309,9 +310,7 @@ public: } private: - void _MozJSCreateFunction(const char* raw, - ScriptingFunction functionNumber, - JS::MutableHandleValue fun); + void _MozJSCreateFunction(StringData raw, JS::MutableHandleValue fun); /** * This structure exists exclusively to construct the runtime and context diff --git a/src/mongo/scripting/mozjs/mongo.cpp b/src/mongo/scripting/mozjs/mongo.cpp index 31164398961..39dc350d7d5 100644 --- a/src/mongo/scripting/mozjs/mongo.cpp +++ b/src/mongo/scripting/mozjs/mongo.cpp @@ -44,12 +44,14 @@ #include "mongo/scripting/mozjs/valuewriter.h" #include "mongo/scripting/mozjs/wrapconstrainedmethod.h" #include "mongo/stdx/memory.h" +#include "mongo/util/assert_util.h" namespace mongo { namespace mozjs { const JSFunctionSpec MongoBase::methods[] = { - MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(auth, MongoLocalInfo, MongoExternalInfo), + MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(auth, MongoExternalInfo), + MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(close, MongoExternalInfo), MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO( copyDatabaseWithSCRAM, MongoLocalInfo, MongoExternalInfo), MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(cursorFromId, MongoLocalInfo, MongoExternalInfo), @@ -89,8 +91,11 @@ const JSFunctionSpec MongoExternalInfo::freeFunctions[4] = { namespace { DBClientBase* getConnection(JS::CallArgs& args) { - return static_cast*>(JS_GetPrivate(args.thisv().toObjectOrNull())) - ->get(); + auto ret = static_cast*>( + JS_GetPrivate(args.thisv().toObjectOrNull()))->get(); + uassert( + ErrorCodes::BadValue, "Trying to get connection for closed Mongo object", ret != nullptr); + return ret; } void setCursor(JS::HandleObject target, @@ -120,6 +125,17 @@ void MongoBase::finalize(JSFreeOp* fop, JSObject* obj) { } } +void MongoBase::Functions::close::call(JSContext* cx, JS::CallArgs args) { + getConnection(args); + + auto thisv = args.thisv().toObjectOrNull(); + auto conn = static_cast*>(JS_GetPrivate(thisv)); + + conn->reset(); + + args.rval().setUndefined(); +} + void MongoBase::Functions::runCommand::call(JSContext* cx, JS::CallArgs args) { if (args.length() != 3) uasserted(ErrorCodes::BadValue, "runCommand needs 3 args"); diff --git a/src/mongo/scripting/mozjs/mongo.h b/src/mongo/scripting/mozjs/mongo.h index c3a82d10998..297b4907c6d 100644 --- a/src/mongo/scripting/mozjs/mongo.h +++ b/src/mongo/scripting/mozjs/mongo.h @@ -46,6 +46,7 @@ struct MongoBase : public BaseInfo { struct Functions { MONGO_DECLARE_JS_FUNCTION(auth); MONGO_DECLARE_JS_FUNCTION(copyDatabaseWithSCRAM); + MONGO_DECLARE_JS_FUNCTION(close); MONGO_DECLARE_JS_FUNCTION(cursorFromId); MONGO_DECLARE_JS_FUNCTION(cursorHandleFromId); MONGO_DECLARE_JS_FUNCTION(find); @@ -63,7 +64,7 @@ struct MongoBase : public BaseInfo { MONGO_DECLARE_JS_FUNCTION(getMaxWireVersion); }; - static const JSFunctionSpec methods[18]; + static const JSFunctionSpec methods[19]; static const char* const className; static const unsigned classFlags = JSCLASS_HAS_PRIVATE; diff --git a/src/mongo/scripting/mozjs/objectwrapper.cpp b/src/mongo/scripting/mozjs/objectwrapper.cpp index 1cecbc169e6..77ea9a1eca8 100644 --- a/src/mongo/scripting/mozjs/objectwrapper.cpp +++ b/src/mongo/scripting/mozjs/objectwrapper.cpp @@ -31,6 +31,7 @@ #include "mongo/scripting/mozjs/objectwrapper.h" #include +#include #include "mongo/base/error_codes.h" #include "mongo/bson/bsonobjbuilder.h" @@ -467,51 +468,60 @@ BSONObj ObjectWrapper::toBSON() { // emplace() inside ValueWriter. The runtime asserts enabled by MozJS's // debug mode will catch runtime errors, but be aware of how difficult this // is to get right and what to look for if one of them bites you. - WriteFieldRecursionFrames frames; - frames.emplace(_context, _object, nullptr, StringData{}); BSONObjBuilder b; - // We special case the _id field in top-level objects and move it to the front. - // This matches other drivers behavior and makes finding the _id field quicker in BSON. - if (hasOwnField(InternedString::_id)) { - _writeField(&b, InternedString::_id, &frames, frames.top().originalBSON); - } + { + // NOTE: Keep the frames in a scope so that it is clear that + // we always destroy them before we destroy 'b'. It is + // important to do so: if 'b' is destroyed before the frames, + // and we don't pop all of the frames (say, due to an + // exeption), then the frame dtors would write to freed + // memory. + WriteFieldRecursionFrames frames; + frames.emplace(_context, _object, nullptr, StringData{}); + + // We special case the _id field in top-level objects and move it to the front. + // This matches other drivers behavior and makes finding the _id field quicker in BSON. + if (hasOwnField(InternedString::_id)) { + _writeField(&b, InternedString::_id, &frames, frames.top().originalBSON); + } - while (frames.size()) { - auto& frame = frames.top(); + while (frames.size()) { + auto& frame = frames.top(); - // If the index is the same as length, we've seen all the keys at this - // level and should go up a level - if (frame.idx == frame.ids.length()) { - frames.pop(); - continue; - } + // If the index is the same as length, we've seen all the keys at this + // level and should go up a level + if (frame.idx == frame.ids.length()) { + frames.pop(); + continue; + } - if (frame.idx == 0 && frame.originalBSON && !frame.altered) { - // If this is our first look at the object and it has an unaltered - // bson behind it, move idx to the end so we'll roll up on the next - // pass through the loop. - frame.subbob_or(&b)->appendElements(*frame.originalBSON); - frame.idx = frame.ids.length(); - continue; - } + if (frame.idx == 0 && frame.originalBSON && !frame.altered) { + // If this is our first look at the object and it has an unaltered + // bson behind it, move idx to the end so we'll roll up on the next + // pass through the loop. + frame.subbob_or(&b)->appendElements(*frame.originalBSON); + frame.idx = frame.ids.length(); + continue; + } - id.set(frame.ids[frame.idx++]); + id.set(frame.ids[frame.idx++]); - if (frames.size() == 1) { - IdWrapper idw(_context, id); + if (frames.size() == 1) { + IdWrapper idw(_context, id); - // TODO: check if it's cheaper to just compare with an interned - // string of "_id" rather than with ascii - if (idw.isString() && idw.equalsAscii("_id")) { - continue; + // TODO: check if it's cheaper to just compare with an interned + // string of "_id" rather than with ascii + if (idw.isString() && idw.equalsAscii("_id")) { + continue; + } } - } - // writeField invokes ValueWriter with the frame stack, which will push - // onto frames for subobjects, which will effectively recurse the loop. - _writeField(frame.subbob_or(&b), JS::HandleId(id), &frames, frame.originalBSON); + // writeField invokes ValueWriter with the frame stack, which will push + // onto frames for subobjects, which will effectively recurse the loop. + _writeField(frame.subbob_or(&b), JS::HandleId(id), &frames, frame.originalBSON); + } } const int sizeWithEOO = b.len() + 1 /*EOO*/ - 4 /*BSONObj::Holder ref count*/; @@ -528,15 +538,42 @@ ObjectWrapper::WriteFieldRecursionFrame::WriteFieldRecursionFrame(JSContext* cx, JSObject* obj, BSONObjBuilder* parent, StringData sd) - : thisv(cx, obj), ids(cx, JS_Enumerate(cx, thisv)) { + : thisv(cx, obj), ids(cx) { + bool isArray = false; if (parent) { - subbob.emplace(JS_IsArrayObject(cx, thisv) ? parent->subarrayStart(sd) - : parent->subobjStart(sd)); + isArray = JS_IsArrayObject(cx, thisv); + + subbob.emplace(isArray ? parent->subarrayStart(sd) : parent->subobjStart(sd)); } - if (!ids) { - throwCurrentJSException( - cx, ErrorCodes::JSInterpreterFailure, "Failure to enumerate object"); + if (isArray) { + uint32_t length; + if (!JS_GetArrayLength(cx, thisv, &length)) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to get array length"); + } + + if (!ids.reserve(length)) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to reserve array"); + } + + JS::RootedId rid(cx); + for (uint32_t i = 0; i < length; i++) { + rid.set(INT_TO_JSID(i)); + ids.infallibleAppend(rid); + } + } else { + JS::AutoIdArray rids(cx, JS_Enumerate(cx, thisv)); + + if (!ids.reserve(rids.length())) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to reserve array"); + } + + for (uint32_t i = 0; i < rids.length(); i++) { + ids.infallibleAppend(rids[i]); + } } if (getScope(cx)->getProto().instanceOf(thisv)) { diff --git a/src/mongo/scripting/mozjs/objectwrapper.h b/src/mongo/scripting/mozjs/objectwrapper.h index 42be21ff36f..983c662efb3 100644 --- a/src/mongo/scripting/mozjs/objectwrapper.h +++ b/src/mongo/scripting/mozjs/objectwrapper.h @@ -198,7 +198,7 @@ private: JS::RootedObject thisv; // ids for the keys of thisv - JS::AutoIdArray ids; + JS::AutoIdVector ids; // Current index of the current key we're working on std::size_t idx = 0; diff --git a/src/mongo/scripting/mozjs/proxyscope.cpp b/src/mongo/scripting/mozjs/proxyscope.cpp index 5e48aabf9cf..bb4cd5c06ff 100644 --- a/src/mongo/scripting/mozjs/proxyscope.cpp +++ b/src/mongo/scripting/mozjs/proxyscope.cpp @@ -229,10 +229,9 @@ void MozJSProxyScope::injectNative(const char* field, NativeFunction func, void* run([&] { _implScope->injectNative(field, func, data); }); } -ScriptingFunction MozJSProxyScope::_createFunction(const char* raw, - ScriptingFunction functionNumber) { +ScriptingFunction MozJSProxyScope::_createFunction(const char* raw) { ScriptingFunction out; - run([&] { out = _implScope->_createFunction(raw, functionNumber); }); + run([&] { out = _implScope->_createFunction(raw); }); return out; } @@ -244,6 +243,10 @@ void MozJSProxyScope::kill() { _implScope->kill(); } +void MozJSProxyScope::interrupt() { + _implScope->interrupt(); +} + /** * Invokes a function on the implementation thread * diff --git a/src/mongo/scripting/mozjs/proxyscope.h b/src/mongo/scripting/mozjs/proxyscope.h index 09a84101dfd..451981330a1 100644 --- a/src/mongo/scripting/mozjs/proxyscope.h +++ b/src/mongo/scripting/mozjs/proxyscope.h @@ -165,8 +165,7 @@ public: void injectNative(const char* field, NativeFunction func, void* data = 0) override; - ScriptingFunction _createFunction(const char* code, - ScriptingFunction functionNumber = 0) override; + ScriptingFunction _createFunction(const char* code) override; OperationContext* getOpContext() const; @@ -175,6 +174,8 @@ public: */ void kill(); + void interrupt(); + private: template void run(Closure&& closure); diff --git a/src/mongo/scripting/mozjs/valuewriter.cpp b/src/mongo/scripting/mozjs/valuewriter.cpp index 0b02a945a41..dc4223050dc 100644 --- a/src/mongo/scripting/mozjs/valuewriter.cpp +++ b/src/mongo/scripting/mozjs/valuewriter.cpp @@ -293,6 +293,10 @@ void ValueWriter::_writeObject(BSONObjBuilder* b, } if (scope->getProto().getJSClass() == jsclass) { + uassert(ErrorCodes::BadValue, + "can't serialize DBPointer prototype", + scope->getProto().getProto() != obj); + JS::RootedValue id(_context); o.getValue("id", &id); diff --git a/src/mongo/scripting/utils.cpp b/src/mongo/scripting/utils.cpp index 10d8fcfe020..a9a9bf0a696 100644 --- a/src/mongo/scripting/utils.cpp +++ b/src/mongo/scripting/utils.cpp @@ -26,8 +26,12 @@ * then also delete it in the license file. */ +#include "mongo/platform/basic.h" + +#include "mongo/bson/json.h" #include "mongo/scripting/engine.h" #include "mongo/util/md5.hpp" +#include "mongo/util/time_support.h" namespace mongo { diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js index 0d1e225a990..21012a68a35 100644 --- a/src/mongo/shell/assert.js +++ b/src/mongo/shell/assert.js @@ -185,7 +185,7 @@ assert.soon = function(f, msg, timeout /*ms*/, interval) { } var start = new Date(); - timeout = timeout || 30000; + timeout = timeout || 5 * 60 * 1000; interval = interval || 200; var last; while (1) { @@ -452,16 +452,33 @@ assert.writeOK = function(res, msg) { }; assert.writeError = function(res, msg) { + return assert.writeErrorWithCode(res, null, msg); +}; + +assert.writeErrorWithCode = function(res, expectedCode, msg) { var errMsg = null; + var foundCode = null; if (res instanceof WriteResult) { - if (!res.hasWriteError() && !res.hasWriteConcernError()) { + if (res.hasWriteError()) { + foundCode = res.getWriteError().code; + } else if (res.hasWriteConcernError()) { + foundCode = res.getWriteConcernError().code; + } else { errMsg = "no write error: " + tojson(res); } } else if (res instanceof BulkWriteResult) { // Can only happen with bulk inserts - if (!res.hasWriteErrors() && !res.hasWriteConcernError()) { + if (res.hasWriteErrors()) { + if (res.getWriteErrorCount() > 1 && expectedCode != null) { + errMsg = "can't check for specific code when there was more than one write error"; + } else { + foundCode = res.getWriteErrorAt(0).code; + } + } else if (res.hasWriteConcernError()) { + foundCode = res.getWriteConcernError().code; + } else { errMsg = "no write errors: " + tojson(res); } } else if (res instanceof WriteCommandError) { @@ -473,6 +490,12 @@ assert.writeError = function(res, msg) { } } + if (!errMsg && expectedCode) { + if (foundCode != expectedCode) { + errMsg = "found code " + foundCode + " does not match expected code " + expectedCode; + } + } + if (errMsg) { if (msg) errMsg = errMsg + ": " + msg; diff --git a/src/mongo/shell/db.js b/src/mongo/shell/db.js index 9a58b6b114a..54070125f54 100644 --- a/src/mongo/shell/db.js +++ b/src/mongo/shell/db.js @@ -1202,7 +1202,7 @@ var DB; var _defaultWriteConcern = { w: 'majority', - wtimeout: 30 * 1000 + wtimeout: 60 * 1000 }; function getUserObjString(userObj) { diff --git a/src/mongo/shell/linenoise.cpp b/src/mongo/shell/linenoise.cpp index 641a4389e2d..8d32b9294bd 100644 --- a/src/mongo/shell/linenoise.cpp +++ b/src/mongo/shell/linenoise.cpp @@ -2765,7 +2765,17 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ int linenoiseHistorySave(const char* filename) { - FILE* fp = fopen(filename, "wt"); + FILE* fp; +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE || defined(__APPLE__) + int fd = open(filename, O_WRONLY | O_TRUNC | O_CREAT, S_IRUSR | S_IWUSR); + if (fd == -1) { + // report errno somehow? + return -1; + } + fp = fdopen(fd, "wt"); +#else + fp = fopen(filename, "wt"); +#endif // _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE || defined(__APPLE__) if (fp == NULL) { return -1; } @@ -2775,7 +2785,7 @@ int linenoiseHistorySave(const char* filename) { fprintf(fp, "%s\n", history[j]); } } - fclose(fp); + fclose(fp); // Also causes fd to be closed. return 0; } diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index fb0807909fb..4a64f5e50a0 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -89,6 +89,8 @@ var ReplSetTest = function(opts) { var _unbridgedPorts; var _unbridgedNodes; + this.kDefaultTimeoutMS = 5 * 60 * 1000; + // Publicly exposed variables /** @@ -161,7 +163,7 @@ var ReplSetTest = function(opts) { return; } - timeout = timeout || 30000; + timeout = timeout || self.kDefaultTimeoutMS; if (!node.getDB) { node = self.nodes[node]; @@ -179,7 +181,7 @@ var ReplSetTest = function(opts) { var currTime = new Date().getTime(); var status; - assert.soon(function() { + assert.soonNoExcept(function() { try { var conn = _callIsMaster(); if (!conn) { @@ -401,9 +403,9 @@ var ReplSetTest = function(opts) { * Blocks until the secondary nodes have completed recovery and their roles are known. */ this.awaitSecondaryNodes = function(timeout) { - timeout = timeout || 60000; + timeout = timeout || self.kDefaultTimeoutMS; - assert.soon(function() { + assert.soonNoExcept(function() { // Reload who the current slaves are self.getPrimary(timeout); @@ -443,45 +445,60 @@ var ReplSetTest = function(opts) { /** * Blocks until all nodes agree on who the primary is. + * If 'expectedPrimaryNodeId' is provided, ensure that every node is seeing this node as the + * primary. Otherwise, ensure that all the nodes in the set agree with the first node on the + * identity of the primary. */ - this.awaitNodesAgreeOnPrimary = function(timeout) { - timeout = timeout || 60000; + this.awaitNodesAgreeOnPrimary = function(timeout, nodes, expectedPrimaryNodeId) { + timeout = timeout || self.kDefaultTimeoutMS; + nodes = nodes || self.nodes; + expectedPrimaryNodeId = expectedPrimaryNodeId || -1; + if (expectedPrimaryNodeId === -1) { + print("AwaitNodesAgreeOnPrimary: Waiting for nodes to agree on any primary."); + } else { + print("AwaitNodesAgreeOnPrimary: Waiting for nodes to agree on " + + nodes[expectedPrimaryNodeId].name + " as primary."); + } - assert.soon(function() { - try { - var primary = -1; - - for (var i = 0; i < self.nodes.length; i++) { - var replSetGetStatus = - self.nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); - var nodesPrimary = -1; - for (var j = 0; j < replSetGetStatus.members.length; j++) { - if (replSetGetStatus.members[j].state === ReplSetTest.State.PRIMARY) { - // Node sees two primaries. - if (nodesPrimary !== -1) { - return false; - } - nodesPrimary = j; - } - } - // Node doesn't see a primary. - if (nodesPrimary < 0) { - return false; - } + assert.soonNoExcept(function() { + var primary = expectedPrimaryNodeId; - if (primary < 0) { - // If we haven't seen a primary yet, set it to this. - primary = nodesPrimary; - } else if (primary !== nodesPrimary) { - return false; + for (var i = 0; i < nodes.length; i++) { + var replSetGetStatus = nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); + var nodesPrimary = -1; + for (var j = 0; j < replSetGetStatus.members.length; j++) { + if (replSetGetStatus.members[j].state === ReplSetTest.State.PRIMARY) { + // Node sees two primaries. + if (nodesPrimary !== -1) { + print("AwaitNodesAgreeOnPrimary: Retrying because " + nodes[i].name + + " thinks both " + nodes[nodesPrimary].name + " and " + + nodes[j].name + " are primary."); + + return false; + } + nodesPrimary = j; } } + // Node doesn't see a primary. + if (nodesPrimary < 0) { + print("AwaitNodesAgreeOnPrimary: Retrying because " + nodes[i].name + + " does not see a primary."); + return false; + } - return true; - } catch (e) { - print("caught exception " + e); - return false; + if (primary < 0) { + // If we haven't seen a primary yet, set it to this. + primary = nodesPrimary; + } else if (primary !== nodesPrimary) { + print("AwaitNodesAgreeOnPrimary: Retrying because " + nodes[i].name + + " thinks the primary is " + nodes[nodesPrimary].name + " instead of " + + nodes[primary].name); + return false; + } } + + print("AwaitNodesAgreeOnPrimary: Nodes agreed on primary " + nodes[primary].name); + return true; }, "Awaiting nodes to agree on primary", timeout); }; @@ -490,10 +507,10 @@ var ReplSetTest = function(opts) { * if primary is available will return a connection to it. Otherwise throws an exception. */ this.getPrimary = function(timeout) { - timeout = timeout || 60000; + timeout = timeout || self.kDefaultTimeoutMS; var primary = null; - assert.soon(function() { + assert.soonNoExcept(function() { primary = _callIsMaster(); return primary; }, "Finding primary", timeout); @@ -503,9 +520,9 @@ var ReplSetTest = function(opts) { this.awaitNoPrimary = function(msg, timeout) { msg = msg || "Timed out waiting for there to be no primary in replset: " + this.name; - timeout = timeout || 30000; + timeout = timeout || self.kDefaultTimeoutMS; - assert.soon(function() { + assert.soonNoExcept(function() { return _callIsMaster() == false; }, msg, timeout); }; @@ -572,7 +589,7 @@ var ReplSetTest = function(opts) { var config = cfg || this.getReplSetConfig(); var cmd = {}; var cmdKey = initCmd || 'replSetInitiate'; - timeout = timeout || 120000; + timeout = timeout || self.kDefaultTimeoutMS; if (jsTestOptions().useLegacyReplicationProtocol && !config.hasOwnProperty("protocolVersion")) { config.protocolVersion = 0; @@ -590,6 +607,49 @@ var ReplSetTest = function(opts) { } }; + /** + * Steps up 'node' as primary. + * Waits for all nodes to reach the same optime before each election. + * Calls awaitReplication() which requires all connections in 'nodes' to be authenticated. + */ + this.stepUp = function(node) { + this.awaitReplication(); + this.awaitNodesAgreeOnPrimary(); + if (this.getPrimary() === node) { + print("Node " + node.host + " is already primary, no need to step it up."); + return; + } + print("Stepping up node " + node.host); + + // Ensure the specified node is primary. + for (var i = 0; i < this.nodes.length; i++) { + var primary = this.getPrimary(); + if (primary === node) { + break; + } + try { + // Make sure the nodes do not step back up for 10 minutes. + assert.commandWorked(primary.adminCommand({replSetStepDown: 10 * 60, force: true})); + } catch (ex) { + print("Caught exception while stepping down node '" + tojson(node.host) + "': " + + tojson(ex)); + } + this.awaitReplication(); + this.awaitNodesAgreeOnPrimary(); + } + + // Reset the rest of the nodes so they can run for election during the test. + for (var i = 0; i < this.nodes.length; i++) { + // Cannot call replSetFreeze on the primary. + if (this.nodes[i] === node) { + continue; + } + assert.commandWorked(this.nodes[i].adminCommand({replSetFreeze: 0})); + } + + assert.eq(this.getPrimary(), node, node.host + " was not primary after stepUp"); + }; + /** * Gets the current replica set config from the specified node index. If no nodeId is specified, * uses the primary node. @@ -625,6 +685,28 @@ var ReplSetTest = function(opts) { } }; + /** + * Blocks until all nodes in the replica set have the same config version as the primary. + **/ + this.awaitNodesAgreeOnConfigVersion = function(timeout) { + timeout = timeout || this.kDefaultTimeoutMS; + + assert.soonNoExcept(function() { + var primaryVersion = self.getPrimary().adminCommand({ismaster: 1}).setVersion; + + for (var i = 0; i < self.nodes.length; i++) { + var version = self.nodes[i].adminCommand({ismaster: 1}).setVersion; + assert.eq(version, + primaryVersion, + "waiting for secondary node " + self.nodes[i].host + + " with config version of " + version + + " to match the version of the primary " + primaryVersion); + } + + return true; + }, "Awaiting nodes to agree on config version", timeout); + }; + /** * Waits for the last oplog entry on the primary to be visible in the committed snapshop view * of the oplog on *all* secondaries. @@ -656,7 +738,7 @@ var ReplSetTest = function(opts) { print("Waiting for op with OpTime " + tojson(opTime) + " to be committed on all secondaries"); - assert.soon(function() { + assert.soonNoExcept(function() { for (var i = 0; i < rst.nodes.length; i++) { var node = rst.nodes[i]; @@ -690,14 +772,14 @@ var ReplSetTest = function(opts) { }; this.awaitReplication = function(timeout) { - timeout = timeout || 30000; + timeout = timeout || self.kDefaultTimeoutMS; var masterLatestOpTime; // Blocking call, which will wait for the last optime written on the master to be available var awaitLastOpTimeWrittenFn = function() { var master = self.getPrimary(); - assert.soon(function() { + assert.soonNoExcept(function() { try { masterLatestOpTime = _getLastOpTime(master); } catch (e) { @@ -706,7 +788,7 @@ var ReplSetTest = function(opts) { } return true; - }, "awaiting oplog query", 30000); + }, "awaiting oplog query", timeout); }; awaitLastOpTimeWrittenFn(); @@ -734,7 +816,7 @@ var ReplSetTest = function(opts) { ", is " + tojson(masterLatestOpTime) + ", last oplog entry is " + tojsononeline(masterOpTime)); - assert.soon(function() { + assert.soonNoExcept(function() { try { print("ReplSetTest awaitReplication: checking secondaries against timestamp " + tojson(masterLatestOpTime)); @@ -752,7 +834,7 @@ var ReplSetTest = function(opts) { ", but expected config version #" + configVersion); if (slaveConfigVersion > configVersion) { - master = this.getPrimary(); + master = self.getPrimary(); configVersion = master.getDB("local")['system.replset'].findOne().version; masterOpTime = _getLastOpTime(master); @@ -1169,7 +1251,7 @@ var ReplSetTest = function(opts) { */ this.waitForMaster = function(timeout) { var master; - assert.soon(function() { + assert.soonNoExcept(function() { return (master = self.getPrimary()); }, "waiting for master", timeout); @@ -1288,7 +1370,7 @@ ReplSetTest.awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { return; } - timeout = timeout || 60000; + timeout = timeout || 5 * 60 * 1000; if (hostOk == undefined) hostOk = { @@ -1303,7 +1385,7 @@ ReplSetTest.awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { var tests = 0; - assert.soon(function() { + assert.soonNoExcept(function() { var rsClientHosts = conn.adminCommand('connPoolStats').replicaSets; if (tests++ % 10 == 0) { printjson(rsClientHosts); diff --git a/src/mongo/shell/shell_utils_launcher.cpp b/src/mongo/shell/shell_utils_launcher.cpp index 81793143ec0..d01b2d9ec98 100644 --- a/src/mongo/shell/shell_utils_launcher.cpp +++ b/src/mongo/shell/shell_utils_launcher.cpp @@ -546,7 +546,10 @@ bool wait_for_pid(ProcessId pid, bool block = true, int* exit_code = NULL) { } #else int tmp; - bool ret = (pid.toNative() == waitpid(pid.toNative(), &tmp, (block ? 0 : WNOHANG))); + int ret; + do { + ret = waitpid(pid.toNative(), &tmp, (block ? 0 : WNOHANG)); + } while (ret == -1 && errno == EINTR); if (ret && exit_code) { if (WIFEXITED(tmp)) { *exit_code = WEXITSTATUS(tmp); @@ -556,7 +559,7 @@ bool wait_for_pid(ProcessId pid, bool block = true, int* exit_code = NULL) { MONGO_UNREACHABLE; } } - return ret; + return ret == pid.toNative(); #endif } diff --git a/src/mongo/shell/types.js b/src/mongo/shell/types.js index 066747176bf..622456d0988 100644 --- a/src/mongo/shell/types.js +++ b/src/mongo/shell/types.js @@ -251,7 +251,7 @@ Array.stdDev = function(arr) { Object.extend = function(dst, src, deep) { for (var k in src) { var v = src[k]; - if (deep && typeof(v) == "object") { + if (deep && typeof(v) == "object" && v !== null) { if (v.constructor === ObjectId) { // convert ObjectId properly eval("v = " + tojson(v)); } else if ("floatApprox" in v) { // convert NumberLong properly diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index 43925df86a4..2d86a1f6ac3 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -283,7 +283,15 @@ jsTest.authenticateNodes = function(nodes) { assert.soonNoExcept(function() { for (var i = 0; i < nodes.length; i++) { // Don't try to authenticate to arbiters - res = nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); + try { + res = nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); + } catch (e) { + // ReplicaSet tests which don't use auth are allowed to have nodes crash during + // startup. To allow tests which use to behavior to work with auth, + // attempting authentication against a dead node should be non-fatal. + print("Caught exception getting replSetStatus while authenticating: " + e); + continue; + } if (res.myState == 7) { continue; } diff --git a/src/mongo/stdx/thread.h b/src/mongo/stdx/thread.h index 9b1d2f5ae10..822dcaf6b87 100644 --- a/src/mongo/stdx/thread.h +++ b/src/mongo/stdx/thread.h @@ -29,6 +29,8 @@ #pragma once #include +#include +#include #include #include #include @@ -109,7 +111,38 @@ inline void swap(thread& lhs, thread& rhs) BOOST_NOEXCEPT { lhs.swap(rhs); } -namespace this_thread = ::std::this_thread; // NOLINT +namespace this_thread { +using std::this_thread::get_id; // NOLINT +using std::this_thread::yield; // NOLINT + +#ifdef _WIN32 +using std::this_thread::sleep_for; // NOLINT +using std::this_thread::sleep_until; // NOLINT +#else +template +inline void sleep_for(const std::chrono::duration& sleep_duration) { // NOLINT + if (sleep_duration <= sleep_duration.zero()) + return; + + const auto seconds = + std::chrono::duration_cast(sleep_duration); // NOLINT + const auto nanoseconds = + std::chrono::duration_cast(sleep_duration - seconds); // NOLINT + struct timespec sleepVal = {static_cast(seconds.count()), // NOLINT + static_cast(nanoseconds.count())}; + struct timespec remainVal; + while (nanosleep(&sleepVal, &remainVal) == -1 && errno == EINTR) { + sleepVal = remainVal; + } +} + +template +void sleep_until(const std::chrono::time_point& sleep_time) { // NOLINT + const auto now = Clock::now(); + sleep_for(sleep_time - now); +} +#endif +} // namespace this_thread } // namespace stdx } // namespace mongo diff --git a/src/mongo/stdx/type_traits.h b/src/mongo/stdx/type_traits.h new file mode 100644 index 00000000000..9fc95279b98 --- /dev/null +++ b/src/mongo/stdx/type_traits.h @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/config.h" + +#if defined(MONGO_CONFIG_HAVE_STD_ENABLE_IF_T) + +namespace mongo { +namespace stdx { + +using ::std::enable_if_t; + +} // namespace stdx +} // namespace mongo + +#else + +namespace mongo { +namespace stdx { + +template +using enable_if_t = typename std::enable_if::type; + +} // namespace stdx +} // namespace mongo + +#endif diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index 17335efc207..9699c0c5555 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -446,3 +446,44 @@ env.CppUnitTest( '$BUILD_DIR/mongo/base', ] ) + +if env.TargetOSIs('linux'): + env.Library( + target='procparser', + source=[ + "procparser.cpp", + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + ], + ) + + env.CppUnitTest( + target='procparser_test', + source=[ + 'procparser_test.cpp', + ], + LIBDEPS=[ + 'procparser', + ]) + +if env.TargetOSIs('windows'): + env.Library( + target='perfctr_collect', + source=[ + "perfctr_collect.cpp", + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/platform/platform', + ], + ) + + env.CppUnitTest( + target='perfctr_collect_test', + source=[ + 'perfctr_collect_test.cpp', + ], + LIBDEPS=[ + 'perfctr_collect', + ]) diff --git a/src/mongo/util/exit_code.h b/src/mongo/util/exit_code.h index 209e27a27a6..004fb579088 100644 --- a/src/mongo/util/exit_code.h +++ b/src/mongo/util/exit_code.h @@ -52,6 +52,7 @@ enum ExitCode : int { EXIT_WINDOWS_SERVICE_STOP = 49, EXIT_POSSIBLE_CORRUPTION = 60, // this means we detected a possible corruption situation, like a buf overflow + EXIT_WATCHDOG = 61, // Internal Storage Node Watchdog has terminated mongod EXIT_UNCAUGHT = 100, // top level exception that wasn't caught EXIT_TEST = 101 }; diff --git a/src/mongo/util/log.h b/src/mongo/util/log.h index b0f53a30b87..3f17151d4f4 100644 --- a/src/mongo/util/log.h +++ b/src/mongo/util/log.h @@ -132,6 +132,22 @@ inline LogstreamBuilder log() { ::MongoLogDefaultComponent_component); } +/** + * Returns a LogstreamBuilder that does not cache its ostream in a threadlocal cache. + * Use this variant when logging from places that may not be able to access threadlocals, + * such as from within other threadlocal-managed objects, or thread_specific_ptr-managed + * objects. + * + * Once SERVER-29377 is completed, this overload can be removed. + */ +inline LogstreamBuilder logNoCache() { + return LogstreamBuilder(logger::globalLogDomain(), + getThreadName(), + logger::LogSeverity::Log(), + ::MongoLogDefaultComponent_component, + false); +} + inline LogstreamBuilder log(logger::LogComponent component) { return LogstreamBuilder( logger::globalLogDomain(), getThreadName(), logger::LogSeverity::Log(), component); diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript index 72d3384fb33..3a72a9b278c 100644 --- a/src/mongo/util/net/SConscript +++ b/src/mongo/util/net/SConscript @@ -38,6 +38,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/server_options_core', + '$BUILD_DIR/mongo/db/server_parameters', '$BUILD_DIR/mongo/util/background_job', '$BUILD_DIR/mongo/util/concurrency/ticketholder', '$BUILD_DIR/mongo/util/fail_point', diff --git a/src/mongo/util/net/message_port.cpp b/src/mongo/util/net/message_port.cpp index c916c2cfa50..1033ba112dd 100644 --- a/src/mongo/util/net/message_port.cpp +++ b/src/mongo/util/net/message_port.cpp @@ -173,9 +173,22 @@ bool MessagingPort::recv(Message& m) { psock->setHandshakeReceived(); goto again; } + + auto sslMode = sslGlobalParams.sslMode.load(); + uassert(17189, "The server is configured to only allow SSL connections", - sslGlobalParams.sslMode.load() != SSLParams::SSLMode_requireSSL); + sslMode != SSLParams::SSLMode_requireSSL); + + // For users attempting to upgrade their applications from no SSL to SSL, provide + // information about connections that still aren't using SSL (but only once per + // connection) + if (!sslGlobalParams.disableNonSSLConnectionLogging && + (sslMode == SSLParams::SSLMode_preferSSL)) { + LOG(0) << "SSL mode is set to 'preferred' and connection " << connectionId() + << " to " << remote() << " is not using SSL."; + } + #endif // MONGO_CONFIG_SSL } if (static_cast(len) < sizeof(MSGHEADER::Value) || diff --git a/src/mongo/util/net/sock.cpp b/src/mongo/util/net/sock.cpp index 4c34ced371a..4ecde553370 100644 --- a/src/mongo/util/net/sock.cpp +++ b/src/mongo/util/net/sock.cpp @@ -33,7 +33,10 @@ #include "mongo/util/net/sock.h" +#include + #if !defined(_WIN32) +#include #include #include #include @@ -71,6 +74,38 @@ using std::vector; MONGO_FP_DECLARE(throwSockExcep); +namespace { + +// Provides a cross-platform function for setting a file descriptor/socket to non-blocking mode. +bool setBlock(int fd, bool block) { +#ifdef _WIN32 + u_long ioMode = block ? 0 : 1; + return (NO_ERROR == ::ioctlsocket(fd, FIONBIO, &ioMode)); +#else + int flags = fcntl(fd, F_GETFL, fd); + if (block) { + return (-1 != fcntl(fd, F_SETFL, (flags & ~O_NONBLOCK))); + } else { + return (-1 != fcntl(fd, F_SETFL, (flags | O_NONBLOCK))); + } +#endif +} + +void networkWarnWithDescription(const Socket& socket, StringData call, int errorCode = -1) { +#ifdef _WIN32 + if (errorCode == -1) { + errorCode = WSAGetLastError(); + } +#endif + auto ewd = errnoWithDescription(errorCode); + warning() << "Failed to connect to " << socket.remoteAddr().getAddr() << ":" + << socket.remoteAddr().getPort() << ", in(" << call << "), reason: " << ewd; +} + +const double kMaxConnectTimeoutMS = 5000; + +} // namespace + static bool ipv6 = false; void enableIPv6(bool state) { ipv6 = state; @@ -538,53 +573,91 @@ std::string Socket::doSSLHandshake(const char* firstBytes, int len) { } #endif -class ConnectBG : public BackgroundJob { -public: - ConnectBG(int sock, SockAddr remote) : _sock(sock), _remote(remote) {} +bool Socket::connect(SockAddr& remote) { + _remote = remote; - void run() { -#if defined(_WIN32) - if ((_res = _connect()) == SOCKET_ERROR) { - _errnoWithDescription = errnoWithDescription(); + _fd = ::socket(remote.getType(), SOCK_STREAM, 0); + if (_fd == INVALID_SOCKET) { + networkWarnWithDescription(*this, "socket"); + return false; + } + + if (!setBlock(_fd, false)) { + networkWarnWithDescription(*this, "set socket to non-blocking mode"); + return false; + } + + const Milliseconds connectTimeoutMillis(static_cast( + _timeout > 0 ? std::min(kMaxConnectTimeoutMS, (_timeout * 1000)) : kMaxConnectTimeoutMS)); + const Date_t expiration = Date_t::now() + connectTimeoutMillis; + + bool connectSucceeded = ::connect(_fd, _remote.raw(), _remote.addressSize) == 0; + + if (!connectSucceeded) { +#ifdef _WIN32 + if (WSAGetLastError() != WSAEWOULDBLOCK) { + networkWarnWithDescription(*this, "connect"); + return false; } #else - while ((_res = _connect()) == -1) { - const int error = errno; - if (error != EINTR) { - _errnoWithDescription = errnoWithDescription(error); - break; - } + if (errno != EINTR && errno != EINPROGRESS) { + networkWarnWithDescription(*this, "connect"); + return false; } #endif - } - std::string name() const { - return "ConnectBG"; - } - std::string getErrnoWithDescription() const { - return _errnoWithDescription; - } - int inError() const { - return _res; - } + pollfd pfd; + pfd.fd = _fd; + pfd.events = POLLOUT; -private: - int _connect() const { - return ::connect(_sock, _remote.raw(), _remote.addressSize); - } + while (true) { + const auto timeout = std::max(Milliseconds(0), expiration - Date_t::now()); - int _sock; - int _res; - SockAddr _remote; - std::string _errnoWithDescription; -}; + int pollReturn = socketPoll(&pfd, 1, timeout.count()); +#ifdef _WIN32 + if (pollReturn == SOCKET_ERROR) { + networkWarnWithDescription(*this, "poll"); + return false; + } +#else + if (pollReturn == -1) { + if (errno != EINTR) { + networkWarnWithDescription(*this, "poll"); + return false; + } -bool Socket::connect(SockAddr& remote) { - _remote = remote; + // EINTR in poll, try again + continue; + } +#endif + // No activity for the full duration of the timeout. + if (pollReturn == 0) { + warning() << "Failed to connect to " << _remote.getAddr() << ":" + << _remote.getPort() << " after " << connectTimeoutMillis + << " milliseconds, giving up."; + return false; + } - _fd = socket(remote.getType(), SOCK_STREAM, 0); - if (_fd == INVALID_SOCKET) { - LOG(_logLevel) << "ERROR: connect invalid socket " << errnoWithDescription() << endl; + // We had a result, see if there's an error on the socket. + int optVal; + socklen_t optLen = sizeof(optVal); + if (::getsockopt( + _fd, SOL_SOCKET, SO_ERROR, reinterpret_cast(&optVal), &optLen) == -1) { + networkWarnWithDescription(*this, "getsockopt"); + return false; + } + if (optVal != 0) { + networkWarnWithDescription(*this, "checking socket for error after poll", optVal); + return false; + } + + // We had activity and we don't have errors on the socket, we're connected. + break; + } + } + + if (!setBlock(_fd, true)) { + networkWarnWithDescription(*this, "could not set socket to blocking mode"); return false; } @@ -592,25 +665,6 @@ bool Socket::connect(SockAddr& remote) { setTimeout(_timeout); } - static const unsigned int connectTimeoutMillis = 5000; - ConnectBG bg(_fd, remote); - bg.go(); - if (bg.wait(connectTimeoutMillis)) { - if (bg.inError()) { - warning() << "Failed to connect to " << _remote.getAddr() << ":" << _remote.getPort() - << ", reason: " << bg.getErrnoWithDescription() << endl; - close(); - return false; - } - } else { - // time out the connect - close(); - bg.wait(); // so bg stays in scope until bg thread terminates - warning() << "Failed to connect to " << _remote.getAddr() << ":" << _remote.getPort() - << " after " << connectTimeoutMillis << " milliseconds, giving up." << endl; - return false; - } - if (remote.getType() != AF_UNIX) disableNagle(_fd); @@ -637,9 +691,7 @@ int Socket::_send(const char* data, int len, const char* context) { } #endif int ret = ::send(_fd, data, len, portSendFlags); - if (ret < 0) { - handleSendError(ret, context); - } + return ret; } @@ -653,11 +705,15 @@ void Socket::send(const char* data, int len, const char* context) { #else errno = ENETUNREACH; #endif - handleSendError(ret, context); } else { ret = _send(data, len, context); } + if (ret < 0) { + handleSendError(ret, context); + continue; + } + _bytesOut += ret; fassert(16507, ret <= len); @@ -717,15 +773,7 @@ void Socket::send(const vector>& data, const char* context) { } if (ret == -1) { - if (errno != EAGAIN || _timeout == 0) { - LOG(_logLevel) << "Socket " << context << " send() " << errnoWithDescription() - << ' ' << remoteString() << endl; - throw SocketException(SocketException::SEND_ERROR, remoteString()); - } else { - LOG(_logLevel) << "Socket " << context << " send() remote timeout " - << remoteString() << endl; - throw SocketException(SocketException::SEND_TIMEOUT, remoteString()); - } + handleSendError(ret, context); } else { struct iovec*& i = meta.msg_iov; while (ret > 0) { @@ -798,7 +846,7 @@ void Socket::handleSendError(int ret, const char* context) { #endif LOG(_logLevel) << "Socket " << context << " send() timed out " << remoteString() << endl; throw SocketException(SocketException::SEND_TIMEOUT, remoteString()); - } else { + } else if (mongo_errno != EINTR) { LOG(_logLevel) << "Socket " << context << " send() " << errnoWithDescription(mongo_errno) << ' ' << remoteString() << endl; throw SocketException(SocketException::SEND_ERROR, remoteString()); @@ -818,7 +866,6 @@ void Socket::handleRecvError(int ret, int len) { int e = errno; #if defined(EINTR) if (e == EINTR) { - LOG(_logLevel) << "EINTR returned from recv(), retrying"; return; } #endif diff --git a/src/mongo/util/net/sock.h b/src/mongo/util/net/sock.h index 74a36c38c09..399cbc09e9f 100644 --- a/src/mongo/util/net/sock.h +++ b/src/mongo/util/net/sock.h @@ -348,7 +348,7 @@ public: } void handleRecvError(int ret, int len); - MONGO_COMPILER_NORETURN void handleSendError(int ret, const char* context); + void handleSendError(int ret, const char* context); private: void _init(); diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 7057bf3c004..9dd51e46fb9 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -43,6 +43,7 @@ #include "mongo/base/init.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/config.h" +#include "mongo/db/server_parameters.h" #include "mongo/platform/atomic_word.h" #include "mongo/stdx/memory.h" #include "mongo/util/concurrency/mutex.h" @@ -77,6 +78,41 @@ const SSLParams& getSSLGlobalParams() { return sslGlobalParams; } +/** + * Configurable via --setParameter disableNonSSLConnectionLogging=true. If false (default) + * if the sslMode is set to preferSSL, we will log connections that are not using SSL. + * If true, such log messages will be suppressed. + */ +ExportedServerParameter + disableNonSSLConnectionLoggingParameter(ServerParameterSet::getGlobal(), + "disableNonSSLConnectionLogging", + &sslGlobalParams.disableNonSSLConnectionLogging); + +class OpenSSLCipherConfigParameter + : public ExportedServerParameter { +public: + OpenSSLCipherConfigParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "opensslCipherConfig", + &sslGlobalParams.sslCipherConfig) {} + Status validate(const std::string& potentialNewValue) final { + if (!sslGlobalParams.sslCipherConfig.empty()) { + return Status( + ErrorCodes::BadValue, + "opensslCipherConfig setParameter is incompatible with net.ssl.sslCipherConfig"); + } + // Note that there is very little validation that we can do here. + // OpenSSL exposes no API to validate a cipher config string. The only way to figure out + // what a string maps to is to make an SSL_CTX object, set the string on it, then parse the + // resulting STACK_OF object. If provided an invalid entry in the string, it will silently + // ignore it. Because an entry in the string may map to multiple ciphers, or remove ciphers + // from the final set produced by the full string, we can't tell if any entry failed + // to parse. + return Status::OK(); + } +} openSSLCipherConfig; + #ifdef MONGO_CONFIG_SSL // Old copies of OpenSSL will not have constants to disable protocols they don't support. // Define them to values we can OR together safely to generically disable these protocols across @@ -709,7 +745,7 @@ unsigned long long SSLManager::_convertASN1ToMillis(ASN1_TIME* asn1time) { bool SSLManager::_parseAndValidateCertificate(const std::string& keyFile, std::string* subjectName, Date_t* serverCertificateExpirationDate) { - BIO* inBIO = BIO_new(BIO_s_file_internal()); + BIO* inBIO = BIO_new(BIO_s_file()); if (inBIO == NULL) { error() << "failed to allocate BIO object: " << getSSLErrorMessage(ERR_get_error()); return false; diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp index 2d29e4704f2..a29b2d1805c 100644 --- a/src/mongo/util/net/ssl_options.cpp +++ b/src/mongo/util/net/ssl_options.cpp @@ -257,6 +257,13 @@ Status storeSSLServerOptions(const moe::Environment& params) { } if (params.count("net.ssl.sslCipherConfig")) { + warning() + << "net.ssl.sslCipherConfig is deprecated. It will be removed in a future release."; + if (!sslGlobalParams.sslCipherConfig.empty()) { + return Status(ErrorCodes::BadValue, + "net.ssl.sslCipherConfig is incompatible with the openSSLCipherConfig " + "setParameter"); + } sslGlobalParams.sslCipherConfig = params["net.ssl.sslCipherConfig"].as(); } diff --git a/src/mongo/util/net/ssl_options.h b/src/mongo/util/net/ssl_options.h index e897656ed25..f148c400b9b 100644 --- a/src/mongo/util/net/ssl_options.h +++ b/src/mongo/util/net/ssl_options.h @@ -58,6 +58,8 @@ struct SSLParams { bool sslFIPSMode; // --sslFIPSMode bool sslAllowInvalidCertificates; // --sslAllowInvalidCertificates bool sslAllowInvalidHostnames; // --sslAllowInvalidHostnames + bool disableNonSSLConnectionLogging = + false; // --setParameter disableNonSSLConnectionLogging=true SSLParams() { sslMode.store(SSLMode_disabled); diff --git a/src/mongo/util/perfctr_collect.cpp b/src/mongo/util/perfctr_collect.cpp new file mode 100644 index 00000000000..89ac6694a60 --- /dev/null +++ b/src/mongo/util/perfctr_collect.cpp @@ -0,0 +1,505 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/perfctr_collect.h" + +#include "mongo/base/init.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/scopeguard.h" +#include "mongo/util/text.h" + +namespace mongo { + +namespace { + +// Handle to the PDH library so that we can format error messages. +HANDLE hPdhLibrary = nullptr; + +/** + * Load PDH.DLL for good error messages. + */ +MONGO_INITIALIZER(PdhInit)(InitializerContext* context) { + hPdhLibrary = LoadLibraryW(L"pdh.dll"); + if (nullptr == hPdhLibrary) { + DWORD gle = GetLastError(); + return {ErrorCodes::WindowsPdhError, + str::stream() << "LoadLibrary of pdh.dll failed with " + << errnoWithDescription(gle)}; + } + + return Status::OK(); +} + +/** + * Output an error message for ether PDH or the system. + */ +std::string errnoWithPdhDescription(PDH_STATUS status) { + LPWSTR errorText = nullptr; + + if (!FormatMessageW( + FORMAT_MESSAGE_FROM_HMODULE | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_ALLOCATE_BUFFER, + hPdhLibrary, + status, + 0, + reinterpret_cast( + &errorText), // fudge the type so FormatMessageW uses it as an out parameter. + 0, + nullptr)) { + DWORD gle = GetLastError(); + return str::stream() << "Format message failed with " << gle << " for status " << status; + } + + ScopeGuard errorTextGuard = MakeGuard([errorText]() { LocalFree(errorText); }); + std::string utf8ErrorText = toUtf8String(errorText); + + auto size = utf8ErrorText.find_first_of("\r\n"); + if (size == std::string::npos) { + size = utf8ErrorText.length(); + } + + return utf8ErrorText.substr(0, size); +} + +/** + * Format an error message for a PDH function call failure. + */ +std::string formatFunctionCallError(StringData functionName, PDH_STATUS status) { + return str::stream() << functionName << " failed with '" << errnoWithPdhDescription(status) + << "'"; +} + +/** + * Transform a vector of string data into a vector of strings. + */ +void transformStringDataVector(const std::vector& input, + std::vector* output) { + output->reserve(input.size()); + for (const auto& str : input) { + output->emplace_back(str.toString()); + } +} + +/** + * Check if a counter depends on system ticks per second to compute its value from raw values. This + * is basically any counter that does not use 100NS as a base. FYI, if we query raw count counters, + * we will get the system ticks as a time base. + */ +bool counterHasTickBasedTimeBase(uint32_t type) { + return ((type & PERF_TYPE_COUNTER) == PERF_TYPE_COUNTER) && + ((type & PERF_TIMER_100NS) != PERF_TIMER_100NS); +} + +} // namespace + +StatusWith> PerfCounterCollection::checkCounters( + StringData name, const std::vector& paths) { + if (_counters.find(name.toString()) != _counters.end() || + _nestedCounters.find(name.toString()) != _nestedCounters.end()) { + return Status(ErrorCodes::BadValue, str::stream() << "Duplicate group name for " << name); + } + + std::vector stringPaths; + transformStringDataVector(paths, &stringPaths); + + // While duplicate counter paths are not a problem for PDH, they are a waste of time. + std::sort(stringPaths.begin(), stringPaths.end()); + + if (std::unique(stringPaths.begin(), stringPaths.end()) != stringPaths.end()) { + return Status(ErrorCodes::BadValue, + str::stream() << "Duplicate counters in paths specified"); + } + + return {stringPaths}; +} + +Status PerfCounterCollection::addCountersGroup(StringData name, + const std::vector& paths) { + auto swCounters = checkCounters(name, paths); + if (!swCounters.getStatus().isOK()) { + return swCounters.getStatus(); + } + + _counters.emplace(name.toString(), std::move(swCounters.getValue())); + + return Status::OK(); +} + +Status PerfCounterCollection::addCountersGroupedByInstanceName( + StringData name, const std::vector& paths) { + auto swCounters = checkCounters(name, paths); + if (!swCounters.getStatus().isOK()) { + return swCounters.getStatus(); + } + + _nestedCounters.emplace(name.toString(), std::move(swCounters.getValue())); + + return Status::OK(); +} + +StatusWith> PerfCounterCollector::create( + PerfCounterCollection builder) { + auto pcc = std::unique_ptr(new PerfCounterCollector()); + + Status s = pcc->open(); + if (!s.isOK()) { + return s; + } + + for (const auto& kvp : builder._counters) { + s = pcc->addCountersGroup(kvp.first, kvp.second); + if (!s.isOK()) { + return s; + } + } + + // Sort to enforce predictable output in final document + std::sort(pcc->_counters.begin(), + pcc->_counters.end(), + [](const CounterGroup& a, const CounterGroup& b) { return a.name < b.name; }); + + for (const auto& kvp : builder._nestedCounters) { + s = pcc->addCountersGroupedByInstanceName(kvp.first, kvp.second); + if (!s.isOK()) { + return s; + } + } + + std::sort( + pcc->_nestedCounters.begin(), + pcc->_nestedCounters.end(), + [](const NestedCounterGroup& a, const NestedCounterGroup& b) { return a.name < b.name; }); + + pcc->checkForTicksTimeBase(); + + return {std::move(pcc)}; +} + +PerfCounterCollector::~PerfCounterCollector() { + /*ignore*/ PdhCloseQuery(_query); +} + +Status PerfCounterCollector::open() { + PDH_STATUS status = PdhOpenQueryW(nullptr, NULL, &_query); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhOpenQueryW", status)}; + } + + return Status::OK(); +} + +StatusWith PerfCounterCollector::addCounter(StringData path) { + PDH_HCOUNTER counter{0}; + + PDH_STATUS status = + PdhAddCounterW(_query, toNativeString(path.toString().c_str()).c_str(), NULL, &counter); + + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhAddCounterW", status)}; + } + + DWORD bufferSize = 0; + + status = PdhGetCounterInfoW(counter, false, &bufferSize, nullptr); + + if (status != PDH_MORE_DATA) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhGetCounterInfoW", status)}; + } + + auto buf = stdx::make_unique(bufferSize); + auto counterInfo = reinterpret_cast(buf.get()); + + status = PdhGetCounterInfoW(counter, false, &bufferSize, counterInfo); + + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhGetCounterInfoW", status)}; + } + + // A full qualified path is as such: + // "\\MYMACHINE\\Processor(0)\\% Idle Time" + // MachineName \\ Object Name (Instance Name) \\ CounterName + // Ex: + // MachineName: MYMACHINE + // Object Name: Processor + // InstanceName: 0 + // CounterName: % Idle Time + // We do not want to use Machine Name, but sometimes we want InstanceName + // + std::string firstName = str::stream() << '\\' << toUtf8String(counterInfo->szObjectName) << '\\' + << toUtf8String(counterInfo->szCounterName); + + // Compute a second name + std::string secondName(firstName); + + bool hasSecondValue = false; + + // Only include base for counters that need it + if ((counterInfo->dwType & PERF_COUNTER_PRECISION) == PERF_COUNTER_PRECISION) { + secondName += " Base"; + hasSecondValue = true; + } + + // InstanceName is null for counters without instance names + return {CounterInfo{std::move(firstName), + std::move(secondName), + hasSecondValue, + counterInfo->szInstanceName ? toUtf8String(counterInfo->szInstanceName) + : std::string(), + counterInfo->dwType, + counter}}; +} + +StatusWith> PerfCounterCollector::addCounters( + StringData path) { + std::wstring pathWide = toNativeString(path.toString().c_str()); + DWORD pathListLength = 0; + PDH_STATUS status = PdhExpandCounterPathW(pathWide.c_str(), nullptr, &pathListLength); + + if (status != PDH_MORE_DATA) { + return {ErrorCodes::WindowsPdhError, + str::stream() << formatFunctionCallError("PdhExpandCounterPathW", status) + << " for counter '" << path << "'"}; + } + + auto buf = stdx::make_unique(pathListLength); + + status = PdhExpandCounterPathW(pathWide.c_str(), buf.get(), &pathListLength); + + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhExpandCounterPathW", status)}; + } + + std::vector counters; + + // Windows' PdhExpandWildCardPathW returns a nullptr terminated array of nullptr separated + // strings. + std::vector counterNames; + + const wchar_t* ptr = buf.get(); + while (ptr && *ptr) { + counterNames.emplace_back(toUtf8String(ptr)); + ptr += wcslen(ptr) + 1; + } + + // Sort to ensure we have a predictable ordering in the final BSON + std::sort(counterNames.begin(), counterNames.end()); + + for (const auto& name : counterNames) { + auto swCounterInfo = addCounter(name); + if (!swCounterInfo.isOK()) { + return swCounterInfo.getStatus(); + } + + counters.emplace_back(std::move(swCounterInfo.getValue())); + } + + return {std::move(counters)}; +} + +Status PerfCounterCollector::addCountersGroup(StringData groupName, + const std::vector& paths) { + CounterGroup group; + group.name = groupName.toString(); + + for (const auto& path : paths) { + auto swCounters = addCounters(path.c_str()); + if (!swCounters.isOK()) { + return swCounters.getStatus(); + } + + auto newCounters = swCounters.getValue(); + + std::copy(newCounters.begin(), newCounters.end(), std::back_inserter(group.counters)); + } + + _counters.emplace_back(group); + + return Status::OK(); +} + +Status PerfCounterCollector::addCountersGroupedByInstanceName( + StringData groupName, const std::vector& paths) { + NestedCounterGroup group; + group.name = groupName.toString(); + + for (const auto& path : paths) { + auto swCounters = addCounters(path.c_str()); + if (!swCounters.isOK()) { + return swCounters.getStatus(); + } + + auto newCounters = swCounters.getValue(); + + for (const auto& counter : newCounters) { + // Verify the counter has an instance name. + if (counter.instanceName.empty()) { + return {ErrorCodes::BadValue, + str::stream() << "Counter '" << counter.firstName + << "' must be an instance specific counter."}; + } + + // Skip counters in the _Total instance category. + if (counter.instanceName == "_Total") { + continue; + } + + group.counters[counter.instanceName].emplace_back(std::move(counter)); + } + } + + _nestedCounters.emplace_back(group); + + return Status::OK(); +} + +Status PerfCounterCollector::collectCounters(const std::vector& counters, + BSONObjBuilder* builder) { + for (const auto& counterInfo : counters) { + DWORD dwType = 0; + + // Elapsed Time is an unusual counter in that being able to control the sample period for + // the counter is uninteresting even though it is computed from two values. Just return the + // computed value instead. + if (counterInfo.type == PERF_ELAPSED_TIME) { + PDH_FMT_COUNTERVALUE counterValue = {0}; + PDH_STATUS status = PdhGetFormattedCounterValue( + counterInfo.handle, PDH_FMT_DOUBLE, &dwType, &counterValue); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhGetFormattedCounterValue", status)}; + } + + builder->append(counterInfo.firstName, counterValue.doubleValue); + + } else { + PDH_RAW_COUNTER rawCounter = {0}; + PDH_STATUS status = PdhGetRawCounterValue(counterInfo.handle, &dwType, &rawCounter); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhGetRawCounterValue", status)}; + } + + if (counterInfo.hasSecondValue) { + // Precise counters require the second value in the raw counter information + builder->append(counterInfo.firstName, rawCounter.FirstValue); + builder->append(counterInfo.secondName, rawCounter.SecondValue); + } else { + builder->append(counterInfo.firstName, rawCounter.FirstValue); + } + } + } + + return Status::OK(); +} + +void PerfCounterCollector::checkForTicksTimeBase() { + for (const auto& counterGroup : _counters) { + for (const auto& counter : counterGroup.counters) { + if (counterHasTickBasedTimeBase(counter.type)) { + _timeBaseTicksCounter = &counter; + return; + } + } + } + + for (const auto& counterGroup : _nestedCounters) { + for (const auto& instanceNamePair : counterGroup.counters) { + for (const auto& counter : instanceNamePair.second) { + if (counterHasTickBasedTimeBase(counter.type)) { + _timeBaseTicksCounter = &counter; + return; + } + } + } + } +} + +Status PerfCounterCollector::collect(BSONObjBuilder* builder) { + // Ask PDH to collect the counters + PDH_STATUS status = PdhCollectQueryData(_query); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhCollectQueryData", status)}; + } + + // Output timebase + // Counters that are based on time either use 100NS or System Ticks Per Second. + // We only need to output system ticks per second once if any counter depends on it. + // This is typically 3320310. + if (_timeBaseTicksCounter) { + int64_t timebase; + + status = PdhGetCounterTimeBase(_timeBaseTicksCounter->handle, &timebase); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhGetCounterTimeBase", status)}; + } + + builder->append("timebase", timebase); + } + + // Retrieve all the values that PDH collected for us. + for (const auto& counterGroup : _counters) { + BSONObjBuilder subObjBuilder(builder->subobjStart(counterGroup.name)); + + Status s = collectCounters(counterGroup.counters, &subObjBuilder); + if (!s.isOK()) { + return s; + } + + subObjBuilder.doneFast(); + } + + for (const auto& counterGroup : _nestedCounters) { + BSONObjBuilder subObjBuilder(builder->subobjStart(counterGroup.name)); + + for (const auto& instanceNamePair : counterGroup.counters) { + BSONObjBuilder instSubObjBuilder(builder->subobjStart(instanceNamePair.first)); + + Status s = collectCounters(instanceNamePair.second, &instSubObjBuilder); + if (!s.isOK()) { + return s; + } + + instSubObjBuilder.doneFast(); + } + + subObjBuilder.doneFast(); + } + + return Status::OK(); +} + +} // namespace mongo diff --git a/src/mongo/util/perfctr_collect.h b/src/mongo/util/perfctr_collect.h new file mode 100644 index 00000000000..5cf42409ff5 --- /dev/null +++ b/src/mongo/util/perfctr_collect.h @@ -0,0 +1,296 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/base/string_data.h" +#include "mongo/stdx/memory.h" + +namespace mongo { + +class BSONObjBuilder; + +/** + * PerfCounterCollection contains a set of counters for PerfCounterCollector to collect. This class + * supports adding counters with wildcards. It also optionally supports grouping counters by + * instance name. + */ +class PerfCounterCollection { + MONGO_DISALLOW_COPYING(PerfCounterCollection); + + friend class PerfCounterCollector; + +public: + PerfCounterCollection() = default; + PerfCounterCollection(PerfCounterCollection&& other) + : _counters(std::move(other._counters)), + _nestedCounters(std::move(other._nestedCounters)) {} + PerfCounterCollection& operator=(PerfCounterCollection&& other) { + _counters = std::move(other._counters); + _nestedCounters = std::move(other._nestedCounters); + return *this; + } + + /** + * Add vector of counters grouped under 'name'. + * + * group name - the name of the BSON document to add these counters into. + * paths - a vector of counter paths. These may contain wildcards. + * + * Errors if groupName duplicates an existing group or if paths has duplicate keys. Does not + * validate if the counters exist. + * + * Output document: + * For the following counters in "cpu": + * "\System\Processes" + * "\Processor(_Total)\% Idle Time" + * + * { + * "cpu" : { + * "\System\Processes" : 42, + * "\Processor\% Idle Time" : 12, + * "\Processor\% Idle Time Base" : 53, + * } + * } + */ + Status addCountersGroup(StringData groupName, const std::vector& paths); + + /** + * Add vector of counters grouped under 'name', and grouped by instance name. + * + * group name - the name of the BSON document to add these counters into. + * paths - a vector of counter paths. These may contain wildcards. The '_Total' instance is + * automatically filtered since it can be computed by summing other instances. + * + * Errors if groupName duplicates an existing group or if paths has duplicate keys. Does not + * validate if the counters exist. + * + * Output document: + * For the following counters in "disks": + * "\PhysicalDisk(*)\% Disk Write Time" + * + * { + * "disks" : { + * "0 C:" : { + * "\PhysicalDisk\% Disk Write Time": 42, + * "\PhysicalDisk\% Disk Write Time Base": 32, + * }, + * "1 D:" : { + * "\PhysicalDisk\% Disk Write Time": 43, + * "\PhysicalDisk\% Disk Write Time Base": 37, + * } + * } + * } + */ + Status addCountersGroupedByInstanceName(StringData groupName, + const std::vector& paths); + +private: + /** + * Check for duplicate group and counters. + */ + StatusWith> checkCounters(StringData groupName, + const std::vector& paths); + +private: + // Vector of counters which are not sub-grouped by instance name. + std::unordered_map> _counters; + + // Vector of counters sub grouped by instance name. + std::unordered_map> _nestedCounters; +}; + +/** + * PerfCounterCollector collects a series of counters from a Performance Data Helper (PDH) Query and + * output the raw counter values to BSONObjBuilder. + */ +class PerfCounterCollector { + MONGO_DISALLOW_COPYING(PerfCounterCollector); + +public: + ~PerfCounterCollector(); + PerfCounterCollector(PerfCounterCollector&& other) + : _query(std::move(other._query)), + _counters(std::move(other._counters)), + _nestedCounters(std::move(other._nestedCounters)), + _timeBaseTicksCounter(std::move(other._timeBaseTicksCounter)) {} + + /** + * Create a PerfCounterCollector to collect the performance counters in the specified + * PerfCounterCollection. + */ + static StatusWith> create(PerfCounterCollection builder); + + /** + * Collect the counters from PDH, and output their raw values into builder. The exception is + * elapsed time counters which returns computed values instead of raw values. + * + * For each counters, if the counter is a precision counter (see PERF_COUNTER_PRECISION), the + * second value is output under the name " Base". Also, a single field is output called + * "timebase" if any counter depends on system ticks per second. See counterHasTickBasedTimeBase + * for more details about timebase. + */ + Status collect(BSONObjBuilder* builder); + +private: + /** + * Describes a counter by querying PDH, and contains the necessary information to retrieve a + * counter from PDH. + */ + struct CounterInfo { + /** + * The name of the first value for a counter. This is output as: + * "\\". + */ + std::string firstName; + + /** + * The name of the second value of a counter if the counter is a precision counter. This is + * output as: "\\ Base". + */ + std::string secondName; + + /** + * True if the counter is a precision counter, and its value should be output in the output + * BSON document. + */ + bool hasSecondValue; + + /** + * Instance name of the counter. Empty if the counter has no instance name. + */ + std::string instanceName; + + /** + * Counter Type. See PERF_* constants in winperf.h. + * https://technet.microsoft.com/en-us/library/cc785636(v=ws.10).aspx + */ + uint32_t type; + + /** + * Handle of counter to collect from. + */ + PDH_HCOUNTER handle; + }; + + /** + * A set of counters that are part of "name" in the final bson document. + */ + struct CounterGroup { + /** + * Name of the counter group. + */ + std::string name; + + /** + * Vector of counters in this group. + */ + std::vector counters; + }; + + /** + * A set of counters that are part of "name" and "instanceName" in the final bson document. + */ + struct NestedCounterGroup { + /** + * Name of the counter group. + */ + std::string name; + + /** + * A map of instance name to vector of counters to collect for each instance name. + * Ordered Map to ensure output is well-ordered. + */ + std::map> counters; + }; + +private: + PerfCounterCollector() = default; + + /** + * Open the PDH Query. + */ + Status open(); + + /** + * Add the specified counter group to the PDH Query. + */ + Status addCountersGroup(StringData groupName, const std::vector& paths); + + /** + * Add the specified counter group to the PDH Query grouped by instance name. + */ + Status addCountersGroupedByInstanceName(StringData groupName, + const std::vector& paths); + + /** + * Add a counter to the PDH query and get a description of it. + */ + StatusWith addCounter(StringData path); + + /** + * Add a set of counters to the PDH query, and get descriptions of them. + */ + StatusWith> addCounters(StringData path); + + /** + * Collect a vector of counters and output them to builder. + */ + Status collectCounters(const std::vector& counters, BSONObjBuilder* builder); + + /** + * Check if any of the counters we want depends on system ticks per second as a time base. + */ + void checkForTicksTimeBase(); + +private: + // PDH Query + HQUERY _query{INVALID_HANDLE_VALUE}; + + // Typically: CPU & Memory counters + std::vector _counters; + + // Typically: Disks counters + std::vector _nestedCounters; + + // A counter that uses ticks as a timebase + const CounterInfo* _timeBaseTicksCounter{nullptr}; +}; + +} // namespace mongo diff --git a/src/mongo/util/perfctr_collect_test.cpp b/src/mongo/util/perfctr_collect_test.cpp new file mode 100644 index 00000000000..a5d519f69ad --- /dev/null +++ b/src/mongo/util/perfctr_collect_test.cpp @@ -0,0 +1,416 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/perfctr_collect.h" + +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" + +namespace mongo { + +namespace { +using StringMap = std::map; + +/** + * Convert BSON document nested up to 3 levels into a map where keys are dot separated paths to + * values. + */ +StringMap toNestedStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& parent : obj) { + if (parent.isABSONObj()) { + std::string parentNamePrefix = std::string(parent.fieldName()) + "."; + + for (const auto& child : parent.Obj()) { + if (child.isABSONObj()) { + std::string childNamePrefix = parentNamePrefix + child.fieldName() + "."; + + for (const auto& grandChild : child.Obj()) { + map[childNamePrefix + grandChild.fieldName()] = grandChild.numberLong(); + } + + } else { + map[parentNamePrefix + child.fieldName()] = child.numberLong(); + } + } + } else { + map[parent.fieldName()] = parent.numberLong(); + } + } + + return map; +} + +#define ASSERT_KEY(_key) ASSERT_TRUE(stringMap.find(_key) != stringMap.end()); +#define ASSERT_NO_KEY(_key) ASSERT_TRUE(stringMap.find(_key) == stringMap.end()); + +#define ASSERT_TIMEBASE ASSERT_KEY("timebase"); +#define ASSERT_NO_TIMEBASE ASSERT_NO_KEY("timebase"); + +#define ASSERT_GROUP_AND_RAW_COUNTER(g, c) \ + ASSERT_KEY(g "." c); \ + ASSERT_NO_KEY(g "." c " Base"); + +#define ASSERT_GROUP_AND_NON_RAW_COUNTER(g, c) \ + ASSERT_KEY(g "." c); \ + ASSERT_KEY(g "." c " Base"); + +#define ASSERT_NESTED_GROUP_AND_RAW_COUNTER(g, p, c) \ + ASSERT_KEY(g "." p "." c); \ + ASSERT_NO_KEY(g "." p "." c " Base"); + +#define ASSERT_NO_NESTED_GROUP_AND_RAW_COUNTER(g, p, c) \ + ASSERT_NO_KEY(g "." p "." c); \ + ASSERT_NO_KEY(g "." p "." c " Base"); + +#define ASSERT_NESTED_GROUP_AND_NON_RAW_COUNTER(g, p, c) \ + ASSERT_KEY(g "." p "." c); \ + ASSERT_KEY(g "." p "." c " Base"); + +#define ASSERT_NO_NESTED_GROUP_AND_NON_RAW_COUNTER(g, p, c) \ + ASSERT_NO_KEY(g "." p "." c); \ + ASSERT_NO_KEY(g "." p "." c " Base"); + +#define COLLECT_COUNTERS_VERBOSE \ + BSONObjBuilder builder; \ + ASSERT_OK(collector->collect(&builder)); \ + auto obj = builder.obj(); \ + log() << "OBJ:" << obj; \ + auto stringMap = toNestedStringMap(obj); \ + for (const auto& kvp : stringMap) { \ + log() << "kvp " << kvp.first << " - " << kvp.second; \ + } + +#define COLLECT_COUNTERS_QUIET \ + BSONObjBuilder builder; \ + ASSERT_OK(collector->collect(&builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toNestedStringMap(obj); + +#define COLLECT_COUNTERS COLLECT_COUNTERS_QUIET + +size_t kDefaultCollectionCount = 2; + +// Simple verification test +TEST(FTDCPerfCollector, TestSingleCounter) { + PerfCounterCollection collection; + // PERF_100NSEC_TIMER + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\% Idle Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_NO_TIMEBASE; + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Idle Time"); + } +} + + +// Simple verification test +TEST(FTDCPerfCollector, TestSingleRawCounter) { + PerfCounterCollection collection; + // PERF_COUNTER_RAWCOUNT + ASSERT_OK(collection.addCountersGroup("cpu", {"\\System\\Processes"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_NO_TIMEBASE; + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Processes"); + } +} + +// Test negative cases for collection +TEST(FTDCPerfCollector, TestBadCollectionInput) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\% Idle Time"})); + + // Duplicate group + ASSERT_NOT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\% Idle Time"})); + + // Duplicate counter + ASSERT_NOT_OK( + collection.addCountersGroup("cpu2", + { + "\\Processor(0)\\% Idle Time", "\\Processor(0)\\% Idle Time", + })); + + // Duplicate group + ASSERT_NOT_OK( + collection.addCountersGroupedByInstanceName("cpu", {"\\Processor(0)\\% Idle Time"})); + + // Duplicate counter + ASSERT_NOT_OK(collection.addCountersGroupedByInstanceName( + "cpu2", + { + "\\Processor(0)\\% Idle Time", "\\Processor(0)\\% Idle Time", + })); +} + +// Test negative collector input +TEST(FTDCPerfCollector, TestBadCollectorInput) { + // Bad counter name + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\DOES NOT EXIST"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } + + // Bad wild card + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\DOES*"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } + + // Use addCounterGroup with instance wildcard + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(*)\\\\% Idle Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } + + // Use addCountersGroupedByInstanceName without instance name + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroupedByInstanceName("cpu", {"\\System\\Processes"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } +} + +// Test all the different counter types we use in the MongoDB code +TEST(FTDCPerfCollector, TestCounterTypes) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup( + "misc", + { + "\\Processor(0)\\% Idle Time", // PERF_100NSEC_TIMER + "\\Processor(0)\\% Processor Time", // PERF_100NSEC_TIMER_INV + "\\System\\Processes", // PERF_COUNTER_RAWCOUNT + "\\System\\System Up Time", // PERF_ELAPSED_TIME + "\\Memory\\Available Bytes", // PERF_COUNTER_LARGE_RAWCOUNT + "\\PhysicalDisk(_Total)\\% Disk Write Time", // PERF_PRECISION_100NS_TIMER + "\\PhysicalDisk(_Total)\\Avg. Disk Bytes/Write", // PERF_AVERAGE_BULK + "\\PhysicalDisk(_Total)\\Avg. Disk Read Queue Length", // PERF_COUNTER_LARGE_QUEUELEN_TYPE + "\\PhysicalDisk(_Total)\\Avg. Disk sec/Write", // PERF_AVERAGE_TIMER + "\\PhysicalDisk(_Total)\\Disk Write Bytes/sec", // PERF_COUNTER_BULK_COUNT + "\\PhysicalDisk(_Total)\\Disk Writes/sec", // PERF_COUNTER_COUNTER + })); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_TIMEBASE + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\Processor\\% Idle Time"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\Processor\\% Processor Time"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\System\\System Up Time"); + ASSERT_GROUP_AND_NON_RAW_COUNTER("misc", "\\PhysicalDisk\\% Disk Write Time"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Avg. Disk Bytes/Write"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Avg. Disk Read Queue Length"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Avg. Disk sec/Write"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Disk Write Bytes/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Disk Writes/sec"); + + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\System\\Processes"); + } +} + +// Test multiple counter groups +TEST(FTDCPerfCollector, TestMultipleCounterGroups) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup( + "cpu", {"\\Processor(0)\\% Idle Time", "\\Processor(0)\\% Processor Time"})); + ASSERT_OK( + collection.addCountersGroup("sys", {"\\System\\Processes", "\\System\\System Up Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_TIMEBASE + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Idle Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Processor Time"); + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\System Up Time"); + + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\Processes"); + } +} + +// Test multiple nested counter groups +TEST(FTDCPerfCollector, TestMultipleNestedCounterGroups) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroupedByInstanceName( + "cpu", {"\\Processor(*)\\% Idle Time", "\\Processor(*)\\% Processor Time"})); + ASSERT_OK( + collection.addCountersGroup("sys", {"\\System\\Processes", "\\System\\System Up Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + ASSERT_TIMEBASE + + ASSERT_NESTED_GROUP_AND_RAW_COUNTER("cpu", "0", "\\Processor\\% Idle Time"); + ASSERT_NESTED_GROUP_AND_RAW_COUNTER("cpu", "0", "\\Processor\\% Processor Time"); + + ASSERT_NO_NESTED_GROUP_AND_RAW_COUNTER("cpu", "_Total", "\\Processor\\% Idle Time"); + ASSERT_NO_NESTED_GROUP_AND_RAW_COUNTER("cpu", "_Total", "\\Processor\\% Processor Time"); + + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\System Up Time"); + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\Processes"); + } +} + +// Test Counters we use in MongoDB +TEST(FTDCPerfCollector, TestLocalCounters) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", + { + "\\Processor(_Total)\\% Idle Time", + "\\Processor(_Total)\\% Interrupt Time", + "\\Processor(_Total)\\% Privileged Time", + "\\Processor(_Total)\\% Processor Time", + "\\Processor(_Total)\\% User Time", + "\\Processor(_Total)\\Interrupts/sec", + "\\System\\Context Switches/sec", + "\\System\\Processes", + "\\System\\Processor Queue Length", + "\\System\\System Up Time", + "\\System\\Threads", + })); + + // TODO: Should we capture the Heap Counters for the current process? + ASSERT_OK(collection.addCountersGroup("memory", + { + "\\Memory\\Available Bytes", + "\\Memory\\Cache Bytes", + "\\Memory\\Cache Faults/sec", + "\\Memory\\Committed Bytes", + "\\Memory\\Commit Limit", + "\\Memory\\Page Reads/sec", + "\\Memory\\Page Writes/sec", + "\\Memory\\Pages Input/sec", + "\\Memory\\Pages Output/sec", + "\\Memory\\Pool Nonpaged Bytes", + "\\Memory\\Pool Paged Bytes", + "\\Memory\\Pool Paged Resident Bytes", + "\\Memory\\System Cache Resident Bytes", + "\\Memory\\System Code Total Bytes", + })); + + ASSERT_OK(collection.addCountersGroupedByInstanceName( + "disks", + { + "\\PhysicalDisk(*)\\% Disk Read Time", + "\\PhysicalDisk(*)\\% Disk Write Time", + "\\PhysicalDisk(*)\\Avg. Disk Bytes/Read", + "\\PhysicalDisk(*)\\Avg. Disk Bytes/Write", + "\\PhysicalDisk(*)\\Avg. Disk Read Queue Length", + "\\PhysicalDisk(*)\\Avg. Disk Write Queue Length", + "\\PhysicalDisk(*)\\Avg. Disk sec/Read", + "\\PhysicalDisk(*)\\Avg. Disk sec/Write", + "\\PhysicalDisk(*)\\Disk Read Bytes/sec", + "\\PhysicalDisk(*)\\Disk Write Bytes/sec", + "\\PhysicalDisk(*)\\Disk Reads/sec", + "\\PhysicalDisk(*)\\Disk Writes/sec", + "\\PhysicalDisk(*)\\Current Disk Queue Length", + })); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + ASSERT_TIMEBASE + + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Idle Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Interrupt Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Privileged Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Processor Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% User Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\Interrupts/sec"); + + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Context Switches/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Processes"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Processor Queue Length"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\System Up Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Threads"); + + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Available Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Cache Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Cache Faults/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Commit Limit"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Committed Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Page Reads/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Page Writes/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Pages Input/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Pages Output/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Pool Paged Resident Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\System Cache Resident Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\System Code Total Bytes"); + } +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/util/processinfo_linux.cpp b/src/mongo/util/processinfo_linux.cpp index 8addc447ef3..a6a31f0c467 100644 --- a/src/mongo/util/processinfo_linux.cpp +++ b/src/mongo/util/processinfo_linux.cpp @@ -459,7 +459,7 @@ void ProcessInfo::SystemInfo::collectSystemInfo() { osName = distroName; osVersion = distroVersion; memSize = LinuxSysHelper::getSystemMemorySize(); - addrSize = (string(unameData.machine).find("x86_64") != string::npos ? 64 : 32); + addrSize = sizeof(void*) * CHAR_BIT; numCores = cpuCount; pageSize = static_cast(sysconf(_SC_PAGESIZE)); cpuArch = unameData.machine; diff --git a/src/mongo/util/procparser.cpp b/src/mongo/util/procparser.cpp new file mode 100644 index 00000000000..bafc674fc06 --- /dev/null +++ b/src/mongo/util/procparser.cpp @@ -0,0 +1,612 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/procparser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mongo/base/parse_number.h" +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/scopeguard.h" +#include "mongo/util/text.h" +namespace mongo { + +namespace { + +/** + * Get USER_HZ for the machine. See time(7) for an explanation. + */ +int64_t getTicksPerSecond() { + int64_t ret = sysconf(_SC_CLK_TCK); + return ret; +} + +/** + * Convert USER_HZ to milliseconds. + */ +double convertTicksToMilliSeconds(const int64_t ticks, const int64_t ticksPerSecond) { + return static_cast(ticks) / (static_cast(ticksPerSecond) / 1000.0); +} + +const size_t kFileBufferSize = 16384; +const size_t kFileReadRetryCount = 5; + +constexpr auto kSysBlockDeviceDirectoryName = "device"; + +/** + * Read a file from disk as a string with a null-terminating byte using the POSIX file api. + * + * This function is designed to get all the data it needs from small /proc files in a single read. + * The /proc/stat and /proc/diskstats files can vary in size, but 16kb will cover most cases. + * + * Finally, we return errors instead of throwing to ensure that FTDC can return partial information + * on failure instead of no information. Some container filesystems may overlay /proc so we may not + * be reading directly from the kernel. + */ +StatusWith readFileAsString(StringData filename) { + int fd = open(filename.toString().c_str(), 0); + if (fd == -1) { + int err = errno; + return Status(ErrorCodes::FileOpenFailed, + str::stream() << "Failed to open file " << filename + << " with error: " << errnoWithDescription(err)); + } + auto scopedGuard = MakeGuard([fd] { close(fd); }); + + BufBuilder builder(kFileBufferSize); + std::array buf; + + ssize_t size_read = 0; + + // Read until the end as needed + do { + // Retry if interrupted + size_t retry = 0; + + do { + size_read = read(fd, buf.data(), kFileBufferSize); + + if (size_read == -1) { + int err = errno; + + // Retry if we hit EGAIN or EINTR a few times before giving up + if (retry < kFileReadRetryCount && (err == EAGAIN || err == EINTR)) { + ++retry; + continue; + } + + return Status(ErrorCodes::FileStreamFailed, + str::stream() << "Failed to read file " << filename + << " with error: " << errnoWithDescription(err)); + } + + break; + } while (true); + + if (size_read != 0) { + builder.appendBuf(buf.data(), size_read); + } + } while (size_read != 0); + + // Null terminate the buffer since we are about to convert it to a string + builder.appendChar(0); + + return std::string(builder.buf(), builder.len()); +} + + +const char* const kAdditionCpuFields[] = {"user_ms", + "nice_ms", + "system_ms", + "idle_ms", + "iowait_ms", + "irq_ms", + "softirq_ms", + "steal_ms", + "guest_ms", + "guest_nice_ms"}; +const size_t kAdditionCpuFieldCount = std::extent::value; + +const char* const kDiskFields[] = { + "reads", + "reads_merged", + "read_sectors", + "read_time_ms", + "writes", + "writes_merged", + "write_sectors", + "write_time_ms", + "io_in_progress", + "io_time_ms", + "io_queued_ms", +}; + +const size_t kDiskFieldCount = std::extent::value; + +} // namespace + +namespace procparser { +// Here is an example of the type of string it supports. +// Note: intr output has been trimmed +// +// The cpu field maps up to 10 individual fields depending on the kernel version. For other views, +// this code assumes there is only a single value. +// +// For more information, see: +// Documentation/filesystems/proc.txt in the Linux kernel +// proc(5) man page +// +// > cat /proc/stat +// cpu 41801 9179 32206 831134223 34279 0 947 0 0 0 +// cpu0 2977 450 2475 69253074 1959 0 116 0 0 0 +// cpu1 6213 4261 9400 69177349 845 0 539 0 0 0 +// cpu2 1949 831 3699 69261035 645 0 0 0 0 0 +// cpu3 2222 644 3283 69264801 783 0 0 0 0 0 +// cpu4 16576 607 4757 69232589 8195 0 291 0 0 0 +// cpu5 3742 391 4571 69257332 2322 0 0 0 0 0 +// cpu6 2173 376 743 69284308 400 0 0 0 0 0 +// cpu7 1232 375 704 69285753 218 0 0 0 0 0 +// cpu8 960 127 576 69262851 18107 0 0 0 0 0 +// cpu9 1755 227 744 69283938 362 0 0 0 0 0 +// cpu10 1380 641 678 69285193 219 0 0 0 0 0 +// cpu11 618 244 572 69285995 218 0 0 0 0 0 +// intr 54084718 135 2 .... +// ctxt 190305514 +// btime 1463584038 +// processes 47438 +// procs_running 1 +// procs_blocked 0 +// softirq 102690251 8 26697410 115481 23345078 816026 0 2296 26068778 0 25645174 +// +Status parseProcStat(const std::vector& keys, + StringData data, + int64_t ticksPerSecond, + BSONObjBuilder* builder) { + bool foundKeys = false; + + using string_split_iterator = boost::split_iterator; + + // Split the file by lines. + // token_compress_on means the iterator skips over consecutive '\n'. This should not be a + // problem in normal /proc/stat output. + for (string_split_iterator lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt) { + StringData line((*lineIt).begin(), (*lineIt).end()); + + // Split the line by spaces since that is the only delimiter for stat files. + // token_compress_on means the iterator skips over consecutive ' '. This is needed for the + // first line which is "cpu ". + string_split_iterator partIt = string_split_iterator( + line.begin(), + line.end(), + boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on)); + + // Skip processing this line if we do not have a key. + if (partIt == string_split_iterator()) { + continue; + } + + StringData key((*partIt).begin(), (*partIt).end()); + + ++partIt; + + // Skip processing this line if we only have a key, and no number. + if (partIt == string_split_iterator()) { + continue; + } + + // Check if the key is in the list. /proc/stat will have extra keys, and + // may not have the keys we want. + if (keys.empty() || std::find(keys.begin(), keys.end(), key) != keys.end()) { + foundKeys = true; + + if (key == "cpu") { + // Cpu is 10 fields, we need to chew through all of them. + // Some kernels we support lack the last field or two: guest and/or guest_nice. + for (size_t index = 0; + partIt != string_split_iterator() && index < kAdditionCpuFieldCount; + ++partIt, ++index) { + StringData stringValue((*partIt).begin(), (*partIt).end() - (*partIt).begin()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + builder->appendNumber(kAdditionCpuFields[index], + convertTicksToMilliSeconds(value, ticksPerSecond)); + } + } else { + StringData stringValue((*partIt).begin(), (*partIt).end() - (*partIt).begin()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + builder->appendNumber(key, static_cast(value)); + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in stat string"); +} + +Status parseProcStatFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + + return parseProcStat(keys, swString.getValue(), getTicksPerSecond(), builder); +} + +// Here is an example of the type of string it supports: +// Note: output has been trimmed +// +// For more information, see: +// Documentation/filesystems/proc.txt in the Linux kernel +// proc(5) man page +// +// > cat /proc/meminfo +// MemTotal: 12294392 kB +// MemFree: 3652612 kB +// MemAvailable: 11831704 kB +// Buffers: 568536 kB +// Cached: 6421520 kB +// SwapCached: 0 kB +// HugePages_Total: 0 +// +// Note: HugePages_* do not end in kB, it is not a typo +// +Status parseProcMemInfo(const std::vector& keys, + StringData data, + BSONObjBuilder* builder) { + bool foundKeys = false; + + using string_split_iterator = boost::split_iterator; + + // Split the file by lines. + // token_compress_on means the iterator skips over consecutive '\n'. This should not be a + // problem in normal /proc/memInfo output. + for (string_split_iterator lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt) { + StringData line((*lineIt).begin(), (*lineIt).end()); + + // Split the line by spaces and colons since these are the delimiters for meminfo files. + // token_compress_on means the iterator skips over consecutive ' '. This is needed for + // every line. + string_split_iterator partIt = string_split_iterator(line.begin(), + line.end(), + boost::token_finder([](char c) { + return c == ' ' || c == ':'; + }, boost::token_compress_on)); + + // Skip processing this line if we do not have a key. + if (partIt == string_split_iterator()) { + continue; + } + + StringData key((*partIt).begin(), (*partIt).end()); + + ++partIt; + + // Skip processing this line if we only have a key, and no number. + if (partIt == string_split_iterator()) { + continue; + } + + // Check if the key is in the list. /proc/meminfo will have extra keys, and may not have the + // keys we want. + if (keys.empty() || std::find(keys.begin(), keys.end(), key) != keys.end()) { + foundKeys = true; + + StringData stringValue((*partIt).begin(), (*partIt).end()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + // Check if the line ends in "kB" + ++partIt; + + // If there is one last token, check if it is actually "kB" + if (partIt != string_split_iterator()) { + StringData kb_token((*partIt).begin(), (*partIt).end()); + auto keyWithSuffix = key.toString(); + + if (kb_token == "kB") { + keyWithSuffix.append("_kb"); + } + + builder->appendNumber(keyWithSuffix, static_cast(value)); + } else { + builder->appendNumber(key, static_cast(value)); + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in meminfo string"); +} + +Status parseProcMemInfoFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + + return parseProcMemInfo(keys, swString.getValue(), builder); +} + +// Here is an example of the type of string it supports: +// +// For more information, see: +// Documentation/iostats.txt in the Linux kernel +// proc(5) man page +// +// > cat /proc/diskstats +// 8 0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 2554160 +// 8 1 sda1 138 37 8642 315 3 0 18 14 0 292 329 +// 8 2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 2550739 +// 8 16 sdb 12707 3876 1525418 57507 997 3561 297576 97976 0 37870 155619 +// 8 17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468 +// 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0 +// 253 0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275 +// 253 1 dm-1 109 0 4584 226 0 0 0 0 0 172 226 +// +Status parseProcDiskStats(const std::vector& disks, + StringData data, + BSONObjBuilder* builder) { + bool foundKeys = false; + std::vector stats; + stats.reserve(kDiskFieldCount); + + using string_split_iterator = boost::split_iterator; + + // Split the file by lines. + // token_compress_on means the iterator skips over consecutive '\n'. This should not be a + // problem in normal /proc/diskstats output. + for (string_split_iterator lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt) { + StringData line((*lineIt).begin(), (*lineIt).end()); + + // Skip leading whitespace so that the split_iterator starts on non-whitespace otherwise we + // get an empty first token. Device major numbers (the first number on each line) are right + // aligned to 4 spaces and start from + // single digits. + auto beginNonWhitespace = + std::find_if_not(line.begin(), line.end(), [](char c) { return c == ' '; }); + + // Split the line by spaces since that is the only delimiter for diskstats files. + // token_compress_on means the iterator skips over consecutive ' '. + string_split_iterator partIt = string_split_iterator( + beginNonWhitespace, + line.end(), + boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on)); + + // Skip processing this line if the line is blank + if (partIt == string_split_iterator()) { + continue; + } + + ++partIt; + + // Skip processing this line if we only have a device major number. + if (partIt == string_split_iterator()) { + continue; + } + + ++partIt; + + // Skip processing this line if we only have a device major minor. + if (partIt == string_split_iterator()) { + continue; + } + + StringData disk((*partIt).begin(), (*partIt).end()); + + // Skip processing this line if we only have a block device name. + if (partIt == string_split_iterator()) { + continue; + } + + ++partIt; + + // Check if the disk is in the list. /proc/diskstats will have extra disks, and may not have + // the disk we want. + if (disks.empty() || std::find(disks.begin(), disks.end(), disk) != disks.end()) { + foundKeys = true; + + stats.clear(); + + // Only generate a disk document if the disk has some activity. For instance, there + // could be a CD-ROM drive that is not used. + bool hasSomeNonZeroStats = false; + + for (size_t index = 0; partIt != string_split_iterator() && index < kDiskFieldCount; + ++partIt, ++index) { + StringData stringValue((*partIt).begin(), (*partIt).end()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + if (value != 0) { + hasSomeNonZeroStats = true; + } + + stats.push_back(value); + } + + if (hasSomeNonZeroStats) { + // Start a new document with disk as the name. + BSONObjBuilder sub(builder->subobjStart(disk)); + + for (size_t index = 0; index < stats.size() && index < kDiskFieldCount; ++index) { + sub.appendNumber(kDiskFields[index], static_cast(stats[index])); + } + + sub.doneFast(); + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in diskstats string"); +} + +Status parseProcDiskStatsFile(StringData filename, + const std::vector& disks, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + + return parseProcDiskStats(disks, swString.getValue(), builder); +} + +namespace { + +/** + * Is this a disk that is interesting to us? We only want physical disks, not multiple disk devices, + * LVM2 devices, partitions, or RAM disks. + * + * A physical disk has a symlink to a directory at /sys/block//device. + * + * Note: returns false upon any errors such as access denied. + */ +bool isInterestingDisk(const boost::filesystem::path& path) { + boost::filesystem::path blockDevicePath(path); + blockDevicePath /= kSysBlockDeviceDirectoryName; + + boost::system::error_code ec; + auto statusSysBlock = boost::filesystem::status(blockDevicePath, ec); + if (!boost::filesystem::exists(statusSysBlock)) { + return false; + } + + if (ec) { + warning() << "Error checking directory '" << blockDevicePath.generic_string() + << "': " << ec.message(); + return false; + } + + if (!boost::filesystem::is_directory(statusSysBlock)) { + return false; + } + + return true; +} + +} // namespace + +std::vector findPhysicalDisks(StringData sysBlockPath) { + boost::system::error_code ec; + auto sysBlockPathStr = sysBlockPath.toString(); + + auto statusSysBlock = boost::filesystem::status(sysBlockPathStr, ec); + if (ec) { + warning() << "Error checking directory '" << sysBlockPathStr << "': " << ec.message(); + return {}; + } + + if (!(boost::filesystem::exists(statusSysBlock) && + boost::filesystem::is_directory(statusSysBlock))) { + warning() << "Could not find directory '" << sysBlockPathStr << "': " << ec.message(); + return {}; + } + + std::vector files; + + // Iterate through directories in /sys/block. The directories in this directory can be physical + // block devices (like SSD or HDD) or virtual devices like the LVM2 device mapper or a multiple + // disk device. It does not contain disk partitions. + boost::filesystem::directory_iterator di(sysBlockPathStr, ec); + if (ec) { + warning() << "Error getting directory iterator '" << sysBlockPathStr + << "': " << ec.message(); + return {}; + } + + for (; di != boost::filesystem::directory_iterator(); di++) { + auto path = (*di).path(); + + if (isInterestingDisk(path)) { + files.push_back(path.filename().generic_string()); + } + } + + return files; +} + +} // namespace procparser +} // namespace mongo diff --git a/src/mongo/util/procparser.h b/src/mongo/util/procparser.h new file mode 100644 index 00000000000..2f9e97e343c --- /dev/null +++ b/src/mongo/util/procparser.h @@ -0,0 +1,120 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" + +namespace mongo { + +class BSONObjBuilder; + +namespace procparser { + +/** + * Reads a string matching /proc/stat format, and writes the values of the specified keys into + * builder. If fields are not in the "data" parameter, they are omitted. Converts fields + * from USER_HZ to milliseconds, and names fields with a "_ms" suffix. If the string is empty, + * corrupt, or missing fields, the builder will simply be missing fields. + * + * keys - sorted vector of field names to include in the output, "cpu" will include the 11 fields + * that make up cpu. If keys is empty, all keys are outputed. + * data - string to parsee + * ticksPerSecond - USER_HZ value + * builder - BSON output + */ +Status parseProcStat(const std::vector& keys, + StringData data, + int64_t ticksPerSecond, + BSONObjBuilder* builder); + +/** +* Read from file, and write the specified list of keys into builder. +* +* See parseProcStat. +* +* Returns Status errors on file reading issues. +*/ +Status parseProcStatFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder); + +/** + * Read a string matching /proc/meminfo format, and write the specified list of keys in builder. + * + * keys - list of keys to output in BSON. If keys is empty, all keys are outputed. + * data - string to parsee + * builder - BSON output + */ +Status parseProcMemInfo(const std::vector& keys, + StringData data, + BSONObjBuilder* builder); + +/** + * Read from file, and write the specified list of keys in builder. + */ +Status parseProcMemInfoFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder); + +/** + * Read a string matching /proc/diskstats format, and write the specified list of disks in builder. + * + * disks - vector of block devices to include in output. For each disk selected, 11 fields are + * output in a nested document. There is no error if the disk is not found in the data. Also + * a disk is excluded if it has no activity since startup (i.e. an idle CD-ROM drive). If + * disks is empty, all non-zero block devices are outputed (this will include partitions, + * etc). + * data - string to parsee + * builder - BSON output + */ +Status parseProcDiskStats(const std::vector& disks, + StringData data, + BSONObjBuilder* builder); + +/** + * Read from file, and write the specified list of disks in builder. + */ +Status parseProcDiskStatsFile(StringData filename, + const std::vector& disks, + BSONObjBuilder* builder); + +/** + * Get a vector of disks to monitor by enumerating the specified directory. + * + * If the directory does not exist, or otherwise permission is denied, returns an empty vector. + */ +std::vector findPhysicalDisks(StringData directory); + +} // namespace procparser +} // namespace mongo diff --git a/src/mongo/util/procparser_test.cpp b/src/mongo/util/procparser_test.cpp new file mode 100644 index 00000000000..4cbc02df68a --- /dev/null +++ b/src/mongo/util/procparser_test.cpp @@ -0,0 +1,526 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/procparser.h" + +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" + +namespace mongo { + +namespace { +using StringMap = std::map; + +StringMap toStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& e : obj) { + map[e.fieldName()] = e.numberLong(); + } + + return map; +} + +StringMap toNestedStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& e : obj) { + if (e.isABSONObj()) { + std::string prefix = std::string(e.fieldName()) + "."; + + for (const auto& child : e.Obj()) { + map[prefix + child.fieldName()] = child.numberLong(); + } + } + } + + return map; +} + +#define ASSERT_KEY(_key) ASSERT_TRUE(stringMap.find(_key) != stringMap.end()); +#define ASSERT_NO_KEY(_key) ASSERT_TRUE(stringMap.find(_key) == stringMap.end()); +#define ASSERT_KEY_AND_VALUE(_key, _value) ASSERT_EQUALS(stringMap.at(_key), _value); + +#define ASSERT_PARSE_STAT(_keys, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcStat(_keys, _x, 1000, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toStringMap(obj); +#define ASSERT_PARSE_MEMINFO(_keys, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcMemInfo(_keys, _x, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toStringMap(obj); +#define ASSERT_PARSE_DISKSTATS(_disks, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcDiskStats(_disks, _x, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toNestedStringMap(obj); + +TEST(FTDCProcStat, TestStat) { + std::vector keys{"cpu", "ctxt", "processes"}; + + // Normal case + { + ASSERT_PARSE_STAT( + keys, + "cpu 41801 9179 32206 831134223 34279 0 947 0 0 0\n" + "cpu0 2977 450 2475 69253074 1959 0 116 0 0 0\n" + "cpu1 6213 4261 9400 69177349 845 0 539 0 0 0\n" + "cpu2 1949 831 3699 69261035 645 0 0 0 0 0\n" + "cpu3 2222 644 3283 69264801 783 0 0 0 0 0\n" + "cpu4 16576 607 4757 69232589 8195 0 291 0 0 0\n" + "cpu5 3742 391 4571 69257332 2322 0 0 0 0 0\n" + "cpu6 2173 376 743 69284308 400 0 0 0 0 0\n" + "cpu7 1232 375 704 69285753 218 0 0 0 0 0\n" + "cpu8 960 127 576 69262851 18107 0 0 0 0 0\n" + "cpu9 1755 227 744 69283938 362 0 0 0 0 0\n" + "cpu10 1380 641 678 69285193 219 0 0 0 0 0\n" + "cpu11 618 244 572 69285995 218 0 0 0 0 0\n" + "intr 54084718 135 2 ....\n" + "ctxt 190305514\n" + "btime 1463584038\n" + "processes 47438\n" + "procs_running 1\n" + "procs_blocked 0\n" + "softirq 102690251 8 26697410 115481 23345078 816026 0 2296 26068778 0 25645174\n"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_KEY_AND_VALUE("nice_ms", 9179UL); + ASSERT_KEY_AND_VALUE("system_ms", 32206UL); + ASSERT_KEY_AND_VALUE("idle_ms", 831134223UL); + ASSERT_KEY_AND_VALUE("iowait_ms", 34279UL); + ASSERT_KEY_AND_VALUE("irq_ms", 0UL); + ASSERT_KEY_AND_VALUE("softirq_ms", 947UL); + ASSERT_KEY_AND_VALUE("steal_ms", 0UL); + ASSERT_KEY_AND_VALUE("guest_ms", 0UL); + ASSERT_KEY_AND_VALUE("guest_nice_ms", 0UL); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_KEY_AND_VALUE("processes", 47438UL); + } + + // Missing fields in cpu and others + { + ASSERT_PARSE_STAT(keys, + "cpu 41801 9179 32206\n" + "ctxt 190305514\n"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_KEY_AND_VALUE("nice_ms", 9179UL); + ASSERT_KEY_AND_VALUE("system_ms", 32206UL); + ASSERT_NO_KEY("idle_ms"); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_NO_KEY("processes"); + } + + // Missing fields in cpu and others + { + ASSERT_PARSE_STAT(keys, + "cpu 41801\n" + "ctxt 190305514\n"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_NO_KEY("nice_ms"); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_NO_KEY("processes"); + } + + // Missing fields in cpu + { + ASSERT_PARSE_STAT(keys, + "cpu \n" + "ctxt 190305514\n"); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_NO_KEY("processes"); + } + + // Single string with only cpu and numbers + { + ASSERT_PARSE_STAT(keys, "cpu 41801 9179 32206"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_KEY_AND_VALUE("nice_ms", 9179UL); + ASSERT_KEY_AND_VALUE("system_ms", 32206UL); + ASSERT_NO_KEY("idle_ms"); + } + + // Single string with only cpu + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcStat(keys, "cpu", 1000, &builder)); + } + + // Single string with only cpu and a number, and empty ctxt + { + ASSERT_PARSE_STAT(keys, + "cpu 123\n" + "ctxt"); + ASSERT_KEY_AND_VALUE("user_ms", 123UL); + } + + // Empty String + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcStat(keys, "", 1000, &builder)); + } +} + +// Test we can parse the /proc/stat on this machine. Also assert we have the expected fields +// This tests is designed to exercise our parsing code on various Linuxes and fail +// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure +// otherwise. +TEST(FTDCProcStat, TestLocalStat) { + std::vector keys{ + "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running", + }; + + BSONObjBuilder builder; + + ASSERT_OK(procparser::parseProcStatFile("/proc/stat", keys, &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + log() << "OBJ:" << obj; + ASSERT_KEY("user_ms"); + ASSERT_KEY("nice_ms"); + ASSERT_KEY("idle_ms"); + ASSERT_KEY("system_ms"); + ASSERT_KEY("iowait_ms"); + ASSERT_KEY("irq_ms"); + ASSERT_KEY("softirq_ms"); + ASSERT_KEY("steal_ms"); + // Needs 2.6.24 - ASSERT_KEY("guest_ms"); + // Needs 2.6.33 - ASSERT_KEY("guest_nice_ms"); + ASSERT_KEY("ctxt"); + ASSERT_KEY("btime"); + ASSERT_KEY("processes"); + ASSERT_KEY("procs_running"); + ASSERT_KEY("procs_blocked"); +} + +TEST(FTDCProcStat, TestLocalNonExistentStat) { + std::vector keys{ + "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running", + }; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcStatFile("/proc/does_not_exist", keys, &builder)); +} + +TEST(FTDCProcMemInfo, TestMemInfo) { + std::vector keys{"Key1", "Key2", "Key3"}; + + // Normal case + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_KEY_AND_VALUE("Key2_kb", 456UL); + } + + // Space in key name + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey 2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_NO_KEY("Key2_kb"); + } + + // No newline + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB Key2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_NO_KEY("Key2_kb"); + } + + // Missing colon on first key + { + ASSERT_PARSE_MEMINFO(keys, "Key1 123 kB\nKey2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_KEY_AND_VALUE("Key2_kb", 456UL); + } + + // One token missing kB, HugePages is not size in kB + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey2: 456\nKey3: 789 kB\nKey4: 789 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_KEY_AND_VALUE("Key2", 456UL); + ASSERT_KEY_AND_VALUE("Key3_kb", 789UL); + ASSERT_NO_KEY("Key4_kb"); + } + + // Empty string + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcMemInfo(keys, "", &builder)); + } +} + +// Test we can parse the /proc/meminfo on this machine. Also assert we have the expected fields +// This tests is designed to exercise our parsing code on various Linuxes and fail +// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure +// otherwise. +TEST(FTDCProcMemInfo, TestLocalMemInfo) { + std::vector keys{ + "Active", + "Active(anon)", + "Active(file)", + "AnonHugePages", + "AnonPages", + "Bounce", + "Buffers", + "Cached", + "CmaFree", + "CmaTotal", + "CommitLimit", + "Committed_AS", + "Dirty", + "HardwareCorrupted", + "Inactive", + "Inactive(anon)", + "Inactive(file)", + "KernelStack", + "Mapped", + "MemAvailable", + "MemFree", + "MemTotal", + "Mlocked", + "NFS_Unstable", + "PageTables", + "SReclaimable", + "SUnreclaim", + "Shmem", + "Slab", + "SwapCached", + "SwapFree", + "SwapTotal", + "Unevictable", + "VmallocChunk", + "VmallocTotal", + "VmallocUsed", + "Writeback", + "WritebackTmp", + }; + + BSONObjBuilder builder; + + ASSERT_OK(procparser::parseProcMemInfoFile("/proc/meminfo", keys, &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + log() << "OBJ:" << obj; + ASSERT_KEY("MemTotal_kb"); + ASSERT_KEY("MemFree_kb"); + // Needs in 3.15+ - ASSERT_KEY("MemAvailable_kb"); + ASSERT_KEY("Buffers_kb"); + ASSERT_KEY("Cached_kb"); + ASSERT_KEY("SwapCached_kb"); + ASSERT_KEY("Active_kb"); + ASSERT_KEY("Inactive_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Active(anon)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Inactive(anon)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Active(file)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Inactive(file)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Unevictable_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Mlocked_kb"); + ASSERT_KEY("SwapTotal_kb"); + ASSERT_KEY("SwapFree_kb"); + ASSERT_KEY("Dirty_kb"); + ASSERT_KEY("Writeback_kb"); + ASSERT_KEY("AnonPages_kb"); + ASSERT_KEY("Mapped_kb"); + // Needs 2.6.32+ - ASSERT_KEY("Shmem_kb"); + ASSERT_KEY("Slab_kb"); + // Needs 2.6.19+ - ASSERT_KEY("SReclaimable_kb"); + // Needs 2.6.19+ - ASSERT_KEY("SUnreclaim_kb"); + // Needs 2.6.32+ - ASSERT_KEY("KernelStack_kb"); + ASSERT_KEY("PageTables_kb"); + ASSERT_KEY("NFS_Unstable_kb"); + ASSERT_KEY("Bounce_kb"); + // Needs 2.6.19+ - ASSERT_KEY("WritebackTmp_kb"); + ASSERT_KEY("CommitLimit_kb"); + ASSERT_KEY("Committed_AS_kb"); + ASSERT_KEY("VmallocTotal_kb"); + ASSERT_KEY("VmallocUsed_kb"); + ASSERT_KEY("VmallocChunk_kb"); + // Needs CONFIG_MEMORY_FAILURE & 2.6.32+ ASSERT_KEY("HardwareCorrupted_kb"); + // Needs CONFIG_TRANSPARENT_HUGEPAGE - ASSERT_KEY("AnonHugePages_kb"); + // Needs CONFIG_CMA & 3.19+ - ASSERT_KEY("CmaTotal_kb"); + // Needs CONFIG_CMA & 3.19+ - ASSERT_KEY("CmaFree_kb"); +} + + +TEST(FTDCProcMemInfo, TestLocalNonExistentMemInfo) { + std::vector keys{}; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcMemInfoFile("/proc/does_not_exist", keys, &builder)); +} + + +TEST(FTDCProcDiskStats, TestDiskStats) { + std::vector disks{"dm-1", "sda", "sdb"}; + + // Normal case including high device major numbers. + { + ASSERT_PARSE_DISKSTATS( + disks, + " 8 0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 " + "2554160\n" + " 8 1 sda1 138 37 8642 315 3 0 18 14 0 292 329\n" + " 8 2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 " + "2550739\n" + " 8 16 sdb 12707 3876 1525418 57507 997 3561 297576 97976 0 37870 155619\n" + " 8 17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468\n" + " 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0\n" + "2253 0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275\n" + "2253 1 dm-1 109 0 4584 226 0 0 0 0 0 172 226"); + ASSERT_KEY_AND_VALUE("sda.reads", 120611UL); + ASSERT_KEY_AND_VALUE("sda.writes", 349797UL); + ASSERT_KEY_AND_VALUE("sda.io_queued_ms", 2554160UL); + ASSERT_KEY_AND_VALUE("sdb.reads", 12707UL); + ASSERT_KEY_AND_VALUE("sdb.writes", 997UL); + ASSERT_KEY_AND_VALUE("sdb.io_queued_ms", 155619UL); + ASSERT_KEY_AND_VALUE("dm-1.reads", 109UL); + ASSERT_KEY_AND_VALUE("dm-1.writes", 0UL); + ASSERT_KEY_AND_VALUE("dm-1.io_queued_ms", 226UL); + } + + // Exclude a block device without any activity + { + ASSERT_PARSE_DISKSTATS( + disks, + " 8 0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 " + "2554160\n" + " 8 1 sda1 138 37 8642 315 3 0 18 14 0 292 329\n" + " 8 2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 " + "2550739\n" + " 8 16 sdb 0 0 0 0 0 0 0 0 0 0 0\n" + " 8 17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468\n" + " 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0\n" + "2253 0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275\n" + "2253 1 dm-1 109 0 4584 226 0 0 0 0 0 172 226"); + ASSERT_KEY_AND_VALUE("sda.reads", 120611UL); + ASSERT_KEY_AND_VALUE("sda.writes", 349797UL); + ASSERT_KEY_AND_VALUE("sda.io_queued_ms", 2554160UL); + ASSERT_NO_KEY("sdb.reads"); + ASSERT_NO_KEY("sdb.writes"); + ASSERT_NO_KEY("sdb.io_queued_ms"); + ASSERT_KEY_AND_VALUE("dm-1.reads", 109UL); + ASSERT_KEY_AND_VALUE("dm-1.writes", 0UL); + ASSERT_KEY_AND_VALUE("dm-1.io_queued_ms", 226UL); + } + + + // Strings with less numbers + { ASSERT_PARSE_DISKSTATS(disks, "8 0 sda 120611 33630 6297628 96550 349797 "); } + + // Strings with no numbers + { ASSERT_PARSE_DISKSTATS(disks, "8 0 sda"); } + + // Strings that are too short + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "8 0", &builder)); + ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "8", &builder)); + ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "", &builder)); + } +} + +TEST(FTDCProcDiskStats, TestLocalNonExistentStat) { + std::vector disks{"dm-1", "sda", "sdb"}; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcDiskStatsFile("/proc/does_not_exist", disks, &builder)); +} + +TEST(FTDCProcDiskStats, TestFindBadPhysicalDiskPaths) { + // Validate nothing goes wrong when we check a non-existent path. + { + auto disks = procparser::findPhysicalDisks("/proc/does_not_exist"); + ASSERT_EQUALS(0UL, disks.size()); + } + + // Validate nothing goes wrong when we check a path we do not have permission. + { + auto disks = procparser::findPhysicalDisks("/sys/kernel/debug"); + ASSERT_EQUALS(0UL, disks.size()); + } +} + +// Test we can parse the /proc/diskstats on this machine. Also assert we have the expected fields +// This tests is designed to exercise our parsing code on various Linuxes and fail +// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure +// otherwise. +TEST(FTDCProcDiskStats, TestLocalDiskStats) { + auto disks = procparser::findPhysicalDisks("/sys/block"); + + std::vector disks2; + for (const auto& disk : disks) { + log() << "DISK:" << disk; + disks2.emplace_back(disk); + } + + ASSERT_NOT_EQUALS(0UL, disks.size()); + + BSONObjBuilder builder; + + ASSERT_OK(procparser::parseProcDiskStatsFile("/proc/diskstats", disks2, &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toNestedStringMap(obj); + log() << "OBJ:" << obj; + + bool foundDisk = false; + + for (const auto& disk : disks) { + std::string prefix(disk); + prefix += "."; + + auto reads = prefix + "reads"; + auto io_queued_ms = prefix + "io_queued_ms"; + + // Make sure that if have the first field, then we have the last field. + if (stringMap.find(reads) != stringMap.end()) { + foundDisk = true; + if (stringMap.find(io_queued_ms) == stringMap.end()) { + FAIL(std::string("Inconsistency for ") + disk); + } + } + } + + if (!foundDisk) { + FAIL("Did not find any interesting disks on this machine."); + } +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/util/signal_handlers_synchronous_test.cpp b/src/mongo/util/signal_handlers_synchronous_test.cpp index f4104bafa16..c3623dde608 100644 --- a/src/mongo/util/signal_handlers_synchronous_test.cpp +++ b/src/mongo/util/signal_handlers_synchronous_test.cpp @@ -59,7 +59,6 @@ IGNORED_SIGNAL(SIGHUP) IGNORED_SIGNAL(SIGPIPE) FATAL_SIGNAL(SIGQUIT) FATAL_SIGNAL(SIGABRT) -FATAL_SIGNAL(SIGFPE) FATAL_SIGNAL(SIGILL) #if not defined(__has_feature) @@ -70,6 +69,7 @@ FATAL_SIGNAL(SIGILL) // These signals trip the leak sanitizer FATAL_SIGNAL(SIGSEGV) FATAL_SIGNAL(SIGBUS) +FATAL_SIGNAL(SIGFPE) #endif DEATH_TEST(FatalTerminateTest, diff --git a/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp b/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp index 54ebbf3e091..eb117d529ee 100644 --- a/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp +++ b/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp @@ -92,7 +92,10 @@ namespace boost } else { - BOOST_VERIFY(false); + // If we attempted to set the data to a null pointer, don't flag an error. + // This happens at shutdown, where boost has a race between thread and + // process exit handlers. + BOOST_VERIFY(!new_data); //boost::throw_exception(thread_resource_error()); } } diff --git a/src/third_party/mozjs-38/extract.sh b/src/third_party/mozjs-38/extract.sh index 101bd56e27c..e955be36387 100755 --- a/src/third_party/mozjs-38/extract.sh +++ b/src/third_party/mozjs-38/extract.sh @@ -182,6 +182,7 @@ cp extract/mfbt/MemoryReporting.h include/mozilla cp extract/mfbt/Move.h include/mozilla cp extract/mfbt/NullPtr.h include/mozilla cp extract/mfbt/NumericLimits.h include/mozilla +cp extract/mfbt/Opaque.h include/mozilla cp extract/mfbt/Pair.h include/mozilla cp extract/mfbt/PodOperations.h include/mozilla cp extract/mfbt/Poison.h include/mozilla diff --git a/src/third_party/mozjs-38/extract/js/public/HashTable.h b/src/third_party/mozjs-38/extract/js/public/HashTable.h index 5fb5012c4a8..36f37d6852a 100644 --- a/src/third_party/mozjs-38/extract/js/public/HashTable.h +++ b/src/third_party/mozjs-38/extract/js/public/HashTable.h @@ -13,6 +13,7 @@ #include "mozilla/Casting.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Move.h" +#include "mozilla/Opaque.h" #include "mozilla/PodOperations.h" #include "mozilla/ReentrancyGuard.h" #include "mozilla/TemplateLib.h" @@ -32,6 +33,8 @@ namespace detail { /*****************************************************************************/ +using Generation = mozilla::Opaque; + // A JS-friendly, STL-like container providing a hash-based map from keys to // values. In particular, HashMap calls constructors and destructors of all // objects added so non-PODs may be used safely. @@ -205,7 +208,9 @@ class HashMap // If |generation()| is the same before and after a HashMap operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -436,7 +441,9 @@ class HashSet // If |generation()| is the same before and after a HashSet operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -771,7 +778,7 @@ class HashTable : private AllocPolicy Entry* entry_; #ifdef JS_DEBUG const HashTable* table_; - uint32_t generation; + Generation generation; #endif protected: @@ -878,7 +885,7 @@ class HashTable : private AllocPolicy #ifdef JS_DEBUG const HashTable* table_; uint64_t mutationCount; - uint32_t generation; + Generation generation; bool validEntry; #endif @@ -1017,8 +1024,8 @@ class HashTable : private AllocPolicy static const size_t CAP_BITS = 24; public: - Entry* table; // entry storage - uint32_t gen; // entry storage generation number + uint64_t gen; // entry storage generation number + Entry* table; // entry storage uint32_t entryCount; // number of entries in table uint32_t removedCount:CAP_BITS; // removed entry sentinels in table uint32_t hashShift:8; // multiplicative hash shift @@ -1102,8 +1109,8 @@ class HashTable : private AllocPolicy public: explicit HashTable(AllocPolicy ap) : AllocPolicy(ap) - , table(nullptr) , gen(0) + , table(nullptr) , entryCount(0) , removedCount(0) , hashShift(sHashBits) @@ -1529,10 +1536,10 @@ class HashTable : private AllocPolicy return JS_BIT(sHashBits - hashShift); } - uint32_t generation() const + Generation generation() const { MOZ_ASSERT(table); - return gen; + return Generation(gen); } size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const diff --git a/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp b/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp index 97ac07a65d7..aec5913b352 100644 --- a/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp @@ -3464,12 +3464,12 @@ CodeGenerator::visitCallDirectEvalV(LCallDirectEvalV* lir) callVM(DirectEvalValueInfo, lir); } +// Registers safe for use before generatePrologue(). +static const uint32_t EntryTempMask = Registers::TempMask & ~(1 << OsrFrameReg.code()); + void CodeGenerator::generateArgumentsChecks(bool bailout) { - // Registers safe for use before generatePrologue(). - static const uint32_t EntryTempMask = Registers::TempMask & ~(1 << OsrFrameReg.code()); - // This function can be used the normal way to check the argument types, // before entering the function and bailout when arguments don't match. // For debug purpose, this is can also be used to force/check that the diff --git a/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp b/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp index eb367e132ef..424c2dd8e0a 100644 --- a/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp @@ -438,6 +438,7 @@ ValueNumberer::fixupOSROnlyLoop(MBasicBlock* block, MBasicBlock* backedge) graph_.insertBlockBefore(block, fake); fake->setImmediateDominator(fake); fake->addNumDominated(1); + fake->setDomIndex(fake->id()); // Create zero-input phis to use as inputs for any phis in |block|. // Again, this is a little odd, but it's the least-odd thing we can do diff --git a/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h b/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h index ef1ec17d766..77ed4382749 100644 --- a/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h +++ b/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h @@ -131,12 +131,6 @@ class Assembler : public AssemblerShared static void ToggleCall(CodeLocationLabel, bool) { MOZ_CRASH(); } static uintptr_t GetPointer(uint8_t*) { MOZ_CRASH(); } - - void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, - const Disassembler::HeapAccess &heapAccess) - { - MOZ_CRASH(); - } }; class Operand diff --git a/src/third_party/mozjs-38/extract/js/src/jsapi.h b/src/third_party/mozjs-38/extract/js/src/jsapi.h index 2b279b52092..0d8cd52b12b 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsapi.h +++ b/src/third_party/mozjs-38/extract/js/src/jsapi.h @@ -275,10 +275,6 @@ class AutoHashMapRooter : protected AutoGCRooter return map.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return map.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { @@ -390,10 +386,6 @@ class AutoHashSetRooter : protected AutoGCRooter return set.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return set.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { diff --git a/src/third_party/mozjs-38/extract/js/src/jscntxt.h b/src/third_party/mozjs-38/extract/js/src/jscntxt.h index 7a46270d7c9..f5e38d3b56a 100644 --- a/src/third_party/mozjs-38/extract/js/src/jscntxt.h +++ b/src/third_party/mozjs-38/extract/js/src/jscntxt.h @@ -35,11 +35,11 @@ typedef HashSet ShapeSet; /* Detects cycles when traversing an object graph. */ class AutoCycleDetector { + Generation hashsetGenerationAtInit; JSContext* cx; RootedObject obj; - bool cyclic; - uint32_t hashsetGenerationAtInit; ObjectSet::AddPtr hashsetAddPointer; + bool cyclic; MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER public: diff --git a/src/third_party/mozjs-38/extract/js/src/jsobj.cpp b/src/third_party/mozjs-38/extract/js/src/jsobj.cpp index f086030580d..2c66de6216b 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsobj.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jsobj.cpp @@ -1579,8 +1579,9 @@ CreateThisForFunctionWithGroup(JSContext* cx, HandleObjectGroup group, HandleObj if (!res) return nullptr; - if (newKind != SingletonObject) - newScript->registerNewObject(res); + // Make sure group->newScript is still there. + if (newKind != SingletonObject && group->newScript()) + group->newScript()->registerNewObject(res); return res; } diff --git a/src/third_party/mozjs-38/extract/js/src/jsscript.cpp b/src/third_party/mozjs-38/extract/js/src/jsscript.cpp index 31724b821e4..dfc4079bc1e 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsscript.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jsscript.cpp @@ -116,7 +116,10 @@ Bindings::initWithTemporaryStorage(ExclusiveContext* cx, InternalBindingsHandle // any time, such accesses are mediated by DebugScopeProxy (see // DebugScopeProxy::handleUnaliasedAccess). uint32_t nslots = CallObject::RESERVED_SLOTS; - uint32_t aliasedBodyLevelLexicalBegin = UINT16_MAX; + + // Unless there are aliased body-level lexical bindings at all, set the + // begin index to an impossible slot number. + uint32_t aliasedBodyLevelLexicalBegin = LOCALNO_LIMIT; for (BindingIter bi(self); bi; bi++) { if (bi->aliased()) { // Per ES6, lexical bindings cannot be accessed until diff --git a/src/third_party/mozjs-38/extract/js/src/jsscript.h b/src/third_party/mozjs-38/extract/js/src/jsscript.h index 7a6bff02a68..98676598cd8 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsscript.h +++ b/src/third_party/mozjs-38/extract/js/src/jsscript.h @@ -206,8 +206,8 @@ class Bindings uint16_t numArgs_; uint16_t numBlockScoped_; uint16_t numBodyLevelLexicals_; - uint16_t aliasedBodyLevelLexicalBegin_; uint16_t numUnaliasedBodyLevelLexicals_; + uint32_t aliasedBodyLevelLexicalBegin_; uint32_t numVars_; uint32_t numUnaliasedVars_; diff --git a/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp b/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp index 4de64410dc9..1861204c8aa 100644 --- a/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp @@ -27,15 +27,15 @@ namespace { class AutoEntryHolder { typedef WatchpointMap::Map Map; + Generation gen; Map& map; Map::Ptr p; - uint32_t gen; RootedObject obj; RootedId id; public: AutoEntryHolder(JSContext* cx, Map& map, Map::Ptr p) - : map(map), p(p), gen(map.generation()), obj(cx, p->key().object), id(cx, p->key().id) + : gen(map.generation()), map(map), p(p), obj(cx, p->key().object), id(cx, p->key().id) { MOZ_ASSERT(!p->value().held); p->value().held = true; diff --git a/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h b/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h index 7e7df5d36e0..50d5c0bda65 100644 --- a/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h +++ b/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h @@ -4,16 +4,16 @@ namespace js { namespace selfhosted { - static const unsigned char data[] = { 120, 156, 237, 125, 107, 119, 219, 198, 174, 232, 247, 254, 10, 70, 235, 158, 70, 222, 150, 124, 108, 39, 237, 110, 157, 56, 93, 169, 243, 168, 79, 147, 56, 55, 118, 219, 221, 237, 235, 147, 69, 75, 148, 68, 75, 34, 85, 146, 178, 45, 55, 254, 239, 119, 48, 79, 96, 30, 20, 101, 59, 143, 221, 189, 181, 86, 98, 105, 136, 193, 96, 48, 0, 6, 192, 60, 120, 30, 23, 81, 89, 245, 223, 63, 45, 138, 120, 241, 62, 205, 250, 201, 229, 193, 32, 218, 141, 248, 239, 125, 241, 243, 209, 87, 231, 18, 234, 176, 42, 210, 108, 248, 190, 156, 159, 150, 252, 27, 3, 180, 139, 12, 240, 111, 73, 60, 126, 29, 207, 24, 140, 252, 134, 241, 228, 179, 253, 42, 41, 226, 42, 205, 51, 142, 4, 253, 126, 244, 213, 96, 158, 245, 248, 147, 87, 105, 89, 181, 215, 162, 63, 191, 138, 216, 167, 26, 165, 229, 198, 36, 201, 134, 213, 136, 85, 217, 124, 244, 213, 245, 87, 240, 96, 146, 84, 28, 238, 109, 145, 87, 57, 123, 0, 248, 15, 78, 207, 146, 94, 245, 190, 87, 36, 113, 149, 180, 179, 249, 100, 178, 246, 136, 129, 106, 176, 13, 211, 81, 167, 243, 20, 240, 44, 79, 51, 2, 5, 5, 20, 100, 54, 47, 71, 4, 4, 10, 40, 72, 57, 73, 123, 9, 129, 225, 37, 22, 80, 94, 84, 20, 134, 21, 0, 200, 235, 120, 156, 236, 229, 25, 227, 239, 156, 177, 229, 116, 146, 180, 161, 82, 199, 84, 93, 3, 94, 104, 166, 189, 75, 122, 121, 209, 215, 108, 43, 146, 106, 94, 100, 97, 182, 92, 127, 229, 226, 23, 40, 58, 209, 159, 215, 107, 104, 52, 126, 138, 75, 214, 220, 44, 41, 170, 69, 59, 239, 68, 51, 171, 133, 89, 196, 88, 149, 19, 82, 142, 242, 31, 243, 124, 146, 196, 89, 251, 220, 2, 190, 119, 239, 220, 130, 124, 51, 159, 158, 38, 133, 3, 184, 78, 225, 246, 70, 73, 111, 44, 250, 177, 151, 39, 69, 143, 147, 171, 235, 164, 131, 168, 125, 30, 237, 238, 238, 70, 115, 54, 152, 131, 52, 75, 250, 209, 135, 15, 145, 40, 226, 253, 229, 96, 240, 57, 26, 21, 249, 197, 243, 162, 200, 139, 246, 214, 118, 135, 181, 47, 4, 153, 225, 234, 68, 173, 156, 55, 208, 90, 179, 72, 124, 197, 133, 207, 52, 199, 240, 178, 210, 253, 172, 74, 134, 156, 242, 71, 136, 136, 199, 76, 68, 77, 107, 178, 51, 155, 143, 236, 33, 121, 29, 87, 163, 247, 211, 148, 49, 168, 19, 109, 94, 110, 13, 240, 135, 54, 127, 24, 79, 147, 95, 227, 201, 60, 249, 103, 82, 228, 237, 203, 78, 180, 176, 56, 117, 201, 123, 185, 128, 14, 183, 47, 163, 123, 236, 251, 101, 244, 245, 215, 172, 0, 190, 46, 40, 178, 151, 73, 245, 58, 169, 70, 121, 191, 125, 208, 137, 222, 42, 68, 130, 58, 80, 82, 0, 228, 157, 19, 172, 110, 31, 172, 29, 191, 61, 49, 221, 19, 143, 109, 54, 235, 82, 202, 105, 73, 158, 6, 53, 104, 238, 237, 151, 123, 241, 100, 18, 195, 24, 66, 229, 53, 239, 240, 124, 223, 137, 170, 197, 44, 201, 7, 188, 129, 53, 194, 66, 40, 33, 29, 219, 215, 34, 250, 115, 178, 104, 199, 197, 112, 62, 77, 178, 74, 15, 24, 235, 26, 224, 98, 93, 147, 40, 21, 4, 193, 42, 64, 88, 71, 90, 194, 166, 181, 160, 119, 168, 112, 49, 61, 205, 39, 45, 155, 161, 194, 128, 49, 130, 153, 244, 116, 162, 41, 103, 47, 22, 76, 213, 148, 49, 98, 12, 217, 150, 233, 178, 168, 193, 72, 51, 131, 195, 49, 129, 152, 164, 18, 247, 154, 25, 34, 85, 196, 42, 244, 24, 19, 95, 72, 74, 218, 2, 77, 39, 98, 117, 215, 8, 171, 229, 80, 106, 84, 94, 102, 63, 252, 30, 233, 130, 1, 37, 236, 81, 197, 84, 60, 103, 73, 47, 77, 74, 109, 71, 20, 31, 216, 160, 199, 243, 73, 133, 202, 241, 96, 236, 49, 234, 25, 216, 70, 207, 60, 54, 52, 239, 81, 17, 83, 245, 16, 37, 46, 114, 81, 251, 154, 136, 24, 34, 105, 111, 13, 99, 193, 54, 96, 179, 19, 61, 99, 86, 111, 58, 75, 39, 201, 211, 98, 216, 222, 234, 68, 123, 170, 223, 215, 184, 247, 123, 164, 219, 120, 158, 108, 151, 73, 92, 244, 70, 207, 39, 9, 12, 115, 132, 251, 121, 128, 149, 9, 38, 50, 52, 144, 76, 26, 216, 211, 118, 251, 64, 202, 197, 90, 244, 228, 201, 19, 102, 60, 12, 35, 56, 196, 174, 207, 160, 116, 183, 12, 30, 192, 226, 200, 216, 147, 104, 43, 250, 1, 25, 41, 13, 112, 188, 117, 178, 22, 237, 40, 139, 4, 173, 100, 209, 147, 93, 32, 166, 182, 145, 177, 5, 143, 72, 26, 179, 246, 51, 241, 56, 153, 148, 9, 226, 51, 60, 129, 62, 172, 171, 231, 10, 197, 56, 122, 140, 49, 40, 216, 77, 123, 20, 153, 86, 199, 189, 113, 210, 231, 220, 102, 198, 8, 15, 226, 128, 169, 64, 251, 81, 4, 168, 88, 27, 236, 203, 250, 58, 126, 172, 80, 28, 28, 143, 79, 56, 19, 201, 32, 209, 182, 81, 159, 199, 134, 80, 65, 198, 181, 221, 169, 134, 237, 142, 97, 86, 60, 0, 67, 124, 107, 10, 176, 20, 194, 152, 216, 98, 120, 88, 49, 255, 169, 167, 132, 113, 194, 125, 4, 191, 72, 122, 205, 209, 99, 108, 140, 144, 106, 60, 248, 142, 77, 77, 157, 232, 62, 111, 67, 121, 78, 247, 145, 0, 15, 138, 124, 202, 91, 245, 11, 224, 54, 19, 64, 35, 118, 219, 39, 70, 234, 100, 95, 136, 249, 194, 10, 213, 137, 60, 157, 232, 152, 246, 214, 92, 30, 188, 138, 203, 234, 95, 88, 29, 1, 117, 55, 10, 234, 155, 122, 190, 230, 168, 150, 174, 196, 165, 84, 128, 63, 182, 181, 147, 232, 32, 0, 250, 116, 183, 177, 190, 129, 246, 179, 191, 221, 238, 167, 85, 183, 218, 102, 63, 185, 182, 97, 121, 251, 72, 26, 55, 49, 77, 120, 181, 238, 81, 184, 1, 166, 124, 100, 232, 124, 138, 202, 52, 242, 145, 159, 229, 142, 166, 64, 7, 215, 30, 145, 231, 245, 218, 98, 55, 74, 36, 149, 112, 216, 181, 1, 136, 179, 55, 176, 3, 207, 207, 147, 98, 209, 6, 172, 167, 76, 146, 7, 217, 29, 170, 191, 215, 143, 219, 108, 48, 144, 51, 136, 213, 192, 141, 220, 72, 128, 186, 251, 212, 61, 211, 158, 176, 33, 58, 232, 15, 19, 79, 133, 53, 128, 171, 152, 174, 28, 133, 45, 16, 182, 59, 204, 236, 156, 231, 105, 95, 25, 101, 174, 104, 220, 244, 68, 66, 211, 2, 19, 156, 81, 55, 159, 34, 222, 35, 3, 106, 232, 99, 222, 101, 135, 235, 102, 39, 26, 179, 47, 107, 65, 133, 28, 196, 76, 24, 107, 149, 146, 249, 116, 73, 72, 45, 197, 240, 11, 169, 241, 8, 65, 64, 27, 183, 27, 12, 226, 157, 14, 221, 214, 138, 67, 231, 153, 74, 241, 208, 5, 117, 137, 179, 67, 105, 17, 25, 12, 143, 234, 28, 230, 211, 228, 139, 213, 156, 146, 17, 247, 151, 86, 156, 219, 234, 141, 208, 138, 26, 181, 145, 138, 229, 215, 27, 62, 246, 31, 67, 109, 238, 114, 220, 62, 149, 214, 0, 51, 154, 42, 205, 139, 188, 120, 30, 247, 70, 95, 172, 222, 12, 4, 125, 127, 89, 213, 105, 168, 54, 181, 154, 161, 40, 178, 199, 246, 117, 60, 251, 98, 199, 117, 26, 207, 190, 208, 49, 133, 154, 79, 89, 205, 55, 201, 197, 179, 36, 43, 19, 17, 75, 64, 126, 225, 174, 198, 28, 42, 179, 254, 207, 146, 62, 207, 139, 218, 137, 176, 38, 82, 0, 159, 95, 178, 50, 30, 36, 111, 231, 149, 116, 44, 203, 246, 83, 14, 135, 80, 215, 203, 205, 211, 144, 53, 5, 193, 249, 24, 198, 244, 14, 7, 253, 83, 217, 82, 198, 138, 198, 254, 7, 231, 157, 50, 168, 31, 131, 127, 119, 108, 12, 239, 152, 135, 46, 243, 36, 47, 154, 50, 240, 93, 210, 159, 247, 190, 92, 23, 174, 224, 228, 125, 116, 171, 133, 18, 137, 240, 51, 238, 245, 230, 211, 249, 36, 38, 25, 102, 159, 85, 195, 86, 6, 85, 34, 33, 243, 86, 40, 100, 14, 164, 135, 108, 206, 124, 139, 12, 202, 178, 124, 139, 75, 7, 179, 100, 235, 235, 39, 200, 36, 217, 84, 104, 22, 188, 45, 146, 18, 178, 16, 187, 118, 56, 5, 159, 6, 9, 76, 69, 159, 223, 254, 6, 169, 59, 121, 228, 133, 67, 228, 80, 47, 149, 0, 173, 175, 251, 31, 156, 22, 73, 60, 118, 31, 93, 127, 21, 254, 197, 101, 75, 181, 106, 123, 204, 161, 241, 192, 6, 126, 9, 139, 194, 172, 161, 44, 49, 2, 218, 70, 15, 154, 59, 38, 68, 118, 253, 230, 82, 234, 188, 109, 32, 238, 192, 88, 222, 173, 186, 122, 109, 101, 48, 119, 165, 49, 6, 231, 21, 209, 111, 159, 101, 196, 22, 118, 205, 147, 242, 92, 25, 101, 208, 210, 190, 75, 135, 163, 234, 63, 230, 118, 215, 205, 94, 255, 181, 140, 110, 183, 123, 135, 70, 55, 152, 198, 86, 228, 125, 74, 155, 219, 237, 126, 26, 155, 187, 162, 213, 245, 242, 232, 139, 51, 186, 66, 253, 63, 154, 229, 229, 232, 191, 108, 243, 203, 73, 252, 8, 54, 56, 128, 215, 151, 126, 73, 179, 126, 123, 198, 24, 150, 246, 226, 42, 89, 209, 0, 155, 229, 56, 109, 136, 239, 54, 247, 194, 136, 11, 13, 160, 166, 185, 177, 249, 69, 53, 110, 18, 162, 91, 219, 112, 154, 70, 225, 28, 68, 133, 217, 212, 220, 56, 89, 75, 77, 33, 143, 187, 69, 45, 111, 218, 82, 173, 189, 113, 8, 207, 162, 16, 162, 213, 55, 224, 124, 237, 231, 63, 163, 254, 17, 71, 125, 201, 208, 6, 243, 209, 116, 81, 117, 233, 82, 234, 94, 62, 91, 252, 150, 178, 81, 202, 218, 21, 35, 60, 129, 165, 62, 246, 133, 253, 73, 50, 216, 16, 229, 108, 1, 186, 229, 8, 3, 72, 145, 48, 211, 158, 158, 39, 71, 188, 65, 2, 45, 104, 64, 176, 124, 95, 169, 85, 225, 113, 180, 201, 216, 108, 182, 240, 197, 151, 109, 177, 188, 78, 225, 58, 182, 71, 226, 251, 236, 208, 173, 128, 54, 6, 147, 51, 195, 132, 31, 2, 135, 8, 221, 156, 103, 214, 10, 49, 34, 92, 84, 88, 74, 247, 161, 224, 124, 3, 178, 67, 116, 75, 12, 126, 178, 159, 243, 17, 229, 227, 74, 246, 15, 254, 0, 240, 203, 155, 212, 45, 155, 110, 51, 92, 184, 211, 105, 22, 79, 80, 175, 161, 189, 165, 125, 102, 64, 183, 25, 40, 94, 157, 118, 183, 151, 207, 51, 181, 155, 87, 195, 11, 218, 186, 124, 96, 58, 210, 91, 230, 187, 119, 161, 22, 223, 87, 9, 35, 246, 24, 228, 237, 235, 175, 225, 255, 199, 178, 104, 93, 224, 163, 251, 48, 196, 232, 226, 231, 198, 249, 134, 15, 151, 90, 246, 159, 247, 225, 197, 136, 217, 22, 166, 220, 252, 9, 196, 28, 158, 165, 40, 142, 154, 59, 90, 14, 99, 14, 142, 171, 252, 132, 207, 4, 0, 100, 249, 158, 100, 138, 87, 159, 126, 50, 73, 152, 141, 230, 21, 45, 151, 152, 97, 176, 29, 208, 42, 183, 75, 56, 165, 184, 48, 176, 79, 228, 11, 235, 152, 157, 205, 168, 114, 187, 36, 212, 49, 108, 52, 15, 124, 19, 224, 100, 194, 76, 56, 159, 90, 75, 105, 10, 54, 63, 133, 193, 92, 110, 119, 198, 62, 163, 67, 250, 124, 3, 3, 116, 231, 182, 102, 117, 27, 226, 12, 253, 141, 108, 74, 51, 27, 130, 50, 79, 156, 26, 103, 106, 22, 251, 155, 162, 243, 128, 235, 228, 145, 152, 253, 172, 55, 153, 247, 147, 178, 29, 218, 70, 3, 18, 180, 170, 192, 200, 45, 241, 30, 23, 42, 188, 75, 14, 133, 197, 106, 163, 156, 25, 9, 188, 169, 71, 11, 148, 193, 42, 55, 157, 90, 155, 75, 51, 204, 129, 187, 220, 124, 42, 173, 137, 244, 143, 108, 215, 136, 110, 201, 183, 216, 10, 3, 228, 119, 139, 104, 32, 174, 19, 158, 75, 86, 233, 247, 248, 169, 13, 49, 146, 114, 3, 246, 211, 74, 236, 177, 30, 167, 32, 60, 25, 30, 59, 177, 71, 59, 233, 139, 209, 195, 3, 105, 246, 133, 91, 187, 200, 223, 36, 23, 4, 123, 91, 130, 137, 245, 177, 195, 164, 122, 199, 98, 250, 226, 60, 233, 31, 78, 114, 179, 141, 156, 59, 191, 180, 181, 134, 245, 182, 128, 228, 102, 160, 219, 162, 143, 13, 182, 161, 123, 184, 100, 120, 100, 29, 152, 88, 202, 209, 77, 79, 164, 169, 32, 247, 251, 108, 148, 211, 106, 97, 159, 179, 1, 61, 9, 215, 122, 147, 92, 154, 3, 77, 116, 95, 62, 87, 48, 56, 104, 192, 138, 40, 253, 252, 137, 103, 247, 59, 241, 204, 33, 182, 32, 213, 196, 17, 130, 253, 193, 111, 5, 95, 199, 236, 112, 202, 58, 181, 46, 78, 203, 161, 181, 69, 118, 192, 243, 148, 30, 147, 20, 49, 92, 47, 147, 74, 144, 254, 130, 169, 44, 25, 56, 222, 146, 206, 99, 114, 49, 147, 246, 69, 215, 244, 192, 111, 81, 177, 156, 254, 156, 114, 243, 173, 171, 48, 11, 241, 96, 59, 208, 214, 54, 177, 254, 229, 124, 2, 34, 255, 167, 176, 142, 40, 250, 233, 68, 253, 60, 99, 5, 92, 187, 162, 107, 99, 86, 4, 125, 79, 32, 25, 27, 91, 201, 88, 204, 120, 191, 156, 74, 242, 225, 0, 15, 58, 186, 99, 6, 11, 200, 217, 128, 134, 157, 36, 156, 28, 72, 1, 130, 57, 189, 164, 33, 65, 238, 186, 102, 25, 239, 130, 102, 217, 174, 101, 33, 37, 5, 231, 50, 122, 143, 143, 121, 245, 147, 6, 100, 56, 120, 183, 236, 156, 192, 44, 78, 11, 103, 145, 127, 27, 117, 31, 0, 142, 55, 97, 170, 74, 205, 102, 85, 253, 96, 235, 36, 64, 16, 161, 24, 64, 27, 80, 171, 172, 3, 169, 139, 90, 181, 42, 218, 58, 202, 45, 121, 201, 44, 64, 214, 200, 78, 72, 33, 23, 70, 204, 143, 203, 182, 13, 62, 227, 100, 116, 197, 217, 194, 154, 85, 69, 186, 10, 146, 45, 15, 146, 159, 147, 197, 10, 24, 182, 125, 169, 53, 166, 113, 92, 8, 74, 190, 37, 98, 144, 237, 34, 117, 130, 106, 79, 139, 225, 174, 215, 225, 132, 195, 61, 194, 28, 170, 18, 216, 82, 33, 142, 141, 114, 76, 252, 100, 154, 231, 92, 152, 2, 99, 65, 16, 206, 154, 240, 58, 141, 243, 156, 18, 154, 102, 75, 36, 189, 166, 48, 174, 170, 162, 132, 89, 255, 114, 115, 59, 250, 0, 127, 182, 196, 159, 135, 6, 102, 94, 50, 98, 246, 205, 68, 105, 206, 103, 73, 182, 120, 78, 104, 65, 55, 104, 189, 123, 161, 179, 76, 106, 163, 140, 125, 82, 137, 185, 151, 89, 114, 17, 237, 181, 225, 96, 192, 49, 210, 14, 107, 222, 198, 71, 207, 36, 65, 164, 101, 107, 199, 54, 242, 113, 224, 35, 61, 28, 176, 75, 190, 253, 53, 25, 155, 4, 64, 139, 36, 174, 141, 140, 79, 96, 52, 130, 161, 243, 24, 64, 120, 182, 39, 162, 193, 250, 102, 203, 131, 0, 170, 113, 35, 233, 91, 241, 120, 106, 206, 253, 122, 150, 35, 204, 54, 28, 92, 122, 237, 237, 139, 74, 97, 242, 230, 144, 243, 140, 225, 232, 158, 34, 37, 140, 63, 88, 199, 236, 64, 188, 180, 2, 116, 12, 114, 230, 60, 192, 128, 233, 223, 20, 255, 251, 103, 92, 0, 158, 197, 85, 172, 207, 210, 194, 118, 163, 245, 117, 178, 225, 168, 35, 4, 51, 176, 66, 129, 196, 151, 239, 148, 79, 199, 9, 246, 240, 184, 16, 132, 125, 117, 93, 199, 141, 241, 106, 197, 144, 59, 192, 59, 183, 222, 207, 69, 50, 201, 156, 84, 146, 77, 190, 241, 0, 140, 49, 247, 199, 22, 235, 67, 108, 175, 97, 186, 96, 53, 146, 60, 232, 12, 54, 163, 124, 219, 23, 144, 219, 103, 214, 244, 40, 157, 38, 47, 242, 98, 26, 87, 123, 113, 111, 36, 36, 236, 66, 31, 197, 70, 135, 168, 25, 9, 201, 251, 42, 127, 149, 179, 206, 36, 242, 156, 37, 54, 154, 151, 246, 54, 54, 48, 46, 188, 18, 151, 86, 56, 12, 129, 66, 49, 208, 28, 113, 112, 250, 125, 90, 190, 137, 223, 180, 47, 215, 156, 112, 171, 181, 159, 177, 170, 105, 159, 55, 221, 66, 98, 193, 73, 40, 253, 57, 228, 77, 146, 67, 222, 244, 228, 144, 1, 69, 62, 3, 18, 3, 40, 150, 167, 161, 93, 230, 161, 240, 81, 81, 71, 34, 119, 54, 35, 232, 54, 67, 246, 20, 106, 123, 134, 100, 131, 150, 133, 235, 195, 199, 116, 236, 40, 127, 38, 235, 29, 136, 178, 182, 124, 214, 137, 90, 113, 182, 104, 193, 159, 201, 164, 101, 89, 180, 38, 237, 51, 215, 164, 154, 188, 127, 70, 74, 85, 183, 59, 138, 2, 199, 2, 184, 216, 25, 166, 229, 205, 73, 153, 182, 131, 227, 219, 244, 243, 54, 253, 33, 177, 46, 175, 39, 224, 85, 109, 107, 0, 59, 145, 117, 198, 135, 104, 17, 252, 248, 143, 38, 125, 98, 77, 186, 59, 45, 2, 108, 45, 245, 183, 161, 30, 125, 66, 29, 186, 3, 253, 9, 245, 240, 139, 209, 32, 128, 251, 143, 6, 125, 66, 13, 170, 238, 116, 30, 2, 108, 45, 245, 183, 129, 6, 85, 159, 110, 22, 170, 238, 102, 6, 10, 245, 240, 115, 106, 16, 15, 102, 244, 93, 25, 107, 252, 50, 34, 46, 136, 167, 103, 40, 240, 165, 193, 17, 36, 95, 133, 82, 224, 29, 4, 172, 99, 252, 27, 239, 161, 68, 8, 223, 185, 150, 137, 238, 242, 248, 37, 158, 130, 99, 9, 247, 101, 192, 87, 40, 150, 69, 109, 241, 151, 74, 210, 15, 10, 43, 95, 233, 144, 100, 2, 160, 70, 56, 45, 135, 18, 223, 52, 41, 203, 120, 200, 81, 138, 194, 54, 255, 227, 32, 36, 184, 24, 200, 154, 234, 162, 38, 160, 213, 90, 195, 124, 101, 48, 10, 68, 97, 180, 32, 84, 87, 208, 207, 104, 61, 106, 237, 68, 45, 246, 135, 87, 199, 183, 140, 84, 113, 241, 50, 201, 80, 50, 149, 241, 200, 202, 167, 30, 206, 203, 89, 194, 168, 238, 19, 96, 55, 139, 26, 78, 191, 146, 138, 248, 185, 55, 157, 239, 164, 97, 73, 117, 127, 26, 22, 50, 147, 108, 128, 157, 238, 96, 217, 230, 43, 12, 46, 37, 44, 62, 155, 228, 101, 210, 175, 161, 40, 156, 248, 132, 48, 95, 229, 61, 85, 27, 26, 255, 126, 249, 110, 158, 101, 48, 178, 30, 212, 72, 94, 183, 191, 33, 26, 84, 21, 11, 55, 55, 13, 73, 182, 105, 66, 185, 47, 123, 125, 31, 66, 99, 181, 51, 230, 154, 177, 175, 234, 141, 162, 118, 226, 12, 205, 138, 125, 215, 160, 135, 73, 133, 161, 76, 103, 43, 232, 66, 164, 87, 203, 66, 98, 197, 123, 250, 23, 146, 43, 222, 159, 91, 10, 150, 96, 29, 67, 254, 153, 69, 135, 211, 241, 5, 203, 206, 59, 222, 133, 191, 144, 240, 136, 14, 221, 177, 89, 226, 237, 124, 42, 131, 196, 151, 132, 206, 249, 122, 126, 19, 2, 106, 133, 176, 16, 82, 216, 131, 158, 126, 89, 82, 248, 42, 25, 198, 189, 197, 178, 169, 209, 35, 86, 86, 69, 191, 96, 133, 132, 202, 170, 92, 43, 86, 30, 10, 91, 40, 114, 240, 210, 17, 226, 22, 31, 123, 114, 105, 100, 27, 161, 90, 46, 63, 62, 217, 185, 139, 105, 204, 35, 3, 43, 245, 235, 142, 165, 32, 52, 147, 125, 65, 98, 64, 38, 167, 149, 229, 192, 154, 150, 62, 225, 224, 251, 38, 162, 47, 108, 244, 121, 117, 216, 86, 83, 100, 241, 68, 199, 245, 143, 156, 255, 111, 206, 53, 63, 131, 144, 211, 73, 45, 229, 234, 28, 225, 165, 240, 31, 35, 168, 70, 66, 31, 45, 227, 66, 131, 45, 23, 159, 64, 1, 108, 217, 231, 164, 221, 88, 246, 5, 45, 162, 50, 33, 168, 78, 12, 116, 194, 6, 95, 153, 9, 207, 102, 12, 29, 154, 244, 97, 92, 219, 195, 36, 147, 226, 142, 55, 207, 12, 242, 162, 151, 232, 58, 73, 129, 45, 239, 88, 237, 19, 104, 101, 220, 188, 127, 69, 249, 102, 203, 139, 105, 64, 194, 251, 48, 85, 194, 66, 52, 71, 85, 97, 147, 66, 86, 108, 106, 42, 245, 212, 88, 32, 198, 84, 249, 211, 195, 189, 253, 253, 95, 216, 24, 22, 123, 49, 147, 162, 210, 189, 40, 85, 111, 54, 105, 181, 172, 101, 176, 84, 44, 131, 165, 209, 227, 72, 37, 162, 216, 47, 119, 45, 12, 238, 89, 45, 143, 83, 235, 64, 69, 43, 110, 193, 21, 178, 61, 200, 50, 245, 224, 91, 235, 170, 69, 45, 69, 207, 151, 156, 147, 183, 49, 87, 185, 38, 186, 19, 245, 144, 241, 144, 228, 174, 179, 170, 158, 36, 136, 217, 25, 33, 246, 236, 8, 153, 97, 99, 61, 121, 151, 12, 159, 95, 206, 202, 168, 230, 122, 101, 15, 248, 198, 60, 75, 123, 121, 63, 17, 73, 198, 231, 151, 85, 146, 149, 140, 210, 195, 228, 143, 121, 146, 245, 146, 119, 207, 35, 113, 87, 172, 191, 242, 36, 206, 134, 243, 120, 152, 28, 197, 195, 37, 144, 253, 249, 108, 194, 79, 37, 252, 26, 23, 105, 156, 85, 77, 193, 15, 25, 175, 38, 73, 149, 103, 75, 42, 164, 229, 111, 9, 99, 116, 94, 76, 147, 254, 222, 188, 40, 24, 241, 139, 61, 214, 175, 37, 213, 122, 18, 244, 89, 58, 76, 171, 18, 1, 107, 9, 27, 38, 213, 47, 75, 88, 100, 111, 222, 184, 17, 155, 63, 124, 192, 162, 211, 190, 217, 80, 89, 46, 253, 48, 185, 156, 189, 215, 247, 182, 190, 207, 242, 247, 37, 63, 20, 87, 182, 91, 221, 121, 187, 123, 28, 119, 175, 54, 187, 223, 159, 252, 185, 221, 249, 238, 122, 109, 189, 181, 70, 117, 171, 72, 166, 249, 121, 34, 59, 175, 155, 43, 101, 174, 14, 39, 161, 19, 253, 208, 104, 93, 3, 185, 106, 196, 90, 129, 81, 110, 129, 104, 155, 150, 248, 222, 95, 222, 193, 228, 50, 233, 225, 190, 45, 107, 185, 35, 83, 217, 44, 168, 187, 167, 110, 66, 70, 10, 47, 30, 214, 104, 110, 145, 204, 38, 49, 63, 125, 204, 33, 59, 168, 251, 199, 155, 39, 29, 72, 158, 25, 109, 94, 70, 140, 185, 58, 208, 222, 215, 42, 133, 73, 52, 66, 6, 134, 241, 237, 21, 86, 60, 50, 117, 46, 85, 83, 103, 170, 92, 90, 195, 12, 234, 211, 87, 111, 127, 130, 13, 8, 45, 16, 157, 167, 221, 127, 158, 160, 245, 129, 103, 251, 47, 247, 143, 248, 51, 144, 41, 244, 32, 158, 204, 70, 113, 54, 135, 147, 16, 173, 246, 15, 59, 144, 51, 20, 120, 214, 163, 214, 7, 248, 37, 106, 178, 95, 107, 45, 108, 181, 135, 243, 73, 92, 200, 74, 113, 81, 117, 39, 249, 217, 105, 156, 125, 232, 37, 147, 238, 48, 158, 79, 210, 114, 244, 33, 203, 187, 167, 249, 24, 254, 100, 139, 236, 195, 213, 168, 59, 156, 231, 139, 57, 124, 25, 197, 227, 113, 12, 95, 166, 105, 38, 255, 116, 179, 152, 127, 189, 100, 246, 103, 136, 155, 74, 11, 218, 88, 146, 117, 95, 254, 216, 205, 147, 254, 135, 180, 27, 79, 83, 246, 255, 105, 150, 177, 255, 229, 85, 198, 236, 91, 146, 229, 189, 17, 67, 195, 190, 178, 134, 216, 255, 227, 9, 147, 140, 28, 126, 79, 230, 151, 236, 255, 41, 252, 100, 127, 179, 248, 60, 62, 131, 47, 179, 11, 120, 88, 197, 57, 255, 127, 1, 255, 151, 243, 15, 229, 48, 235, 254, 248, 188, 251, 226, 157, 250, 246, 230, 21, 255, 182, 247, 83, 247, 217, 115, 76, 226, 176, 136, 179, 254, 32, 174, 70, 73, 145, 244, 17, 35, 13, 233, 138, 153, 232, 55, 70, 48, 43, 210, 115, 102, 78, 231, 101, 34, 107, 95, 178, 127, 198, 2, 108, 113, 11, 128, 43, 148, 202, 240, 162, 214, 244, 64, 125, 56, 126, 218, 253, 237, 247, 238, 63, 227, 238, 197, 162, 123, 117, 130, 235, 105, 101, 64, 245, 12, 174, 117, 94, 214, 133, 66, 45, 22, 172, 76, 90, 32, 140, 231, 92, 204, 19, 8, 11, 169, 240, 13, 171, 240, 193, 34, 139, 0, 60, 184, 94, 179, 196, 137, 210, 164, 69, 240, 207, 237, 107, 42, 133, 80, 21, 51, 162, 87, 164, 179, 202, 91, 243, 225, 181, 213, 115, 80, 27, 47, 228, 131, 107, 213, 109, 92, 182, 246, 231, 102, 103, 155, 224, 80, 122, 231, 39, 180, 99, 208, 168, 198, 96, 152, 127, 248, 96, 147, 101, 21, 0, 179, 236, 86, 170, 24, 40, 213, 237, 161, 129, 145, 29, 230, 152, 85, 153, 100, 31, 46, 35, 115, 141, 246, 149, 248, 152, 1, 216, 223, 16, 165, 82, 32, 72, 49, 18, 72, 142, 213, 195, 132, 35, 78, 98, 235, 127, 37, 35, 20, 213, 74, 210, 41, 6, 94, 68, 213, 4, 240, 254, 159, 22, 113, 47, 151, 155, 199, 21, 38, 81, 84, 143, 89, 253, 212, 158, 63, 153, 153, 126, 230, 120, 61, 75, 109, 181, 235, 40, 53, 50, 216, 110, 181, 207, 98, 181, 239, 88, 107, 237, 94, 161, 193, 241, 226, 7, 51, 210, 101, 158, 12, 146, 206, 86, 27, 160, 176, 100, 118, 233, 227, 32, 154, 191, 17, 184, 255, 247, 255, 182, 172, 122, 247, 236, 122, 107, 13, 100, 205, 231, 7, 55, 22, 56, 187, 114, 141, 192, 33, 191, 185, 185, 200, 161, 74, 171, 9, 29, 170, 248, 89, 196, 238, 166, 147, 149, 219, 1, 44, 97, 221, 182, 51, 121, 45, 23, 158, 245, 143, 45, 56, 36, 34, 90, 93, 116, 116, 117, 75, 120, 96, 245, 2, 42, 205, 11, 230, 246, 194, 230, 248, 180, 143, 156, 76, 203, 233, 247, 218, 106, 229, 211, 91, 190, 41, 74, 44, 73, 2, 171, 164, 12, 217, 81, 236, 157, 219, 18, 136, 206, 164, 57, 135, 222, 241, 155, 143, 224, 184, 97, 9, 103, 213, 141, 135, 222, 186, 236, 182, 92, 132, 230, 160, 7, 247, 144, 242, 178, 198, 233, 79, 245, 53, 215, 10, 101, 23, 112, 26, 114, 160, 58, 132, 19, 93, 116, 95, 248, 210, 80, 66, 191, 146, 201, 224, 133, 139, 4, 114, 188, 53, 216, 27, 57, 7, 230, 150, 58, 201, 118, 234, 17, 19, 65, 4, 48, 52, 80, 46, 41, 122, 180, 162, 175, 191, 198, 178, 184, 20, 3, 106, 92, 227, 160, 167, 196, 226, 44, 103, 177, 19, 147, 195, 171, 100, 153, 32, 46, 229, 51, 108, 162, 186, 80, 201, 22, 221, 90, 80, 146, 100, 242, 100, 20, 151, 7, 23, 153, 218, 140, 220, 225, 178, 206, 72, 120, 45, 54, 59, 151, 97, 73, 181, 0, 143, 5, 220, 9, 178, 87, 243, 83, 230, 201, 212, 201, 91, 201, 248, 84, 97, 105, 107, 161, 193, 77, 77, 172, 40, 3, 99, 158, 194, 18, 72, 245, 217, 40, 154, 195, 18, 79, 33, 59, 36, 192, 156, 116, 150, 40, 39, 47, 190, 129, 204, 22, 67, 13, 219, 187, 62, 124, 208, 24, 32, 227, 119, 217, 178, 146, 226, 214, 53, 68, 126, 132, 15, 237, 189, 83, 154, 168, 70, 89, 50, 1, 205, 98, 236, 53, 175, 247, 25, 69, 77, 244, 76, 32, 49, 71, 217, 224, 115, 109, 94, 201, 144, 114, 37, 222, 132, 158, 187, 244, 111, 223, 5, 253, 222, 141, 90, 171, 138, 225, 33, 71, 101, 36, 81, 162, 14, 210, 231, 214, 57, 22, 207, 220, 235, 178, 86, 33, 69, 70, 33, 205, 233, 176, 42, 40, 34, 54, 102, 69, 50, 72, 88, 44, 219, 127, 68, 234, 137, 17, 209, 178, 88, 87, 59, 21, 239, 214, 82, 210, 13, 66, 242, 167, 35, 36, 78, 191, 228, 91, 220, 70, 233, 160, 82, 196, 227, 133, 28, 77, 135, 125, 213, 193, 181, 103, 12, 141, 98, 105, 45, 67, 10, 65, 143, 16, 243, 189, 98, 176, 103, 109, 226, 19, 30, 243, 26, 187, 78, 144, 100, 120, 57, 157, 38, 89, 156, 240, 93, 179, 237, 4, 201, 150, 193, 97, 7, 241, 178, 190, 37, 150, 195, 200, 62, 239, 203, 61, 169, 239, 150, 61, 209, 184, 213, 101, 3, 169, 167, 179, 43, 180, 131, 111, 154, 163, 50, 128, 145, 217, 25, 134, 187, 96, 29, 237, 137, 205, 71, 248, 24, 62, 242, 55, 9, 154, 44, 36, 217, 211, 128, 160, 224, 181, 128, 120, 54, 150, 161, 234, 47, 34, 251, 210, 50, 83, 143, 199, 110, 235, 86, 73, 165, 187, 232, 168, 43, 32, 61, 53, 201, 130, 124, 112, 113, 52, 164, 161, 238, 152, 109, 190, 134, 58, 83, 117, 125, 23, 176, 226, 72, 191, 228, 239, 100, 108, 183, 168, 131, 164, 187, 131, 241, 57, 55, 25, 73, 172, 222, 54, 3, 237, 26, 204, 104, 200, 236, 123, 64, 112, 87, 237, 10, 214, 107, 78, 36, 160, 90, 100, 201, 39, 253, 195, 106, 49, 193, 94, 136, 178, 64, 176, 97, 133, 215, 109, 205, 226, 238, 219, 159, 97, 71, 36, 251, 242, 180, 136, 79, 225, 151, 56, 201, 221, 186, 26, 117, 247, 222, 192, 35, 246, 229, 167, 56, 43, 225, 151, 121, 244, 211, 207, 230, 81, 5, 191, 204, 163, 195, 151, 184, 22, 251, 101, 30, 29, 253, 134, 107, 177, 95, 95, 93, 227, 3, 76, 34, 81, 41, 146, 206, 100, 187, 184, 240, 39, 14, 6, 240, 230, 150, 119, 137, 124, 157, 101, 139, 39, 60, 157, 173, 222, 236, 201, 187, 121, 6, 219, 121, 45, 124, 200, 163, 111, 22, 53, 184, 14, 146, 67, 134, 229, 200, 45, 241, 254, 86, 247, 221, 106, 6, 209, 227, 199, 105, 58, 106, 170, 81, 175, 142, 115, 163, 221, 203, 39, 252, 54, 66, 181, 150, 43, 219, 79, 147, 114, 35, 62, 143, 83, 126, 70, 86, 112, 177, 108, 175, 41, 4, 212, 117, 206, 248, 246, 124, 177, 159, 249, 22, 104, 232, 198, 232, 149, 16, 73, 62, 120, 86, 68, 66, 195, 214, 96, 193, 98, 63, 184, 62, 215, 100, 245, 172, 102, 117, 175, 193, 186, 89, 221, 218, 96, 227, 0, 186, 117, 252, 191, 60, 135, 225, 132, 205, 126, 220, 109, 181, 178, 136, 181, 79, 188, 175, 83, 110, 205, 214, 0, 70, 239, 132, 9, 102, 34, 15, 41, 126, 103, 105, 187, 135, 196, 222, 64, 42, 67, 9, 211, 244, 131, 186, 152, 121, 73, 108, 183, 100, 140, 58, 136, 56, 202, 129, 184, 223, 63, 112, 181, 164, 108, 219, 2, 134, 25, 161, 213, 74, 159, 41, 65, 170, 203, 40, 65, 170, 251, 38, 158, 50, 225, 172, 209, 67, 251, 108, 43, 90, 212, 183, 154, 169, 91, 226, 167, 160, 88, 243, 69, 93, 39, 92, 178, 187, 119, 92, 103, 42, 40, 178, 147, 19, 58, 165, 133, 81, 73, 205, 198, 119, 82, 208, 11, 80, 173, 154, 225, 8, 154, 63, 231, 238, 223, 132, 14, 71, 240, 240, 140, 35, 75, 182, 3, 201, 163, 202, 132, 31, 92, 182, 31, 1, 82, 249, 246, 86, 140, 91, 189, 171, 213, 54, 180, 32, 0, 210, 250, 148, 40, 74, 166, 47, 54, 147, 100, 163, 89, 170, 246, 106, 102, 235, 28, 125, 248, 150, 32, 14, 104, 174, 224, 197, 111, 80, 62, 128, 163, 202, 116, 216, 245, 181, 185, 158, 131, 248, 129, 203, 55, 85, 213, 123, 138, 37, 10, 206, 122, 123, 173, 222, 243, 36, 158, 175, 213, 159, 208, 127, 176, 253, 173, 21, 168, 0, 9, 34, 202, 210, 54, 102, 108, 191, 26, 67, 211, 178, 100, 226, 230, 65, 92, 125, 243, 223, 116, 34, 26, 205, 194, 71, 180, 31, 154, 189, 93, 120, 158, 39, 96, 50, 164, 222, 89, 201, 65, 56, 103, 186, 91, 110, 251, 28, 146, 123, 224, 161, 56, 218, 127, 83, 19, 212, 35, 170, 241, 35, 51, 126, 79, 169, 238, 56, 10, 109, 82, 91, 56, 213, 132, 255, 151, 30, 116, 63, 133, 201, 86, 79, 143, 68, 222, 236, 59, 27, 188, 150, 67, 227, 176, 236, 130, 113, 72, 197, 99, 26, 6, 213, 231, 43, 39, 248, 213, 124, 26, 131, 29, 216, 168, 196, 165, 143, 225, 254, 119, 66, 227, 90, 79, 118, 163, 109, 190, 249, 73, 161, 63, 134, 210, 110, 180, 45, 222, 234, 8, 77, 17, 140, 252, 41, 171, 99, 48, 97, 238, 53, 200, 221, 160, 126, 144, 52, 41, 221, 98, 153, 231, 227, 249, 236, 53, 236, 246, 132, 183, 119, 58, 163, 90, 192, 222, 139, 178, 74, 250, 158, 121, 9, 37, 214, 140, 133, 177, 43, 168, 105, 68, 131, 89, 109, 216, 126, 52, 76, 157, 102, 251, 204, 43, 87, 72, 82, 97, 148, 128, 149, 22, 170, 240, 201, 67, 237, 147, 217, 212, 145, 137, 202, 109, 152, 215, 168, 221, 215, 99, 106, 59, 196, 52, 212, 27, 183, 85, 44, 114, 110, 34, 68, 111, 208, 163, 247, 49, 4, 244, 37, 124, 75, 139, 188, 86, 72, 179, 198, 59, 46, 10, 235, 196, 32, 243, 208, 235, 177, 238, 119, 187, 161, 9, 99, 214, 113, 51, 23, 218, 59, 216, 216, 84, 211, 132, 200, 193, 161, 230, 142, 55, 79, 106, 192, 213, 206, 36, 90, 103, 195, 186, 39, 10, 49, 223, 219, 80, 61, 164, 211, 198, 62, 197, 30, 184, 69, 212, 30, 109, 111, 144, 26, 216, 59, 73, 38, 130, 23, 105, 181, 186, 181, 144, 40, 87, 54, 54, 164, 117, 136, 161, 38, 231, 97, 69, 178, 107, 235, 227, 177, 240, 104, 146, 156, 179, 216, 95, 139, 1, 220, 34, 165, 4, 0, 238, 80, 193, 102, 109, 42, 232, 3, 175, 86, 212, 151, 92, 147, 116, 27, 131, 85, 240, 99, 165, 10, 26, 108, 248, 132, 119, 208, 50, 228, 63, 172, 222, 111, 82, 127, 231, 6, 124, 55, 68, 14, 114, 166, 253, 198, 154, 109, 224, 105, 215, 150, 245, 98, 195, 18, 66, 87, 186, 113, 2, 208, 77, 9, 138, 146, 87, 200, 226, 147, 220, 88, 216, 24, 57, 226, 93, 108, 132, 4, 220, 110, 109, 249, 178, 144, 174, 17, 204, 84, 18, 194, 177, 114, 29, 146, 108, 227, 10, 166, 88, 93, 146, 199, 100, 75, 51, 31, 13, 5, 94, 220, 154, 205, 242, 130, 13, 221, 115, 52, 16, 173, 238, 28, 239, 184, 243, 207, 179, 30, 145, 38, 116, 210, 249, 210, 113, 226, 147, 69, 8, 11, 153, 20, 45, 25, 2, 101, 209, 206, 27, 252, 104, 163, 135, 246, 197, 92, 201, 130, 84, 178, 208, 28, 179, 231, 86, 67, 234, 142, 59, 82, 147, 216, 92, 63, 203, 158, 246, 251, 105, 37, 89, 215, 242, 160, 124, 155, 151, 116, 62, 115, 132, 40, 40, 153, 168, 51, 111, 253, 62, 164, 72, 31, 235, 37, 111, 87, 63, 88, 85, 143, 35, 47, 17, 202, 85, 112, 207, 242, 11, 2, 90, 143, 182, 216, 40, 6, 196, 149, 174, 37, 235, 143, 13, 125, 108, 112, 161, 229, 131, 109, 55, 110, 112, 41, 81, 60, 208, 102, 70, 133, 110, 117, 109, 184, 235, 66, 2, 141, 24, 142, 6, 140, 36, 66, 208, 177, 26, 247, 44, 59, 41, 158, 233, 38, 194, 172, 37, 212, 96, 183, 208, 115, 193, 25, 254, 212, 75, 30, 79, 175, 131, 94, 173, 203, 239, 158, 235, 216, 212, 231, 218, 41, 245, 190, 127, 231, 246, 108, 107, 65, 80, 101, 223, 161, 81, 199, 174, 165, 204, 18, 24, 93, 132, 161, 151, 232, 92, 19, 125, 148, 115, 170, 146, 32, 249, 211, 178, 5, 64, 23, 1, 36, 234, 105, 75, 252, 138, 28, 193, 136, 215, 156, 94, 255, 233, 40, 170, 67, 200, 185, 168, 233, 142, 212, 57, 237, 84, 64, 152, 26, 155, 175, 16, 27, 197, 220, 194, 89, 22, 237, 218, 66, 230, 153, 79, 214, 119, 107, 218, 12, 198, 26, 98, 83, 130, 93, 13, 91, 14, 251, 230, 212, 34, 193, 115, 88, 131, 8, 21, 77, 8, 60, 70, 165, 83, 190, 53, 155, 176, 0, 182, 186, 13, 254, 32, 114, 234, 33, 145, 94, 172, 123, 217, 73, 73, 161, 174, 51, 245, 178, 157, 25, 63, 236, 91, 11, 31, 241, 80, 53, 167, 214, 25, 86, 139, 200, 27, 134, 224, 192, 162, 68, 249, 45, 118, 146, 178, 113, 26, 48, 28, 84, 219, 215, 31, 222, 46, 176, 246, 36, 13, 238, 40, 184, 94, 26, 46, 19, 85, 20, 92, 19, 25, 53, 135, 200, 64, 42, 77, 84, 225, 75, 203, 109, 235, 74, 92, 233, 212, 223, 98, 196, 73, 84, 117, 19, 60, 244, 38, 154, 213, 17, 152, 11, 136, 220, 240, 201, 184, 255, 234, 66, 164, 160, 139, 133, 111, 76, 82, 215, 10, 57, 215, 52, 53, 138, 202, 84, 147, 10, 184, 214, 171, 51, 24, 205, 5, 64, 162, 200, 227, 173, 97, 140, 42, 204, 131, 20, 20, 41, 63, 101, 204, 137, 6, 105, 213, 90, 146, 12, 254, 123, 39, 114, 91, 194, 247, 161, 18, 37, 37, 33, 166, 153, 4, 63, 124, 136, 72, 236, 89, 211, 56, 11, 65, 111, 33, 109, 46, 186, 157, 91, 201, 28, 96, 240, 30, 42, 21, 202, 18, 88, 118, 242, 93, 120, 42, 106, 116, 162, 180, 35, 43, 179, 200, 165, 254, 66, 120, 125, 29, 242, 214, 38, 255, 179, 189, 73, 82, 31, 117, 141, 180, 4, 97, 173, 14, 37, 52, 220, 30, 107, 235, 59, 111, 91, 196, 56, 16, 29, 132, 27, 232, 185, 116, 155, 155, 194, 102, 122, 81, 30, 214, 67, 58, 98, 186, 103, 229, 3, 249, 246, 56, 172, 122, 150, 255, 113, 172, 234, 162, 85, 247, 115, 215, 157, 178, 51, 238, 208, 142, 20, 169, 60, 159, 36, 113, 102, 73, 148, 106, 229, 40, 255, 81, 60, 23, 72, 113, 92, 173, 118, 198, 25, 84, 206, 90, 22, 197, 36, 245, 207, 135, 136, 212, 160, 106, 46, 120, 225, 56, 135, 75, 29, 66, 197, 67, 233, 200, 249, 242, 249, 68, 95, 191, 195, 195, 96, 211, 40, 71, 51, 244, 222, 18, 53, 78, 246, 56, 139, 139, 5, 195, 163, 61, 77, 179, 116, 58, 159, 130, 165, 184, 20, 95, 236, 17, 247, 47, 177, 220, 94, 6, 204, 168, 8, 26, 157, 81, 113, 238, 69, 148, 124, 100, 22, 73, 212, 125, 172, 168, 55, 69, 79, 84, 63, 106, 216, 252, 48, 200, 92, 253, 130, 153, 1, 51, 189, 148, 160, 229, 204, 22, 93, 212, 250, 156, 51, 46, 179, 134, 84, 143, 125, 42, 47, 65, 58, 232, 246, 116, 121, 151, 250, 195, 53, 251, 48, 121, 249, 58, 158, 73, 79, 234, 183, 36, 30, 179, 95, 228, 118, 100, 198, 135, 42, 229, 75, 123, 176, 199, 2, 93, 154, 231, 156, 13, 208, 8, 163, 154, 51, 233, 156, 253, 10, 112, 67, 168, 23, 236, 162, 42, 160, 145, 150, 253, 124, 18, 95, 45, 100, 238, 69, 156, 211, 166, 143, 83, 180, 207, 165, 36, 48, 142, 2, 201, 174, 189, 23, 230, 22, 117, 189, 19, 241, 215, 139, 232, 34, 106, 226, 116, 49, 25, 144, 18, 78, 61, 8, 202, 244, 14, 148, 82, 217, 55, 69, 179, 184, 126, 208, 123, 189, 135, 175, 127, 234, 107, 128, 69, 240, 199, 166, 193, 221, 230, 131, 169, 33, 204, 105, 66, 140, 205, 77, 242, 123, 233, 200, 32, 210, 166, 241, 226, 52, 169, 35, 206, 71, 141, 56, 93, 162, 145, 186, 45, 25, 245, 55, 28, 182, 180, 204, 140, 191, 127, 16, 55, 172, 46, 33, 154, 211, 114, 95, 11, 122, 223, 47, 233, 190, 171, 70, 176, 104, 141, 226, 210, 35, 90, 206, 153, 45, 131, 92, 177, 168, 20, 111, 184, 233, 77, 226, 178, 132, 253, 47, 204, 100, 242, 235, 74, 224, 123, 189, 154, 5, 105, 25, 122, 197, 156, 220, 192, 129, 143, 135, 217, 23, 169, 50, 171, 103, 73, 32, 24, 91, 77, 160, 255, 173, 155, 219, 91, 129, 62, 160, 226, 38, 218, 53, 76, 40, 103, 168, 180, 220, 37, 11, 86, 144, 185, 123, 65, 161, 11, 74, 151, 77, 173, 85, 44, 21, 155, 242, 217, 52, 104, 60, 143, 61, 185, 203, 16, 249, 30, 182, 170, 22, 98, 101, 105, 207, 218, 143, 88, 90, 170, 226, 113, 107, 232, 197, 172, 203, 155, 120, 230, 221, 104, 232, 107, 104, 25, 166, 55, 158, 157, 143, 8, 143, 224, 196, 138, 86, 206, 43, 93, 90, 217, 249, 6, 14, 201, 162, 159, 147, 133, 179, 171, 119, 156, 237, 68, 127, 42, 119, 99, 39, 106, 101, 243, 105, 82, 164, 189, 150, 176, 236, 59, 198, 157, 188, 22, 62, 243, 120, 64, 225, 123, 113, 153, 188, 72, 139, 178, 50, 53, 74, 117, 115, 172, 240, 216, 118, 162, 227, 214, 28, 118, 248, 193, 93, 178, 19, 56, 163, 4, 95, 248, 174, 189, 214, 201, 53, 217, 212, 91, 59, 164, 170, 212, 157, 103, 28, 129, 107, 58, 25, 115, 224, 141, 121, 41, 78, 163, 219, 141, 136, 7, 166, 9, 245, 12, 20, 48, 184, 9, 214, 128, 107, 152, 242, 144, 133, 91, 226, 125, 51, 254, 38, 164, 179, 205, 160, 156, 13, 202, 82, 75, 29, 92, 254, 248, 229, 7, 77, 35, 223, 158, 111, 242, 167, 126, 240, 29, 4, 206, 223, 26, 103, 42, 24, 58, 188, 139, 77, 176, 1, 74, 85, 245, 2, 160, 250, 176, 199, 154, 44, 2, 235, 154, 238, 254, 220, 186, 56, 208, 225, 157, 147, 216, 88, 165, 50, 243, 180, 235, 224, 253, 107, 207, 117, 13, 152, 85, 105, 159, 140, 153, 180, 155, 179, 150, 59, 78, 22, 78, 164, 210, 81, 175, 27, 177, 214, 19, 59, 119, 176, 142, 216, 116, 13, 81, 174, 100, 9, 225, 236, 229, 45, 59, 19, 163, 72, 142, 248, 99, 96, 45, 195, 98, 37, 194, 245, 82, 205, 70, 47, 231, 136, 64, 9, 225, 6, 103, 121, 207, 8, 92, 228, 12, 207, 80, 66, 197, 183, 160, 98, 94, 216, 228, 49, 101, 214, 42, 132, 69, 154, 172, 186, 161, 138, 2, 244, 121, 144, 160, 87, 64, 109, 44, 139, 174, 49, 174, 115, 179, 243, 209, 90, 115, 65, 103, 225, 176, 112, 152, 104, 207, 93, 24, 240, 108, 37, 42, 125, 118, 164, 136, 47, 14, 97, 28, 171, 244, 60, 85, 157, 228, 107, 1, 245, 151, 194, 59, 118, 197, 57, 213, 6, 163, 43, 111, 22, 104, 45, 25, 37, 126, 38, 23, 47, 157, 23, 104, 37, 221, 230, 59, 6, 117, 151, 167, 205, 51, 43, 171, 7, 244, 208, 138, 204, 116, 89, 253, 54, 236, 186, 246, 104, 34, 2, 135, 105, 194, 167, 172, 233, 48, 203, 139, 228, 45, 204, 74, 243, 88, 46, 241, 56, 28, 119, 128, 124, 152, 78, 97, 41, 65, 223, 10, 111, 237, 51, 12, 77, 217, 216, 41, 116, 167, 66, 238, 54, 187, 190, 178, 237, 34, 214, 57, 221, 198, 187, 34, 152, 108, 47, 209, 158, 82, 151, 135, 56, 174, 147, 45, 3, 49, 53, 44, 254, 46, 59, 124, 171, 243, 238, 92, 135, 245, 99, 184, 77, 232, 90, 66, 21, 33, 41, 82, 180, 210, 168, 173, 72, 110, 70, 29, 241, 33, 16, 98, 41, 28, 254, 171, 45, 31, 108, 227, 55, 73, 226, 97, 245, 102, 38, 52, 50, 234, 225, 99, 113, 93, 230, 17, 145, 141, 2, 230, 200, 194, 146, 253, 245, 162, 238, 210, 137, 217, 179, 218, 228, 46, 52, 4, 54, 230, 155, 101, 134, 63, 229, 245, 200, 196, 207, 94, 186, 10, 193, 183, 53, 70, 187, 190, 36, 109, 139, 59, 95, 224, 140, 106, 143, 245, 88, 120, 97, 80, 196, 29, 162, 22, 220, 117, 198, 139, 66, 125, 85, 206, 227, 220, 52, 151, 207, 252, 187, 139, 124, 62, 136, 200, 249, 153, 186, 102, 121, 195, 71, 48, 89, 60, 177, 8, 151, 203, 27, 29, 180, 166, 112, 130, 127, 72, 34, 24, 50, 186, 6, 195, 21, 219, 218, 35, 39, 131, 0, 181, 220, 239, 35, 197, 196, 9, 122, 74, 36, 119, 174, 154, 161, 52, 67, 77, 208, 6, 214, 236, 172, 166, 201, 207, 31, 162, 251, 48, 163, 222, 103, 78, 195, 125, 30, 63, 220, 55, 125, 26, 103, 22, 52, 114, 196, 85, 136, 82, 215, 31, 28, 199, 96, 182, 6, 35, 23, 183, 139, 156, 140, 1, 15, 207, 113, 131, 134, 144, 50, 208, 54, 154, 146, 112, 235, 1, 111, 243, 184, 117, 202, 240, 243, 55, 97, 245, 122, 73, 198, 9, 238, 201, 18, 53, 83, 123, 168, 91, 226, 50, 152, 185, 144, 155, 156, 89, 128, 84, 103, 214, 11, 10, 0, 103, 83, 168, 109, 223, 12, 155, 206, 180, 29, 247, 37, 26, 241, 164, 229, 132, 131, 246, 77, 178, 147, 247, 234, 241, 251, 210, 90, 238, 58, 24, 232, 179, 64, 228, 184, 214, 237, 223, 219, 99, 135, 51, 181, 113, 162, 39, 248, 185, 189, 37, 150, 179, 217, 173, 22, 133, 237, 164, 129, 75, 186, 206, 29, 208, 24, 115, 71, 87, 57, 36, 229, 29, 57, 170, 52, 192, 68, 208, 214, 19, 1, 255, 222, 166, 121, 135, 199, 14, 226, 161, 251, 76, 13, 126, 219, 62, 216, 105, 66, 105, 160, 27, 174, 29, 222, 112, 80, 211, 152, 71, 177, 148, 168, 31, 57, 248, 137, 42, 168, 235, 140, 252, 136, 201, 217, 203, 208, 33, 194, 148, 136, 171, 43, 22, 234, 229, 177, 50, 1, 227, 13, 220, 32, 199, 194, 66, 52, 166, 37, 227, 172, 117, 66, 114, 42, 254, 65, 241, 220, 112, 170, 99, 56, 153, 7, 159, 24, 90, 246, 244, 35, 186, 135, 34, 176, 100, 55, 207, 228, 109, 19, 6, 101, 39, 66, 206, 134, 122, 17, 131, 102, 78, 47, 223, 193, 176, 38, 84, 205, 160, 99, 194, 224, 170, 77, 113, 39, 130, 25, 68, 227, 67, 162, 100, 245, 210, 219, 238, 49, 16, 118, 178, 164, 77, 243, 24, 217, 234, 29, 99, 111, 107, 104, 218, 203, 167, 179, 184, 72, 142, 242, 31, 211, 172, 223, 190, 236, 68, 228, 184, 236, 63, 240, 66, 234, 37, 50, 1, 191, 227, 7, 11, 199, 91, 5, 137, 225, 120, 5, 68, 41, 47, 61, 255, 71, 39, 250, 189, 206, 24, 246, 68, 37, 200, 19, 183, 107, 34, 7, 215, 235, 150, 87, 135, 203, 250, 45, 143, 167, 47, 131, 29, 73, 86, 56, 236, 132, 230, 94, 32, 211, 72, 248, 243, 136, 128, 157, 122, 47, 151, 86, 63, 222, 159, 178, 10, 157, 232, 5, 121, 251, 27, 167, 42, 64, 17, 195, 135, 99, 105, 39, 149, 141, 161, 107, 120, 40, 35, 147, 190, 122, 61, 216, 77, 248, 104, 225, 104, 17, 219, 175, 222, 27, 175, 59, 36, 164, 120, 7, 231, 234, 197, 246, 56, 13, 193, 189, 80, 12, 192, 11, 2, 82, 107, 160, 80, 177, 129, 117, 230, 102, 92, 195, 121, 168, 36, 223, 144, 239, 79, 23, 214, 204, 132, 53, 9, 68, 223, 142, 147, 218, 220, 151, 239, 220, 243, 42, 155, 230, 81, 230, 200, 74, 126, 253, 128, 179, 92, 209, 78, 48, 19, 229, 201, 52, 89, 107, 217, 98, 132, 113, 206, 79, 243, 215, 164, 130, 234, 15, 247, 72, 39, 189, 230, 22, 5, 45, 64, 19, 52, 229, 226, 42, 255, 230, 19, 46, 94, 137, 185, 197, 164, 155, 205, 173, 233, 118, 233, 66, 15, 126, 130, 15, 49, 221, 112, 41, 35, 224, 46, 250, 90, 114, 226, 114, 59, 100, 245, 86, 34, 225, 41, 126, 42, 226, 173, 26, 17, 12, 172, 101, 96, 28, 27, 230, 73, 221, 138, 1, 169, 178, 250, 170, 193, 82, 70, 172, 140, 96, 201, 234, 1, 161, 247, 19, 45, 37, 80, 32, 49, 50, 204, 40, 29, 46, 88, 63, 167, 28, 58, 67, 57, 138, 160, 140, 148, 160, 46, 62, 140, 252, 1, 202, 152, 234, 228, 114, 75, 221, 242, 65, 86, 8, 104, 101, 5, 18, 106, 85, 61, 119, 39, 113, 90, 255, 89, 90, 206, 38, 241, 82, 52, 18, 12, 155, 80, 138, 79, 238, 116, 2, 153, 29, 38, 133, 120, 51, 66, 8, 169, 15, 214, 199, 32, 9, 247, 162, 136, 185, 37, 104, 132, 148, 2, 123, 177, 138, 13, 88, 13, 177, 250, 128, 205, 120, 181, 100, 171, 135, 108, 22, 79, 7, 44, 152, 204, 42, 1, 210, 98, 141, 122, 49, 226, 17, 13, 141, 77, 8, 233, 146, 174, 59, 240, 65, 252, 162, 83, 205, 241, 7, 224, 195, 210, 48, 47, 147, 151, 69, 62, 159, 153, 53, 90, 7, 41, 2, 249, 40, 121, 126, 255, 124, 113, 23, 185, 126, 140, 249, 203, 207, 247, 251, 249, 240, 217, 114, 254, 152, 156, 54, 158, 237, 110, 147, 251, 199, 120, 238, 36, 255, 79, 16, 210, 53, 0, 91, 144, 63, 246, 58, 192, 210, 169, 246, 115, 175, 5, 212, 37, 231, 125, 211, 252, 151, 153, 160, 15, 102, 146, 97, 146, 182, 218, 237, 39, 189, 116, 26, 79, 160, 148, 41, 136, 206, 24, 171, 73, 27, 104, 80, 32, 117, 188, 176, 230, 127, 117, 215, 151, 55, 151, 174, 112, 99, 66, 150, 172, 13, 244, 220, 13, 225, 247, 194, 23, 143, 133, 212, 230, 129, 121, 207, 20, 39, 208, 158, 1, 107, 61, 22, 78, 69, 64, 244, 236, 150, 30, 108, 162, 132, 3, 240, 33, 120, 149, 89, 144, 161, 200, 39, 234, 33, 92, 122, 114, 219, 35, 239, 109, 210, 248, 204, 70, 128, 94, 127, 9, 255, 165, 27, 100, 201, 3, 156, 220, 226, 69, 139, 233, 105, 206, 197, 2, 94, 118, 45, 86, 186, 68, 209, 90, 13, 195, 154, 245, 201, 248, 105, 61, 148, 32, 159, 102, 169, 36, 217, 191, 143, 190, 229, 115, 181, 24, 129, 91, 157, 8, 246, 89, 110, 213, 137, 103, 192, 163, 131, 38, 49, 1, 131, 190, 188, 67, 3, 2, 124, 87, 26, 126, 208, 252, 223, 193, 135, 247, 161, 94, 19, 194, 169, 227, 213, 226, 199, 35, 183, 217, 63, 212, 110, 131, 62, 56, 190, 30, 84, 71, 196, 92, 106, 100, 77, 198, 9, 129, 75, 227, 207, 247, 228, 51, 47, 169, 13, 120, 59, 170, 199, 107, 198, 162, 98, 140, 202, 100, 172, 128, 112, 115, 205, 99, 156, 151, 214, 122, 176, 70, 187, 88, 207, 111, 159, 163, 219, 18, 140, 150, 44, 191, 108, 198, 242, 128, 123, 13, 213, 241, 248, 151, 253, 200, 156, 97, 171, 247, 96, 69, 7, 104, 133, 90, 151, 148, 111, 52, 130, 22, 238, 57, 71, 197, 46, 237, 82, 108, 176, 36, 85, 75, 165, 210, 117, 247, 29, 149, 18, 35, 180, 12, 93, 160, 23, 156, 239, 101, 95, 96, 220, 222, 90, 102, 247, 106, 226, 5, 64, 179, 172, 118, 56, 26, 128, 14, 216, 54, 114, 24, 92, 250, 215, 254, 124, 112, 5, 147, 95, 108, 86, 35, 59, 52, 108, 24, 214, 175, 94, 90, 110, 184, 55, 216, 210, 75, 111, 196, 242, 235, 76, 222, 143, 63, 61, 219, 137, 30, 136, 204, 193, 143, 251, 47, 152, 133, 146, 223, 127, 127, 167, 191, 239, 189, 122, 129, 190, 191, 213, 223, 159, 253, 143, 41, 223, 255, 191, 6, 207, 203, 55, 168, 252, 240, 103, 253, 253, 127, 14, 12, 204, 255, 188, 253, 93, 151, 255, 252, 218, 192, 255, 252, 238, 55, 243, 253, 55, 3, 255, 234, 119, 243, 253, 224, 245, 59, 253, 253, 237, 239, 47, 53, 252, 187, 223, 12, 158, 163, 55, 6, 254, 151, 151, 255, 208, 229, 191, 252, 190, 175, 191, 255, 10, 48, 234, 251, 47, 191, 234, 239, 255, 120, 106, 240, 252, 227, 0, 125, 127, 11, 223, 73, 102, 14, 242, 241, 214, 171, 16, 155, 220, 203, 234, 188, 62, 177, 193, 109, 172, 238, 43, 23, 173, 220, 102, 205, 29, 172, 255, 123, 12, 119, 176, 254, 249, 224, 250, 255, 216, 247, 176, 218, 14, 129, 117, 251, 42, 14, 231, 86, 185, 54, 152, 18, 107, 126, 123, 94, 56, 75, 32, 143, 213, 207, 19, 18, 86, 109, 187, 75, 39, 36, 215, 250, 89, 215, 227, 87, 190, 121, 248, 203, 89, 147, 119, 179, 5, 58, 187, 231, 123, 95, 165, 220, 96, 107, 65, 170, 149, 93, 43, 61, 72, 87, 117, 37, 197, 199, 122, 248, 253, 184, 76, 10, 179, 21, 23, 241, 41, 223, 144, 194, 254, 194, 27, 116, 193, 162, 50, 174, 136, 88, 71, 88, 216, 126, 114, 30, 163, 45, 45, 173, 1, 139, 57, 47, 82, 225, 137, 14, 231, 103, 133, 248, 91, 204, 225, 239, 40, 102, 94, 91, 194, 247, 22, 141, 71, 211, 2, 87, 27, 103, 253, 152, 239, 198, 137, 243, 92, 252, 173, 248, 22, 148, 73, 58, 229, 20, 76, 39, 139, 41, 134, 159, 230, 162, 249, 233, 98, 202, 155, 200, 139, 5, 175, 95, 197, 211, 137, 108, 162, 74, 38, 115, 92, 165, 26, 197, 156, 242, 42, 61, 149, 11, 187, 39, 132, 255, 254, 69, 148, 101, 75, 205, 217, 124, 167, 110, 228, 196, 180, 21, 108, 71, 252, 47, 215, 144, 201, 77, 38, 48, 214, 151, 81, 224, 220, 39, 94, 47, 22, 40, 36, 144, 88, 134, 188, 244, 44, 21, 19, 93, 29, 136, 63, 75, 86, 139, 253, 57, 27, 185, 210, 57, 144, 103, 108, 130, 11, 198, 42, 107, 182, 108, 189, 56, 196, 14, 107, 201, 120, 112, 23, 75, 198, 58, 147, 119, 58, 88, 190, 98, 44, 128, 151, 112, 114, 133, 69, 227, 90, 118, 222, 245, 194, 177, 101, 8, 48, 104, 22, 82, 119, 158, 9, 32, 203, 200, 80, 96, 158, 251, 66, 49, 12, 238, 123, 238, 212, 166, 30, 185, 167, 58, 5, 64, 245, 125, 30, 61, 169, 239, 3, 192, 75, 233, 218, 171, 163, 11, 234, 186, 88, 233, 170, 98, 188, 176, 210, 214, 226, 171, 177, 158, 40, 29, 226, 150, 233, 16, 29, 153, 172, 160, 215, 142, 96, 66, 174, 184, 178, 87, 240, 199, 123, 93, 184, 67, 107, 221, 202, 249, 76, 199, 48, 184, 134, 115, 74, 164, 169, 167, 129, 252, 225, 153, 117, 245, 114, 104, 137, 156, 172, 141, 47, 89, 20, 183, 215, 96, 151, 28, 219, 67, 207, 238, 104, 29, 150, 34, 21, 39, 21, 106, 223, 80, 16, 88, 33, 165, 120, 26, 174, 145, 90, 149, 86, 95, 37, 117, 89, 178, 242, 58, 169, 7, 133, 32, 254, 160, 126, 189, 212, 162, 253, 179, 172, 152, 66, 57, 115, 46, 10, 14, 214, 139, 125, 32, 203, 22, 85, 151, 28, 126, 161, 200, 224, 165, 35, 255, 204, 51, 117, 240, 208, 195, 58, 5, 97, 26, 16, 147, 233, 129, 94, 172, 247, 84, 210, 32, 166, 214, 44, 174, 160, 97, 158, 164, 132, 27, 101, 246, 247, 126, 121, 43, 138, 208, 57, 155, 142, 65, 238, 229, 161, 65, 34, 191, 125, 148, 101, 176, 144, 202, 222, 197, 66, 152, 117, 234, 247, 139, 95, 10, 11, 241, 226, 211, 44, 134, 73, 121, 230, 20, 192, 158, 56, 38, 135, 153, 216, 207, 110, 18, 18, 23, 73, 50, 238, 199, 11, 190, 15, 38, 46, 138, 252, 130, 167, 149, 71, 242, 4, 197, 4, 92, 111, 185, 115, 50, 41, 226, 6, 80, 139, 36, 46, 0, 108, 187, 219, 135, 217, 172, 133, 142, 25, 72, 8, 230, 206, 87, 163, 0, 8, 79, 100, 215, 226, 151, 180, 134, 209, 143, 242, 249, 50, 2, 210, 108, 94, 37, 245, 48, 101, 194, 34, 251, 126, 61, 140, 210, 109, 144, 62, 128, 180, 232, 133, 196, 133, 119, 4, 26, 188, 71, 36, 48, 106, 248, 126, 19, 179, 46, 73, 133, 172, 77, 39, 172, 219, 44, 114, 82, 76, 119, 178, 204, 105, 161, 164, 11, 157, 174, 41, 252, 216, 75, 157, 13, 230, 203, 224, 98, 39, 94, 170, 84, 120, 84, 104, 96, 146, 148, 113, 198, 215, 110, 160, 223, 216, 209, 215, 83, 106, 112, 33, 179, 110, 30, 142, 118, 13, 2, 27, 167, 94, 113, 212, 163, 117, 39, 139, 156, 158, 153, 219, 89, 246, 212, 52, 57, 139, 159, 228, 183, 161, 184, 186, 66, 9, 118, 58, 85, 242, 219, 30, 174, 194, 137, 115, 94, 213, 89, 176, 211, 187, 173, 171, 171, 53, 28, 26, 26, 100, 173, 95, 142, 246, 90, 53, 87, 33, 125, 207, 226, 202, 43, 226, 163, 46, 153, 217, 129, 140, 43, 255, 244, 222, 120, 96, 113, 37, 207, 228, 159, 138, 125, 166, 40, 34, 64, 209, 128, 107, 94, 66, 209, 0, 224, 64, 14, 173, 169, 224, 236, 160, 61, 15, 31, 178, 2, 36, 88, 92, 2, 150, 138, 111, 130, 245, 156, 96, 146, 61, 16, 61, 20, 64, 17, 57, 48, 125, 109, 113, 178, 153, 56, 19, 88, 75, 156, 79, 227, 82, 158, 113, 91, 77, 154, 129, 134, 81, 177, 181, 29, 88, 125, 128, 121, 102, 107, 123, 165, 179, 115, 28, 91, 224, 204, 156, 102, 202, 134, 64, 204, 90, 5, 240, 250, 5, 9, 199, 29, 10, 132, 69, 196, 69, 115, 28, 72, 153, 79, 240, 93, 41, 89, 142, 19, 253, 190, 114, 121, 116, 188, 188, 72, 225, 237, 15, 138, 19, 27, 210, 129, 80, 181, 224, 20, 155, 158, 196, 119, 76, 190, 65, 33, 130, 183, 33, 62, 135, 15, 58, 137, 142, 94, 202, 43, 170, 139, 153, 52, 84, 187, 174, 38, 159, 122, 107, 154, 109, 97, 41, 179, 187, 194, 60, 156, 85, 186, 241, 18, 62, 55, 238, 70, 109, 205, 186, 110, 132, 91, 245, 247, 10, 60, 50, 218, 45, 229, 210, 4, 26, 88, 44, 234, 72, 83, 78, 80, 168, 242, 42, 164, 113, 87, 112, 37, 218, 94, 191, 190, 5, 109, 245, 117, 107, 199, 250, 53, 124, 110, 60, 214, 75, 234, 214, 141, 118, 184, 93, 63, 75, 29, 85, 92, 194, 208, 126, 255, 22, 12, 13, 212, 53, 54, 28, 108, 217, 161, 172, 177, 55, 226, 129, 121, 235, 12, 189, 87, 86, 81, 45, 109, 94, 237, 237, 152, 20, 150, 78, 225, 190, 118, 70, 136, 56, 231, 58, 75, 95, 133, 159, 106, 141, 3, 212, 88, 133, 179, 78, 11, 235, 78, 209, 141, 25, 223, 8, 81, 64, 229, 120, 240, 179, 146, 136, 76, 167, 183, 16, 145, 64, 93, 63, 113, 34, 234, 90, 137, 184, 178, 188, 5, 113, 129, 186, 126, 226, 112, 184, 71, 73, 172, 213, 253, 171, 155, 106, 254, 21, 251, 212, 209, 135, 23, 134, 100, 62, 135, 77, 249, 74, 44, 244, 172, 174, 112, 82, 39, 160, 38, 100, 129, 104, 39, 45, 192, 135, 145, 107, 134, 101, 104, 117, 186, 241, 86, 79, 115, 191, 225, 106, 155, 61, 13, 128, 27, 5, 186, 219, 66, 179, 36, 81, 219, 133, 248, 177, 37, 253, 46, 74, 160, 181, 173, 186, 37, 182, 68, 241, 120, 12, 182, 231, 208, 98, 136, 214, 214, 240, 91, 14, 108, 55, 199, 221, 223, 131, 167, 89, 231, 169, 177, 57, 100, 202, 11, 99, 113, 154, 112, 222, 114, 107, 245, 18, 189, 57, 245, 58, 212, 89, 144, 220, 149, 58, 11, 246, 37, 76, 163, 48, 33, 13, 250, 42, 212, 249, 14, 250, 67, 96, 190, 254, 58, 106, 247, 117, 5, 50, 152, 180, 56, 158, 76, 90, 107, 120, 22, 177, 239, 163, 213, 190, 60, 140, 29, 78, 240, 160, 96, 37, 88, 135, 15, 228, 170, 149, 250, 98, 39, 167, 85, 165, 113, 71, 245, 64, 222, 180, 163, 48, 176, 43, 119, 148, 15, 247, 170, 181, 196, 216, 7, 59, 43, 141, 151, 132, 167, 215, 243, 67, 188, 246, 2, 7, 116, 6, 173, 136, 146, 72, 136, 194, 223, 102, 16, 79, 222, 38, 44, 38, 226, 103, 62, 183, 182, 55, 77, 124, 23, 203, 151, 109, 152, 199, 248, 41, 152, 223, 87, 73, 89, 154, 167, 223, 209, 135, 175, 225, 160, 172, 126, 248, 45, 90, 63, 5, 147, 79, 171, 218, 79, 105, 221, 7, 40, 151, 79, 214, 24, 91, 210, 178, 0, 171, 88, 0, 210, 66, 226, 168, 69, 172, 79, 87, 23, 213, 40, 154, 145, 49, 220, 198, 179, 84, 11, 189, 164, 246, 92, 165, 125, 87, 206, 187, 26, 28, 16, 33, 31, 246, 114, 126, 14, 187, 187, 159, 13, 32, 195, 183, 160, 143, 213, 66, 186, 42, 243, 190, 65, 75, 142, 226, 10, 239, 204, 26, 168, 245, 8, 89, 213, 201, 88, 148, 146, 172, 205, 71, 158, 122, 111, 117, 242, 4, 62, 122, 73, 245, 76, 36, 81, 206, 88, 147, 102, 72, 116, 242, 228, 140, 38, 79, 172, 193, 131, 33, 53, 149, 142, 207, 60, 47, 39, 148, 66, 251, 86, 100, 93, 114, 255, 173, 224, 136, 40, 73, 168, 187, 232, 162, 62, 171, 44, 211, 14, 100, 242, 87, 53, 231, 121, 75, 46, 105, 82, 252, 8, 81, 135, 38, 123, 1, 111, 111, 243, 71, 184, 106, 223, 252, 192, 149, 131, 15, 85, 119, 215, 214, 77, 218, 228, 181, 217, 197, 140, 91, 118, 14, 24, 224, 94, 52, 110, 153, 218, 12, 111, 195, 110, 109, 107, 76, 213, 251, 242, 2, 247, 103, 56, 87, 222, 163, 170, 214, 125, 120, 10, 185, 233, 202, 170, 184, 77, 205, 0, 234, 126, 50, 209, 201, 116, 189, 103, 219, 252, 72, 179, 182, 221, 120, 215, 233, 107, 39, 218, 94, 235, 68, 221, 109, 79, 19, 48, 74, 178, 9, 54, 6, 158, 55, 155, 17, 238, 91, 134, 213, 69, 167, 71, 222, 224, 12, 188, 29, 75, 227, 180, 45, 110, 35, 164, 161, 119, 110, 81, 172, 200, 202, 55, 195, 218, 164, 255, 65, 164, 190, 215, 78, 241, 173, 252, 188, 246, 19, 99, 131, 109, 1, 199, 198, 153, 3, 63, 114, 30, 191, 160, 86, 244, 145, 175, 41, 239, 219, 116, 176, 101, 119, 95, 164, 211, 108, 174, 150, 168, 154, 204, 238, 238, 62, 44, 154, 211, 252, 188, 251, 79, 151, 109, 8, 249, 162, 119, 160, 226, 5, 201, 80, 7, 34, 223, 181, 19, 180, 210, 191, 249, 197, 19, 150, 56, 222, 226, 190, 167, 88, 120, 97, 214, 5, 20, 33, 102, 47, 189, 11, 41, 222, 177, 111, 125, 146, 59, 98, 244, 38, 85, 188, 109, 112, 213, 221, 172, 148, 44, 178, 159, 149, 108, 91, 22, 47, 74, 247, 232, 220, 38, 209, 185, 205, 128, 206, 193, 204, 215, 22, 72, 232, 140, 254, 3, 159, 191, 128, 249, 239, 179, 252, 130, 181, 185, 99, 182, 204, 2, 124, 112, 199, 172, 26, 175, 154, 61, 179, 214, 144, 54, 219, 53, 27, 218, 222, 113, 215, 251, 102, 195, 140, 167, 238, 238, 23, 180, 115, 214, 226, 231, 10, 123, 103, 151, 48, 245, 174, 119, 207, 170, 61, 99, 24, 70, 149, 221, 110, 143, 173, 10, 197, 48, 176, 42, 83, 218, 37, 24, 203, 123, 132, 86, 228, 12, 188, 76, 236, 117, 100, 223, 168, 132, 211, 187, 132, 210, 222, 92, 214, 135, 148, 240, 81, 174, 87, 100, 53, 67, 158, 239, 68, 38, 218, 228, 37, 47, 119, 100, 208, 201, 127, 193, 109, 104, 34, 246, 228, 63, 95, 239, 232, 16, 148, 255, 126, 101, 253, 238, 239, 224, 200, 116, 180, 163, 2, 83, 254, 243, 39, 250, 115, 76, 127, 254, 76, 127, 78, 119, 76, 48, 203, 11, 202, 29, 19, 211, 242, 130, 171, 29, 43, 180, 21, 197, 231, 254, 226, 95, 237, 98, 223, 245, 62, 136, 223, 54, 151, 221, 109, 164, 145, 253, 106, 70, 8, 84, 101, 53, 105, 223, 108, 197, 237, 153, 125, 118, 199, 204, 173, 178, 247, 219, 138, 212, 205, 125, 231, 74, 246, 32, 126, 136, 118, 52, 190, 19, 177, 153, 225, 190, 231, 14, 115, 237, 194, 121, 11, 130, 183, 126, 247, 152, 54, 131, 168, 108, 61, 186, 17, 57, 208, 155, 158, 247, 101, 202, 54, 57, 240, 225, 141, 217, 15, 174, 29, 154, 60, 175, 241, 85, 89, 121, 167, 45, 145, 80, 127, 137, 178, 233, 166, 244, 185, 183, 244, 202, 91, 122, 238, 45, 253, 213, 42, 229, 61, 227, 151, 175, 3, 215, 30, 239, 70, 15, 252, 78, 191, 126, 117, 175, 72, 170, 212, 4, 16, 146, 255, 140, 141, 15, 151, 160, 226, 105, 25, 63, 166, 250, 138, 50, 197, 227, 86, 69, 107, 11, 180, 219, 11, 47, 51, 250, 222, 210, 145, 183, 244, 39, 111, 233, 212, 91, 90, 122, 75, 199, 222, 210, 159, 107, 7, 164, 38, 12, 213, 220, 80, 105, 176, 27, 113, 82, 230, 205, 86, 96, 229, 107, 111, 55, 94, 45, 237, 70, 32, 70, 109, 64, 138, 71, 180, 110, 199, 19, 11, 217, 95, 77, 228, 101, 82, 157, 14, 8, 53, 74, 43, 157, 148, 8, 205, 199, 112, 191, 133, 219, 223, 192, 233, 137, 32, 150, 227, 222, 137, 251, 214, 62, 77, 163, 152, 93, 70, 124, 177, 64, 254, 248, 217, 51, 87, 4, 26, 53, 155, 155, 208, 249, 105, 204, 116, 220, 200, 79, 184, 145, 241, 77, 26, 193, 215, 76, 27, 158, 95, 75, 191, 102, 194, 194, 47, 22, 122, 57, 111, 61, 106, 197, 69, 213, 157, 228, 103, 167, 113, 214, 98, 243, 253, 217, 105, 46, 103, 255, 86, 47, 153, 116, 135, 108, 40, 211, 114, 4, 79, 240, 79, 9, 145, 100, 221, 225, 105, 55, 79, 250, 240, 156, 253, 120, 249, 35, 255, 33, 159, 166, 221, 120, 154, 194, 19, 248, 163, 203, 78, 51, 222, 14, 252, 209, 101, 234, 21, 40, 59, 248, 135, 126, 154, 100, 121, 111, 148, 10, 242, 208, 47, 253, 124, 20, 143, 225, 17, 252, 209, 101, 227, 9, 235, 100, 206, 171, 84, 147, 145, 41, 159, 204, 47, 161, 108, 114, 106, 138, 166, 0, 41, 112, 139, 175, 250, 73, 198, 226, 191, 51, 254, 40, 59, 55, 165, 179, 11, 142, 22, 254, 232, 178, 42, 230, 96, 240, 7, 149, 45, 68, 217, 2, 149, 149, 115, 94, 86, 170, 99, 150, 173, 179, 184, 11, 7, 55, 187, 163, 100, 118, 202, 156, 81, 248, 203, 92, 107, 62, 18, 113, 247, 21, 60, 137, 39, 241, 164, 247, 253, 223, 85, 133, 44, 239, 158, 230, 188, 195, 217, 41, 42, 203, 22, 156, 42, 195, 213, 114, 152, 117, 79, 147, 238, 160, 128, 226, 114, 112, 106, 149, 103, 19, 40, 63, 31, 86, 164, 156, 3, 159, 94, 149, 184, 176, 55, 98, 67, 194, 145, 12, 135, 164, 156, 119, 185, 87, 146, 22, 5, 228, 176, 36, 144, 125, 78, 110, 191, 156, 224, 194, 164, 228, 56, 203, 25, 46, 20, 212, 14, 40, 228, 240, 148, 83, 53, 72, 73, 97, 33, 26, 34, 164, 166, 188, 245, 148, 182, 158, 10, 185, 42, 19, 92, 120, 54, 227, 44, 166, 13, 77, 185, 108, 76, 7, 4, 103, 198, 69, 56, 235, 209, 194, 137, 232, 17, 193, 153, 9, 81, 161, 56, 103, 188, 245, 89, 89, 224, 194, 82, 240, 243, 130, 64, 206, 57, 67, 98, 138, 243, 42, 22, 195, 167, 91, 191, 26, 117, 123, 83, 62, 212, 240, 135, 20, 50, 69, 200, 74, 249, 164, 251, 19, 124, 119, 30, 87, 232, 113, 133, 30, 15, 133, 118, 13, 99, 140, 114, 56, 207, 23, 115, 183, 37, 166, 103, 227, 216, 82, 56, 86, 204, 116, 7, 10, 229, 55, 82, 206, 20, 73, 8, 39, 65, 127, 49, 231, 200, 225, 143, 41, 188, 100, 154, 61, 228, 200, 75, 12, 187, 152, 115, 142, 193, 159, 142, 58, 16, 0, 6, 237, 112, 126, 90, 249, 108, 218, 143, 191, 0, 248, 235, 215, 10, 197, 179, 103, 240, 251, 217, 115, 245, 251, 197, 63, 224, 247, 139, 119, 234, 247, 209, 91, 248, 125, 244, 74, 253, 254, 157, 195, 255, 174, 225, 255, 249, 14, 126, 239, 61, 83, 191, 227, 5, 23, 149, 108, 174, 137, 60, 61, 227, 70, 176, 95, 232, 49, 237, 245, 254, 128, 146, 98, 172, 229, 182, 119, 198, 133, 118, 154, 79, 117, 137, 224, 100, 79, 159, 236, 102, 95, 185, 178, 92, 246, 180, 209, 234, 23, 220, 252, 142, 71, 99, 83, 114, 193, 133, 170, 208, 35, 60, 140, 207, 121, 235, 137, 182, 83, 163, 162, 16, 70, 68, 211, 147, 158, 114, 73, 206, 103, 177, 46, 17, 38, 213, 24, 108, 142, 118, 164, 5, 240, 140, 87, 88, 104, 250, 207, 248, 243, 51, 221, 198, 120, 40, 40, 155, 234, 54, 38, 162, 207, 179, 153, 238, 207, 148, 43, 69, 161, 237, 226, 180, 228, 50, 56, 45, 180, 85, 156, 46, 236, 146, 178, 39, 228, 107, 162, 41, 169, 132, 102, 23, 177, 238, 113, 53, 230, 124, 170, 46, 52, 47, 171, 9, 39, 239, 34, 209, 109, 85, 217, 192, 226, 211, 226, 180, 111, 141, 201, 98, 202, 219, 154, 20, 133, 145, 172, 228, 178, 2, 225, 242, 76, 149, 220, 204, 255, 57, 43, 146, 65, 82, 20, 9, 68, 238, 80, 4, 235, 131, 108, 114, 190, 132, 159, 133, 122, 59, 96, 43, 62, 29, 57, 192, 167, 163, 32, 240, 185, 11, 124, 30, 2, 238, 77, 29, 96, 86, 20, 2, 254, 195, 5, 254, 35, 8, 124, 225, 2, 95, 4, 129, 47, 93, 224, 203, 32, 240, 194, 5, 94, 132, 128, 251, 3, 7, 152, 21, 133, 128, 75, 23, 184, 196, 192, 204, 144, 26, 232, 228, 212, 129, 78, 78, 67, 168, 147, 158, 11, 220, 11, 2, 247, 93, 224, 126, 152, 142, 204, 133, 206, 130, 208, 3, 151, 234, 65, 144, 234, 193, 208, 5, 30, 6, 81, 243, 169, 144, 66, 159, 205, 66, 168, 103, 46, 67, 102, 65, 134, 204, 92, 134, 204, 250, 33, 224, 194, 237, 97, 17, 236, 97, 225, 202, 116, 17, 148, 233, 194, 149, 143, 162, 12, 2, 187, 98, 90, 4, 197, 180, 184, 114, 129, 175, 66, 192, 124, 210, 167, 192, 252, 189, 10, 254, 81, 41, 93, 13, 40, 7, 97, 104, 119, 12, 203, 89, 24, 218, 229, 94, 249, 71, 24, 218, 53, 9, 229, 69, 16, 122, 238, 178, 100, 30, 100, 201, 249, 196, 1, 62, 159, 132, 128, 23, 174, 57, 93, 4, 205, 233, 194, 197, 188, 8, 99, 118, 85, 113, 145, 5, 129, 93, 86, 47, 66, 234, 114, 122, 234, 112, 3, 138, 2, 192, 131, 212, 1, 6, 159, 215, 207, 232, 211, 193, 216, 133, 30, 7, 161, 207, 156, 46, 66, 17, 130, 158, 150, 6, 56, 119, 236, 7, 20, 133, 80, 255, 225, 162, 254, 35, 104, 200, 78, 255, 112, 244, 11, 138, 66, 208, 213, 153, 3, 205, 138, 2, 100, 159, 59, 10, 6, 69, 33, 96, 71, 64, 78, 169, 232, 17, 58, 206, 231, 46, 244, 60, 132, 250, 202, 177, 55, 60, 162, 242, 163, 238, 245, 29, 151, 2, 138, 16, 244, 213, 8, 1, 59, 168, 123, 225, 169, 174, 119, 230, 240, 26, 138, 2, 168, 167, 206, 48, 114, 183, 223, 15, 156, 199, 14, 112, 30, 7, 248, 209, 155, 57, 110, 2, 20, 5, 48, 151, 206, 4, 3, 69, 161, 30, 150, 206, 12, 3, 69, 65, 104, 71, 64, 122, 97, 11, 220, 115, 45, 112, 47, 108, 129, 123, 165, 163, 51, 80, 20, 132, 118, 164, 15, 138, 130, 208, 238, 208, 148, 65, 13, 235, 185, 214, 189, 23, 182, 238, 189, 178, 112, 161, 139, 32, 244, 149, 99, 131, 161, 40, 48, 148, 87, 174, 100, 95, 133, 36, 187, 159, 59, 84, 67, 81, 128, 142, 190, 59, 146, 253, 240, 72, 246, 93, 110, 247, 195, 220, 238, 207, 29, 3, 223, 159, 207, 2, 178, 157, 244, 28, 133, 132, 162, 0, 234, 196, 37, 36, 9, 19, 146, 184, 195, 158, 132, 135, 61, 41, 29, 110, 67, 81, 8, 186, 114, 6, 50, 169, 70, 33, 232, 129, 219, 203, 65, 184, 151, 3, 119, 112, 6, 225, 193, 25, 184, 60, 25, 132, 121, 50, 40, 93, 74, 202, 32, 37, 34, 171, 65, 160, 121, 134, 195, 43, 129, 67, 215, 182, 14, 195, 182, 117, 152, 59, 33, 24, 20, 33, 232, 113, 62, 54, 208, 46, 75, 134, 97, 150, 12, 93, 91, 50, 12, 219, 146, 97, 233, 82, 82, 78, 195, 208, 110, 47, 107, 24, 56, 119, 161, 231, 65, 232, 81, 236, 184, 240, 163, 248, 52, 12, 237, 216, 87, 40, 10, 66, 59, 238, 14, 79, 63, 121, 135, 114, 228, 14, 229, 40, 60, 148, 163, 51, 199, 239, 130, 34, 191, 194, 143, 198, 46, 234, 113, 24, 117, 238, 66, 231, 97, 232, 153, 11, 61, 11, 67, 151, 142, 10, 67, 81, 16, 218, 81, 179, 81, 88, 205, 70, 174, 233, 25, 149, 33, 197, 73, 123, 14, 106, 40, 10, 160, 78, 39, 78, 39, 161, 40, 4, 157, 185, 184, 179, 48, 238, 204, 197, 157, 133, 113, 187, 74, 153, 134, 149, 50, 117, 149, 50, 13, 43, 101, 234, 78, 171, 105, 120, 90, 61, 115, 133, 251, 140, 10, 55, 146, 192, 179, 216, 113, 167, 160, 40, 0, 236, 90, 238, 179, 176, 229, 62, 27, 185, 208, 163, 48, 180, 59, 146, 103, 225, 145, 60, 115, 85, 225, 44, 172, 10, 103, 174, 184, 158, 133, 197, 245, 204, 53, 83, 103, 97, 51, 53, 30, 58, 250, 14, 69, 33, 232, 204, 81, 5, 40, 10, 24, 250, 241, 185, 99, 2, 161, 200, 63, 56, 227, 115, 103, 216, 161, 40, 68, 200, 185, 35, 82, 80, 20, 64, 125, 233, 56, 198, 80, 228, 7, 158, 156, 58, 236, 131, 162, 0, 29, 147, 158, 163, 54, 80, 20, 64, 221, 115, 140, 60, 20, 5, 128, 83, 39, 211, 0, 69, 1, 96, 87, 250, 38, 97, 233, 155, 184, 218, 59, 9, 107, 239, 196, 149, 190, 73, 88, 250, 38, 174, 231, 53, 9, 123, 94, 19, 55, 85, 51, 9, 167, 106, 38, 60, 91, 110, 65, 87, 97, 104, 39, 224, 131, 162, 16, 116, 229, 242, 164, 34, 60, 153, 156, 27, 224, 115, 151, 221, 231, 101, 8, 216, 13, 19, 38, 193, 48, 97, 234, 218, 179, 105, 208, 158, 77, 251, 206, 200, 64, 81, 160, 135, 211, 196, 25, 25, 40, 10, 160, 30, 56, 49, 45, 20, 133, 128, 29, 61, 159, 14, 66, 122, 14, 11, 138, 14, 112, 80, 84, 197, 58, 26, 133, 78, 67, 169, 154, 105, 230, 72, 19, 20, 5, 56, 253, 135, 51, 224, 80, 20, 192, 92, 56, 106, 14, 69, 33, 162, 221, 64, 124, 26, 14, 196, 167, 165, 99, 129, 161, 40, 64, 136, 59, 149, 78, 195, 83, 233, 116, 238, 162, 158, 7, 81, 95, 57, 201, 6, 40, 10, 161, 190, 114, 217, 119, 21, 180, 33, 211, 43, 71, 23, 161, 40, 0, 157, 185, 145, 74, 22, 140, 84, 50, 215, 94, 103, 97, 123, 157, 185, 46, 64, 22, 118, 1, 50, 119, 104, 178, 50, 56, 57, 102, 174, 161, 204, 194, 134, 50, 115, 77, 95, 22, 54, 125, 153, 59, 238, 89, 120, 220, 51, 55, 233, 150, 133, 147, 110, 249, 216, 161, 27, 138, 66, 208, 133, 51, 56, 80, 228, 151, 169, 220, 93, 109, 200, 233, 106, 3, 2, 158, 37, 14, 29, 80, 20, 0, 30, 58, 6, 10, 138, 252, 201, 228, 153, 27, 167, 204, 194, 113, 202, 172, 112, 233, 40, 130, 252, 152, 185, 203, 30, 51, 186, 236, 65, 160, 221, 140, 222, 44, 156, 209, 155, 185, 134, 100, 22, 54, 36, 51, 215, 133, 159, 149, 33, 95, 100, 230, 250, 0, 179, 176, 15, 48, 115, 69, 123, 22, 22, 237, 153, 235, 3, 204, 194, 62, 192, 204, 85, 132, 89, 88, 17, 102, 174, 34, 204, 194, 138, 48, 91, 184, 3, 191, 8, 14, 124, 49, 117, 160, 161, 40, 4, 237, 26, 135, 34, 108, 28, 10, 151, 131, 69, 152, 131, 165, 59, 179, 151, 225, 153, 189, 116, 231, 223, 114, 16, 76, 53, 148, 238, 4, 92, 134, 39, 96, 216, 230, 100, 67, 15, 131, 114, 82, 14, 29, 247, 5, 138, 66, 208, 35, 103, 33, 1, 138, 252, 58, 92, 78, 28, 231, 25, 138, 66, 168, 93, 135, 184, 12, 59, 196, 229, 31, 78, 236, 1, 69, 65, 104, 23, 247, 31, 97, 220, 110, 102, 162, 44, 67, 43, 117, 165, 171, 11, 101, 88, 23, 74, 87, 23, 202, 176, 46, 148, 110, 128, 85, 134, 3, 172, 242, 194, 49, 84, 80, 132, 161, 47, 16, 176, 219, 199, 139, 81, 16, 216, 149, 237, 139, 176, 108, 47, 28, 207, 1, 138, 2, 208, 213, 212, 9, 154, 160, 200, 111, 2, 43, 215, 94, 86, 225, 148, 71, 229, 102, 22, 171, 112, 102, 177, 114, 215, 29, 170, 240, 186, 67, 229, 230, 33, 171, 112, 30, 178, 114, 3, 155, 42, 28, 216, 84, 87, 206, 68, 9, 69, 1, 232, 249, 208, 153, 221, 161, 40, 8, 237, 80, 2, 69, 33, 104, 215, 207, 152, 135, 253, 140, 185, 59, 103, 207, 195, 115, 246, 188, 112, 132, 27, 138, 252, 3, 63, 191, 114, 59, 121, 69, 58, 57, 191, 66, 192, 46, 29, 212, 151, 66, 192, 176, 131, 212, 2, 230, 155, 74, 253, 68, 159, 143, 29, 162, 161, 200, 79, 244, 249, 216, 69, 61, 174, 66, 192, 238, 220, 116, 30, 158, 155, 206, 221, 185, 233, 60, 60, 55, 157, 151, 206, 230, 44, 40, 10, 64, 139, 125, 141, 4, 154, 239, 113, 244, 250, 242, 151, 99, 135, 108, 40, 10, 160, 190, 156, 58, 100, 67, 81, 16, 218, 209, 95, 40, 242, 243, 239, 210, 245, 4, 46, 195, 158, 192, 194, 205, 176, 47, 194, 25, 246, 133, 203, 237, 69, 152, 219, 98, 175, 39, 133, 158, 39, 1, 254, 93, 165, 142, 35, 0, 69, 1, 212, 87, 19, 135, 35, 80, 228, 231, 200, 213, 212, 25, 26, 40, 10, 0, 187, 93, 188, 10, 119, 241, 202, 181, 172, 87, 165, 135, 14, 124, 220, 110, 191, 74, 138, 184, 202, 139, 253, 126, 146, 85, 105, 181, 176, 95, 216, 5, 231, 58, 213, 25, 198, 87, 201, 48, 238, 45, 84, 141, 223, 138, 24, 110, 246, 125, 29, 207, 228, 101, 186, 112, 218, 226, 183, 36, 30, 179, 146, 54, 190, 11, 155, 86, 123, 147, 92, 86, 237, 184, 24, 146, 51, 166, 21, 191, 135, 216, 57, 187, 33, 177, 193, 65, 223, 78, 176, 117, 114, 96, 182, 42, 22, 232, 228, 153, 58, 130, 45, 78, 98, 236, 240, 118, 54, 50, 69, 64, 39, 234, 243, 243, 159, 252, 128, 131, 58, 238, 121, 205, 168, 224, 71, 216, 18, 251, 134, 195, 36, 74, 179, 178, 138, 179, 94, 146, 15, 120, 95, 15, 171, 124, 38, 200, 97, 244, 122, 143, 184, 235, 134, 209, 213, 179, 162, 77, 56, 185, 161, 154, 228, 116, 195, 45, 199, 145, 190, 90, 235, 58, 196, 61, 126, 27, 114, 59, 185, 204, 62, 27, 251, 42, 77, 193, 191, 34, 255, 218, 208, 7, 115, 63, 96, 128, 97, 229, 114, 134, 117, 56, 55, 232, 81, 117, 81, 227, 101, 146, 73, 149, 250, 136, 109, 186, 234, 136, 111, 137, 175, 187, 152, 157, 92, 81, 111, 161, 32, 7, 191, 103, 77, 94, 202, 50, 19, 175, 237, 96, 42, 213, 28, 114, 67, 157, 255, 114, 13, 131, 3, 154, 240, 3, 116, 112, 87, 2, 189, 125, 14, 129, 244, 242, 108, 144, 14, 231, 10, 200, 92, 44, 136, 96, 46, 138, 180, 242, 63, 63, 6, 162, 83, 73, 194, 73, 163, 78, 208, 42, 186, 55, 182, 37, 13, 215, 168, 237, 148, 5, 91, 223, 59, 11, 216, 219, 77, 87, 84, 222, 22, 121, 149, 123, 187, 250, 50, 169, 8, 80, 181, 152, 37, 237, 181, 142, 104, 76, 242, 224, 117, 60, 78, 246, 212, 139, 21, 83, 214, 90, 155, 98, 183, 165, 152, 35, 34, 66, 32, 148, 181, 145, 188, 112, 208, 128, 192, 112, 91, 232, 2, 215, 139, 140, 128, 89, 34, 51, 2, 104, 9, 55, 29, 61, 255, 56, 108, 117, 154, 233, 212, 182, 47, 177, 133, 109, 195, 70, 74, 236, 132, 232, 25, 50, 98, 111, 146, 11, 143, 201, 236, 68, 23, 194, 24, 41, 11, 1, 22, 253, 94, 179, 86, 140, 105, 247, 154, 28, 114, 105, 2, 248, 17, 178, 165, 118, 200, 192, 170, 186, 135, 163, 116, 106, 187, 43, 46, 245, 250, 242, 82, 62, 205, 217, 194, 217, 200, 128, 223, 166, 33, 7, 25, 109, 145, 217, 246, 23, 121, 241, 60, 238, 141, 248, 201, 213, 211, 184, 55, 30, 100, 194, 218, 63, 45, 134, 145, 231, 230, 17, 16, 192, 215, 242, 250, 29, 115, 117, 201, 189, 253, 82, 182, 254, 58, 240, 38, 144, 78, 212, 98, 141, 181, 196, 197, 39, 208, 32, 251, 10, 114, 200, 38, 227, 215, 222, 217, 223, 153, 169, 96, 150, 26, 197, 108, 22, 82, 240, 254, 185, 126, 197, 70, 175, 113, 31, 246, 88, 155, 160, 114, 136, 25, 254, 238, 124, 223, 137, 158, 37, 189, 124, 58, 75, 39, 9, 99, 84, 123, 179, 19, 225, 42, 70, 87, 153, 45, 46, 196, 173, 73, 222, 254, 164, 90, 169, 20, 61, 242, 74, 5, 126, 180, 214, 186, 51, 66, 223, 96, 82, 139, 234, 61, 204, 56, 29, 213, 176, 245, 158, 11, 129, 98, 3, 156, 24, 234, 241, 88, 39, 158, 21, 233, 11, 241, 182, 34, 168, 99, 93, 183, 64, 104, 240, 200, 142, 160, 96, 113, 188, 117, 162, 190, 109, 158, 160, 161, 147, 62, 4, 126, 231, 227, 30, 27, 159, 196, 121, 49, 134, 49, 12, 28, 244, 125, 149, 139, 27, 143, 228, 171, 60, 236, 183, 100, 58, 172, 145, 213, 56, 245, 112, 43, 30, 242, 54, 233, 125, 82, 55, 190, 145, 232, 14, 46, 18, 195, 108, 48, 74, 165, 105, 179, 175, 53, 244, 95, 173, 108, 249, 187, 14, 107, 55, 112, 73, 232, 90, 102, 248, 44, 171, 41, 46, 79, 194, 111, 177, 84, 164, 226, 235, 196, 124, 216, 162, 221, 37, 200, 149, 98, 91, 215, 144, 220, 180, 125, 247, 46, 108, 242, 146, 84, 140, 214, 185, 248, 73, 10, 78, 90, 190, 128, 185, 4, 124, 131, 41, 158, 124, 164, 25, 97, 165, 226, 178, 21, 129, 11, 29, 112, 151, 173, 34, 15, 64, 205, 49, 172, 74, 151, 255, 15, 99, 176, 233, 111, 244, 77, 252, 230, 110, 91, 132, 42, 236, 175, 191, 181, 195, 120, 144, 200, 183, 117, 74, 166, 248, 27, 102, 15, 56, 162, 251, 226, 251, 253, 186, 182, 185, 73, 245, 48, 17, 144, 215, 213, 83, 151, 62, 13, 121, 80, 121, 148, 91, 116, 209, 235, 178, 134, 146, 32, 249, 112, 9, 57, 250, 78, 203, 248, 180, 84, 213, 215, 224, 126, 150, 239, 55, 55, 255, 190, 245, 253, 247, 219, 223, 60, 252, 251, 195, 205, 239, 191, 223, 114, 16, 25, 7, 140, 98, 70, 204, 124, 57, 201, 79, 227, 137, 25, 58, 196, 68, 53, 103, 227, 193, 213, 119, 147, 41, 150, 248, 145, 81, 190, 249, 209, 73, 152, 122, 140, 98, 122, 62, 228, 47, 33, 127, 90, 150, 233, 48, 107, 87, 204, 48, 65, 212, 55, 72, 139, 178, 58, 204, 231, 69, 143, 188, 119, 143, 59, 147, 198, 171, 224, 192, 136, 253, 142, 177, 123, 140, 47, 16, 81, 124, 203, 233, 53, 73, 242, 254, 160, 126, 110, 77, 109, 48, 103, 9, 10, 176, 25, 117, 222, 67, 138, 193, 118, 197, 205, 241, 112, 189, 132, 85, 28, 48, 107, 204, 233, 174, 210, 12, 207, 97, 252, 94, 213, 34, 159, 226, 126, 26, 92, 107, 20, 112, 156, 44, 74, 126, 195, 42, 131, 70, 87, 121, 192, 101, 125, 109, 64, 98, 129, 139, 203, 140, 117, 37, 114, 157, 49, 238, 181, 186, 205, 21, 221, 77, 204, 95, 27, 31, 189, 222, 63, 60, 220, 127, 243, 18, 142, 233, 94, 211, 106, 179, 36, 235, 179, 185, 239, 249, 101, 47, 225, 214, 142, 129, 72, 96, 3, 39, 221, 8, 211, 130, 115, 115, 50, 38, 130, 117, 2, 19, 123, 172, 171, 89, 87, 253, 82, 15, 13, 99, 233, 39, 101, 47, 170, 121, 115, 221, 51, 246, 188, 72, 103, 224, 168, 2, 179, 58, 170, 217, 224, 125, 177, 12, 157, 115, 157, 47, 148, 162, 248, 202, 127, 45, 76, 149, 31, 75, 220, 16, 78, 67, 99, 250, 183, 117, 163, 148, 215, 125, 196, 84, 184, 140, 222, 213, 172, 246, 55, 238, 25, 26, 235, 138, 42, 122, 101, 140, 102, 52, 185, 133, 139, 200, 188, 131, 242, 94, 136, 8, 17, 215, 218, 240, 106, 74, 149, 18, 177, 190, 14, 183, 117, 217, 202, 75, 163, 32, 208, 90, 199, 116, 60, 227, 227, 112, 152, 192, 117, 51, 237, 44, 158, 194, 219, 35, 248, 15, 114, 169, 42, 7, 53, 70, 2, 188, 45, 162, 170, 186, 192, 247, 74, 8, 94, 25, 238, 249, 227, 214, 79, 96, 161, 239, 132, 80, 16, 52, 216, 33, 113, 136, 246, 225, 37, 117, 94, 255, 125, 235, 59, 126, 47, 58, 0, 224, 59, 2, 199, 92, 15, 142, 114, 164, 220, 188, 171, 8, 68, 10, 186, 17, 152, 247, 239, 103, 16, 248, 190, 127, 143, 111, 53, 229, 148, 107, 65, 21, 249, 194, 14, 86, 112, 29, 254, 219, 207, 24, 81, 59, 146, 177, 98, 224, 68, 211, 72, 177, 236, 91, 245, 121, 105, 7, 104, 239, 112, 234, 124, 118, 95, 12, 222, 75, 60, 120, 195, 47, 122, 240, 134, 203, 6, 111, 248, 37, 14, 222, 16, 6, 111, 120, 215, 131, 247, 138, 191, 114, 17, 105, 30, 30, 180, 6, 189, 174, 101, 188, 51, 21, 175, 96, 201, 17, 245, 86, 156, 201, 187, 98, 89, 213, 85, 174, 199, 130, 250, 66, 67, 91, 158, 219, 176, 164, 153, 226, 211, 1, 3, 161, 6, 86, 62, 244, 92, 147, 127, 237, 138, 34, 237, 165, 78, 142, 29, 12, 100, 223, 214, 44, 235, 41, 43, 222, 147, 74, 17, 30, 173, 151, 255, 134, 163, 53, 92, 62, 90, 195, 47, 103, 180, 68, 2, 225, 125, 57, 63, 21, 175, 131, 100, 29, 140, 11, 158, 56, 209, 161, 244, 222, 40, 233, 141, 69, 131, 123, 121, 82, 244, 120, 90, 212, 74, 32, 176, 202, 124, 172, 212, 171, 69, 173, 244, 2, 247, 2, 25, 12, 241, 255, 100, 136, 115, 8, 13, 146, 24, 135, 147, 64, 223, 152, 251, 60, 131, 52, 105, 59, 201, 250, 238, 133, 199, 128, 125, 7, 85, 7, 210, 77, 101, 6, 22, 79, 84, 27, 228, 58, 127, 114, 209, 191, 34, 164, 19, 109, 174, 117, 184, 147, 104, 161, 16, 20, 212, 34, 120, 14, 87, 7, 123, 170, 115, 79, 79, 197, 0, 32, 81, 136, 166, 199, 26, 59, 22, 61, 233, 142, 27, 56, 35, 16, 60, 30, 81, 117, 2, 137, 2, 92, 253, 57, 190, 0, 217, 84, 70, 88, 73, 122, 224, 80, 9, 194, 207, 73, 145, 37, 19, 214, 199, 162, 35, 240, 125, 136, 54, 59, 108, 54, 204, 89, 212, 206, 29, 125, 40, 240, 203, 18, 143, 174, 136, 72, 193, 31, 230, 40, 57, 162, 21, 136, 160, 182, 252, 201, 83, 54, 225, 49, 18, 238, 139, 102, 54, 52, 254, 251, 212, 109, 35, 234, 106, 139, 55, 16, 225, 16, 19, 86, 8, 165, 13, 244, 246, 218, 219, 43, 68, 35, 177, 47, 211, 171, 36, 160, 54, 137, 80, 7, 201, 48, 247, 10, 112, 168, 137, 85, 130, 184, 183, 50, 107, 160, 228, 111, 211, 48, 27, 145, 229, 85, 142, 104, 157, 163, 238, 240, 129, 87, 180, 136, 196, 232, 43, 73, 74, 141, 134, 36, 74, 61, 56, 121, 93, 221, 218, 154, 166, 138, 160, 122, 188, 139, 73, 147, 131, 171, 222, 176, 90, 39, 173, 154, 90, 46, 177, 4, 103, 35, 145, 181, 229, 149, 14, 254, 237, 69, 182, 177, 188, 58, 194, 170, 135, 209, 211, 131, 73, 218, 75, 254, 122, 198, 91, 24, 50, 42, 174, 242, 142, 123, 45, 87, 128, 98, 61, 194, 6, 156, 97, 36, 98, 104, 158, 81, 203, 204, 205, 161, 180, 220, 245, 168, 165, 105, 247, 32, 230, 79, 40, 218, 114, 22, 103, 182, 10, 105, 187, 105, 148, 167, 153, 197, 229, 200, 150, 201, 173, 28, 252, 143, 98, 102, 1, 119, 19, 145, 5, 184, 198, 230, 181, 151, 247, 147, 183, 57, 99, 223, 211, 170, 61, 203, 203, 85, 164, 245, 176, 70, 86, 25, 170, 84, 38, 28, 140, 72, 1, 126, 199, 170, 30, 18, 105, 230, 25, 6, 85, 151, 9, 2, 132, 121, 250, 247, 147, 93, 94, 201, 177, 69, 158, 21, 12, 158, 67, 244, 173, 191, 168, 110, 143, 226, 98, 143, 117, 253, 41, 99, 205, 97, 71, 55, 129, 12, 179, 192, 192, 72, 184, 124, 246, 221, 38, 167, 67, 148, 60, 129, 146, 31, 95, 188, 32, 148, 173, 71, 91, 92, 163, 188, 228, 241, 122, 168, 223, 226, 69, 131, 43, 211, 6, 141, 32, 250, 36, 26, 78, 224, 158, 32, 80, 22, 113, 10, 95, 188, 120, 81, 71, 136, 122, 185, 136, 232, 83, 87, 246, 114, 45, 250, 27, 251, 246, 144, 97, 91, 215, 13, 116, 101, 3, 107, 172, 108, 243, 114, 107, 147, 125, 212, 142, 155, 94, 206, 162, 99, 230, 202, 47, 89, 41, 147, 221, 42, 146, 89, 18, 87, 226, 30, 229, 187, 18, 51, 42, 95, 2, 181, 97, 81, 70, 39, 85, 164, 90, 223, 126, 143, 227, 124, 6, 248, 55, 45, 135, 172, 78, 123, 43, 122, 252, 56, 218, 254, 110, 205, 31, 239, 255, 93, 13, 3, 180, 158, 69, 95, 71, 109, 93, 129, 113, 107, 11, 145, 119, 20, 153, 87, 145, 243, 55, 185, 61, 122, 228, 44, 139, 177, 250, 214, 173, 217, 71, 240, 70, 213, 67, 227, 43, 50, 193, 127, 66, 174, 186, 231, 245, 104, 157, 67, 171, 142, 115, 217, 116, 232, 141, 172, 71, 62, 163, 160, 214, 110, 219, 119, 53, 78, 10, 33, 3, 121, 147, 92, 8, 24, 189, 83, 64, 194, 253, 146, 149, 241, 0, 18, 122, 239, 146, 50, 41, 206, 147, 254, 225, 36, 175, 218, 102, 69, 154, 153, 194, 195, 134, 160, 91, 142, 109, 87, 207, 60, 189, 109, 186, 229, 51, 80, 141, 111, 224, 36, 219, 64, 244, 222, 3, 206, 4, 80, 76, 86, 100, 245, 153, 63, 194, 178, 224, 51, 232, 144, 125, 162, 245, 94, 39, 213, 40, 239, 239, 15, 196, 86, 184, 190, 220, 7, 231, 132, 184, 248, 211, 114, 201, 165, 111, 183, 84, 99, 39, 152, 250, 50, 169, 68, 133, 23, 108, 226, 35, 236, 21, 91, 238, 54, 137, 71, 33, 214, 10, 116, 77, 166, 136, 15, 182, 3, 21, 183, 150, 153, 126, 250, 82, 148, 224, 86, 71, 178, 189, 82, 120, 208, 64, 133, 154, 27, 8, 75, 245, 230, 2, 178, 103, 9, 177, 91, 189, 148, 132, 114, 67, 24, 96, 242, 134, 137, 27, 204, 41, 156, 46, 103, 66, 97, 116, 202, 25, 5, 222, 7, 40, 230, 152, 93, 53, 165, 176, 34, 209, 27, 152, 79, 30, 59, 29, 186, 201, 236, 161, 241, 89, 57, 22, 53, 87, 236, 170, 249, 131, 181, 173, 166, 148, 93, 53, 129, 216, 111, 173, 64, 124, 217, 182, 51, 35, 215, 53, 154, 41, 37, 192, 80, 163, 49, 105, 45, 53, 123, 58, 106, 250, 134, 162, 71, 213, 181, 58, 156, 214, 123, 103, 108, 43, 39, 22, 233, 225, 142, 250, 184, 0, 107, 22, 223, 217, 180, 116, 52, 138, 43, 250, 56, 198, 158, 120, 237, 38, 143, 102, 59, 52, 106, 55, 121, 108, 19, 20, 219, 1, 20, 106, 246, 54, 18, 122, 187, 13, 30, 196, 27, 216, 80, 191, 234, 87, 64, 253, 53, 196, 214, 136, 61, 249, 187, 110, 51, 7, 170, 227, 71, 22, 200, 203, 172, 218, 150, 187, 113, 67, 138, 141, 24, 224, 82, 119, 158, 11, 230, 145, 24, 238, 96, 148, 0, 49, 197, 158, 242, 187, 219, 218, 3, 39, 111, 10, 86, 25, 5, 103, 124, 81, 238, 97, 146, 240, 7, 210, 243, 226, 47, 226, 91, 67, 206, 134, 187, 164, 27, 89, 235, 175, 252, 221, 177, 104, 205, 207, 54, 54, 114, 35, 179, 145, 37, 223, 66, 172, 2, 220, 123, 203, 69, 94, 45, 250, 179, 18, 203, 228, 72, 32, 72, 127, 162, 109, 20, 188, 144, 207, 146, 116, 179, 137, 40, 183, 60, 35, 228, 132, 49, 119, 64, 235, 151, 2, 246, 182, 39, 163, 8, 249, 235, 9, 119, 98, 95, 16, 247, 248, 230, 168, 193, 82, 190, 240, 88, 74, 53, 52, 27, 179, 121, 57, 82, 56, 30, 89, 194, 111, 47, 255, 155, 12, 51, 173, 206, 62, 178, 189, 174, 242, 192, 215, 162, 255, 22, 142, 186, 72, 254, 113, 215, 156, 59, 240, 143, 66, 56, 92, 12, 255, 165, 93, 125, 233, 224, 123, 68, 61, 252, 122, 54, 230, 127, 76, 22, 157, 8, 25, 103, 46, 213, 114, 242, 233, 136, 229, 43, 77, 68, 157, 54, 20, 241, 5, 207, 236, 31, 50, 31, 173, 19, 109, 108, 136, 228, 76, 90, 205, 133, 2, 34, 157, 16, 27, 73, 14, 6, 135, 24, 0, 226, 123, 252, 219, 81, 147, 94, 158, 143, 249, 222, 94, 189, 128, 161, 90, 195, 185, 179, 248, 130, 64, 240, 58, 27, 172, 20, 155, 109, 238, 68, 78, 14, 147, 161, 82, 187, 163, 92, 164, 179, 218, 12, 112, 195, 73, 237, 217, 240, 181, 121, 52, 227, 253, 8, 238, 160, 200, 33, 176, 53, 163, 102, 103, 38, 223, 52, 146, 12, 241, 20, 196, 40, 68, 218, 139, 228, 196, 106, 148, 252, 92, 87, 152, 92, 233, 223, 215, 126, 10, 24, 120, 171, 171, 222, 211, 43, 24, 179, 107, 61, 216, 152, 134, 26, 121, 236, 31, 120, 107, 147, 160, 192, 129, 187, 76, 196, 194, 223, 121, 39, 76, 50, 104, 20, 247, 27, 242, 8, 147, 239, 236, 162, 184, 14, 75, 63, 117, 66, 152, 119, 179, 197, 83, 55, 219, 183, 201, 24, 73, 151, 127, 131, 224, 110, 53, 117, 64, 154, 121, 15, 181, 14, 200, 3, 130, 226, 129, 7, 69, 77, 254, 138, 80, 205, 153, 33, 89, 210, 137, 60, 179, 179, 203, 216, 211, 116, 216, 56, 110, 85, 74, 248, 152, 85, 122, 210, 98, 99, 73, 61, 58, 86, 208, 122, 252, 223, 252, 153, 183, 165, 73, 154, 141, 111, 208, 22, 84, 11, 183, 38, 158, 122, 219, 203, 39, 253, 27, 52, 23, 110, 202, 223, 204, 32, 189, 76, 110, 208, 78, 85, 5, 27, 130, 71, 254, 60, 67, 60, 73, 123, 229, 234, 109, 165, 193, 166, 82, 127, 75, 229, 148, 73, 218, 234, 237, 240, 106, 193, 182, 228, 83, 255, 220, 86, 164, 227, 228, 6, 13, 242, 122, 225, 22, 229, 227, 192, 66, 217, 13, 218, 155, 135, 197, 131, 63, 243, 183, 52, 187, 73, 75, 179, 154, 150, 102, 86, 75, 207, 203, 94, 60, 99, 49, 44, 3, 61, 157, 87, 201, 175, 16, 24, 182, 207, 233, 139, 88, 103, 115, 176, 193, 216, 226, 159, 147, 228, 4, 123, 254, 138, 47, 153, 40, 80, 199, 53, 200, 231, 149, 70, 130, 103, 219, 222, 104, 158, 141, 213, 138, 202, 166, 229, 82, 139, 215, 107, 70, 176, 109, 77, 181, 241, 8, 94, 28, 105, 199, 67, 170, 85, 245, 194, 201, 251, 173, 251, 182, 155, 104, 218, 95, 111, 20, 244, 42, 148, 29, 68, 33, 43, 5, 46, 222, 255, 250, 143, 121, 94, 61, 186, 111, 57, 153, 184, 35, 41, 76, 215, 254, 176, 157, 199, 111, 8, 118, 215, 231, 170, 168, 214, 31, 249, 170, 24, 102, 160, 61, 88, 119, 208, 61, 42, 70, 26, 163, 79, 44, 227, 172, 55, 202, 233, 30, 151, 219, 4, 242, 178, 201, 251, 143, 227, 8, 80, 238, 182, 238, 51, 254, 121, 229, 82, 180, 200, 198, 160, 245, 4, 96, 14, 133, 80, 199, 1, 235, 202, 252, 126, 22, 45, 230, 144, 174, 102, 255, 223, 41, 165, 128, 59, 226, 104, 107, 168, 149, 205, 218, 228, 66, 221, 48, 197, 144, 138, 106, 227, 124, 212, 221, 209, 11, 88, 107, 200, 21, 141, 54, 167, 150, 207, 201, 243, 98, 114, 199, 34, 48, 42, 146, 65, 13, 149, 188, 193, 165, 18, 144, 84, 55, 56, 221, 118, 88, 115, 186, 237, 48, 124, 186, 141, 53, 230, 61, 104, 166, 114, 232, 75, 78, 183, 177, 234, 226, 116, 155, 130, 111, 116, 186, 109, 89, 163, 31, 249, 116, 27, 207, 31, 122, 15, 183, 65, 119, 204, 250, 128, 34, 231, 70, 135, 219, 48, 42, 121, 184, 77, 180, 107, 229, 8, 86, 57, 219, 166, 18, 159, 55, 61, 219, 198, 43, 232, 63, 134, 219, 72, 248, 142, 216, 72, 244, 249, 118, 250, 231, 226, 40, 30, 57, 171, 118, 80, 35, 100, 7, 106, 13, 195, 160, 96, 101, 77, 22, 47, 76, 5, 103, 225, 226, 128, 137, 137, 67, 18, 93, 152, 144, 88, 247, 248, 161, 97, 14, 166, 151, 79, 14, 196, 122, 130, 191, 123, 231, 73, 177, 104, 170, 96, 225, 5, 27, 212, 217, 198, 139, 53, 53, 253, 21, 41, 112, 223, 224, 17, 217, 176, 63, 45, 171, 95, 238, 210, 13, 29, 57, 179, 79, 196, 84, 148, 57, 138, 131, 186, 195, 50, 214, 140, 239, 137, 41, 255, 203, 32, 252, 175, 141, 153, 218, 235, 184, 145, 96, 170, 62, 154, 98, 31, 201, 78, 50, 32, 203, 27, 27, 11, 111, 108, 44, 18, 154, 236, 139, 155, 197, 28, 255, 42, 181, 235, 224, 120, 108, 229, 45, 171, 164, 132, 149, 10, 143, 166, 227, 145, 58, 234, 72, 28, 236, 47, 147, 92, 75, 207, 239, 25, 36, 222, 188, 7, 58, 244, 69, 228, 218, 57, 106, 110, 24, 252, 34, 101, 193, 138, 84, 230, 82, 185, 161, 29, 185, 217, 236, 115, 203, 48, 166, 139, 211, 212, 88, 130, 161, 91, 119, 32, 192, 194, 60, 79, 226, 42, 61, 79, 150, 111, 123, 26, 115, 179, 138, 161, 31, 71, 155, 132, 98, 239, 86, 35, 82, 165, 131, 117, 3, 62, 214, 238, 35, 11, 152, 110, 66, 82, 15, 197, 230, 43, 103, 239, 85, 179, 125, 179, 168, 19, 114, 139, 148, 195, 244, 218, 110, 200, 109, 83, 78, 165, 64, 71, 172, 189, 84, 98, 239, 2, 87, 50, 78, 141, 163, 102, 160, 89, 140, 68, 52, 111, 17, 81, 63, 8, 203, 57, 236, 74, 95, 193, 25, 250, 228, 243, 212, 205, 236, 118, 203, 238, 163, 43, 246, 159, 212, 74, 15, 8, 17, 159, 206, 76, 139, 35, 1, 252, 77, 211, 250, 50, 15, 190, 234, 246, 30, 253, 100, 62, 49, 25, 49, 83, 117, 15, 54, 9, 204, 146, 30, 115, 13, 16, 60, 204, 254, 46, 82, 172, 242, 201, 172, 242, 44, 138, 241, 200, 62, 158, 49, 17, 224, 203, 1, 118, 92, 127, 187, 153, 164, 76, 38, 76, 6, 229, 58, 195, 143, 121, 62, 73, 226, 172, 221, 120, 70, 113, 22, 231, 5, 50, 59, 95, 0, 29, 19, 235, 57, 162, 182, 189, 164, 36, 59, 231, 30, 222, 51, 98, 247, 84, 50, 102, 175, 173, 160, 157, 37, 67, 185, 84, 200, 248, 160, 64, 216, 47, 202, 140, 167, 199, 25, 232, 59, 208, 195, 190, 121, 52, 254, 105, 80, 227, 179, 126, 123, 198, 80, 166, 44, 154, 72, 190, 36, 215, 204, 37, 106, 133, 121, 45, 235, 127, 113, 142, 217, 0, 17, 101, 43, 188, 238, 107, 99, 125, 71, 53, 62, 174, 87, 230, 156, 243, 65, 227, 226, 104, 141, 207, 221, 18, 16, 30, 145, 68, 43, 16, 65, 209, 228, 171, 54, 127, 73, 249, 228, 61, 251, 34, 133, 148, 80, 246, 165, 73, 234, 18, 113, 4, 217, 173, 21, 70, 143, 28, 118, 183, 66, 2, 184, 90, 110, 104, 69, 249, 251, 40, 190, 79, 203, 161, 254, 99, 139, 152, 103, 127, 188, 65, 132, 37, 75, 144, 115, 191, 145, 203, 243, 105, 66, 210, 58, 111, 0, 9, 210, 13, 76, 23, 87, 161, 131, 1, 243, 27, 226, 162, 55, 122, 46, 54, 93, 136, 179, 11, 122, 211, 192, 103, 50, 91, 33, 146, 26, 27, 47, 217, 183, 59, 146, 43, 14, 226, 93, 221, 232, 162, 157, 158, 116, 91, 185, 166, 153, 108, 45, 127, 178, 203, 3, 164, 58, 52, 99, 11, 126, 19, 115, 20, 164, 36, 195, 125, 178, 182, 168, 193, 115, 17, 195, 101, 116, 118, 28, 211, 93, 237, 10, 118, 19, 227, 66, 1, 91, 200, 170, 137, 144, 13, 206, 44, 224, 17, 186, 189, 21, 251, 159, 28, 142, 155, 37, 179, 88, 92, 239, 246, 185, 164, 78, 182, 79, 178, 140, 64, 218, 29, 229, 31, 88, 3, 176, 241, 72, 53, 227, 132, 244, 173, 78, 139, 135, 244, 106, 43, 138, 230, 71, 19, 81, 196, 75, 145, 114, 7, 213, 38, 119, 148, 54, 79, 204, 131, 119, 120, 205, 66, 65, 217, 190, 252, 152, 239, 96, 174, 245, 195, 96, 121, 225, 29, 156, 39, 76, 102, 52, 162, 145, 56, 189, 209, 142, 220, 24, 104, 183, 143, 226, 17, 32, 239, 80, 110, 142, 241, 136, 207, 187, 128, 244, 240, 235, 107, 190, 160, 196, 52, 208, 179, 74, 86, 122, 59, 148, 149, 126, 21, 151, 85, 3, 51, 253, 185, 189, 204, 91, 155, 107, 212, 207, 79, 110, 178, 17, 35, 61, 25, 182, 110, 180, 69, 178, 108, 182, 89, 87, 250, 34, 76, 53, 73, 167, 49, 139, 150, 117, 20, 18, 56, 123, 72, 12, 179, 54, 182, 79, 196, 188, 223, 237, 126, 42, 91, 11, 87, 139, 255, 245, 146, 103, 88, 156, 224, 186, 204, 79, 147, 58, 187, 31, 8, 81, 166, 241, 172, 153, 19, 233, 98, 254, 87, 203, 155, 153, 220, 144, 149, 250, 109, 24, 210, 79, 249, 176, 255, 234, 59, 196, 177, 204, 221, 133, 207, 83, 145, 68, 70, 88, 86, 72, 44, 189, 75, 250, 243, 30, 30, 146, 232, 6, 22, 244, 110, 101, 189, 101, 83, 135, 183, 96, 126, 230, 192, 187, 32, 4, 125, 20, 193, 38, 182, 27, 78, 145, 120, 169, 15, 108, 100, 253, 214, 50, 202, 155, 230, 103, 220, 235, 205, 167, 115, 117, 124, 163, 217, 9, 26, 38, 111, 235, 235, 39, 206, 98, 134, 79, 144, 41, 250, 160, 16, 163, 115, 105, 168, 198, 178, 64, 14, 129, 214, 202, 241, 187, 116, 56, 170, 190, 108, 97, 230, 36, 126, 113, 18, 77, 168, 250, 242, 197, 90, 186, 20, 183, 22, 238, 110, 215, 22, 110, 175, 47, 242, 217, 101, 251, 60, 41, 202, 164, 193, 105, 217, 143, 226, 219, 182, 28, 74, 238, 40, 24, 156, 166, 253, 254, 68, 220, 16, 35, 61, 198, 193, 36, 207, 249, 101, 47, 209, 127, 243, 144, 64, 15, 141, 216, 125, 127, 193, 239, 146, 101, 227, 35, 190, 194, 137, 40, 129, 67, 150, 184, 83, 235, 28, 174, 128, 215, 2, 35, 170, 117, 241, 110, 78, 141, 216, 164, 213, 249, 79, 43, 112, 227, 120, 12, 8, 255, 137, 64, 84, 37, 240, 32, 53, 36, 126, 44, 42, 0, 33, 186, 45, 143, 40, 132, 214, 123, 15, 3, 119, 163, 124, 174, 64, 15, 237, 97, 192, 210, 193, 201, 92, 217, 237, 252, 207, 182, 132, 79, 189, 45, 65, 156, 238, 18, 231, 145, 73, 35, 130, 182, 46, 152, 42, 124, 94, 253, 115, 122, 212, 248, 110, 10, 179, 202, 202, 127, 202, 93, 136, 122, 119, 133, 103, 161, 149, 102, 96, 198, 120, 129, 55, 51, 135, 140, 144, 34, 132, 220, 228, 195, 124, 154, 252, 5, 247, 198, 65, 183, 190, 184, 197, 173, 18, 17, 245, 111, 185, 51, 110, 201, 198, 56, 115, 35, 3, 17, 93, 247, 42, 115, 195, 94, 222, 214, 23, 149, 32, 20, 20, 173, 146, 34, 180, 46, 144, 194, 43, 29, 189, 201, 188, 207, 186, 247, 197, 47, 227, 212, 170, 165, 201, 25, 210, 101, 28, 209, 185, 143, 158, 20, 180, 174, 238, 175, 95, 202, 249, 188, 171, 52, 200, 240, 91, 151, 161, 3, 134, 195, 120, 42, 14, 23, 252, 51, 41, 114, 91, 38, 64, 97, 253, 75, 206, 244, 158, 147, 177, 119, 118, 168, 83, 49, 113, 37, 63, 92, 226, 210, 46, 249, 189, 243, 29, 200, 200, 12, 50, 60, 67, 44, 205, 52, 238, 249, 148, 18, 79, 156, 123, 129, 171, 148, 55, 45, 187, 183, 213, 137, 246, 176, 185, 27, 136, 4, 209, 32, 51, 152, 7, 238, 197, 236, 216, 210, 14, 154, 25, 88, 214, 208, 64, 53, 164, 174, 70, 50, 107, 219, 192, 141, 61, 18, 24, 41, 222, 12, 52, 47, 66, 106, 205, 43, 247, 76, 223, 89, 5, 249, 162, 131, 180, 74, 166, 165, 100, 175, 65, 99, 177, 17, 213, 124, 84, 171, 119, 77, 62, 183, 199, 112, 123, 36, 183, 169, 142, 98, 46, 102, 158, 196, 105, 108, 33, 157, 68, 6, 150, 100, 86, 231, 37, 186, 16, 137, 1, 188, 76, 42, 97, 246, 218, 114, 68, 240, 107, 228, 144, 209, 161, 245, 238, 133, 46, 67, 225, 135, 4, 9, 118, 252, 238, 48, 192, 79, 16, 161, 73, 147, 156, 97, 177, 183, 47, 194, 39, 112, 96, 69, 213, 149, 203, 114, 170, 113, 241, 110, 79, 107, 147, 32, 157, 56, 248, 53, 29, 174, 255, 141, 20, 229, 155, 45, 15, 2, 254, 22, 65, 247, 100, 11, 124, 172, 211, 45, 130, 52, 232, 146, 185, 11, 67, 156, 111, 65, 104, 175, 9, 7, 132, 249, 151, 149, 124, 239, 169, 16, 250, 195, 122, 224, 201, 87, 195, 167, 169, 11, 164, 240, 105, 55, 72, 180, 73, 124, 33, 5, 67, 115, 219, 74, 250, 126, 160, 83, 169, 212, 100, 236, 21, 193, 106, 209, 216, 10, 163, 225, 163, 251, 16, 72, 122, 83, 182, 40, 219, 174, 42, 217, 115, 104, 204, 167, 201, 116, 156, 68, 232, 18, 11, 46, 108, 200, 118, 202, 105, 85, 221, 89, 161, 235, 208, 155, 43, 150, 49, 248, 70, 254, 165, 110, 203, 241, 51, 239, 146, 175, 203, 120, 74, 143, 95, 24, 94, 214, 204, 131, 7, 131, 118, 132, 45, 178, 224, 97, 248, 78, 30, 206, 115, 12, 65, 67, 200, 155, 206, 137, 242, 24, 221, 30, 14, 32, 147, 139, 27, 12, 144, 9, 27, 121, 117, 193, 40, 78, 52, 251, 138, 217, 35, 158, 187, 188, 17, 178, 197, 108, 26, 191, 165, 189, 16, 164, 65, 105, 39, 202, 7, 131, 50, 169, 108, 5, 91, 246, 1, 162, 57, 51, 16, 250, 253, 242, 105, 85, 193, 61, 78, 253, 182, 66, 47, 121, 131, 163, 175, 135, 127, 231, 61, 46, 47, 82, 126, 46, 113, 201, 45, 116, 146, 220, 77, 233, 15, 247, 98, 230, 199, 109, 237, 56, 243, 189, 238, 223, 33, 147, 192, 184, 8, 246, 242, 145, 194, 177, 93, 131, 227, 29, 188, 121, 58, 201, 122, 201, 114, 52, 223, 212, 145, 146, 78, 251, 203, 49, 60, 220, 81, 223, 30, 212, 224, 122, 150, 20, 41, 99, 73, 45, 58, 208, 19, 248, 187, 124, 247, 89, 99, 180, 43, 10, 5, 105, 255, 77, 114, 33, 209, 163, 54, 107, 58, 176, 140, 200, 253, 65, 168, 114, 7, 188, 45, 57, 151, 171, 27, 82, 249, 149, 120, 63, 172, 218, 93, 116, 225, 73, 29, 69, 7, 179, 248, 143, 121, 88, 58, 110, 196, 53, 80, 253, 156, 227, 85, 109, 69, 252, 214, 205, 3, 82, 166, 89, 200, 71, 92, 168, 27, 126, 70, 49, 4, 4, 69, 157, 202, 39, 176, 75, 187, 123, 7, 252, 95, 198, 183, 134, 236, 95, 162, 227, 156, 14, 62, 31, 178, 39, 209, 210, 123, 46, 37, 154, 239, 136, 85, 130, 170, 200, 226, 108, 18, 213, 124, 149, 199, 204, 209, 204, 170, 239, 218, 97, 181, 222, 210, 106, 253, 157, 91, 119, 190, 164, 242, 182, 183, 185, 173, 111, 107, 170, 60, 240, 183, 82, 91, 231, 161, 183, 153, 7, 219, 53, 85, 190, 241, 55, 83, 91, 231, 91, 183, 206, 96, 146, 199, 245, 149, 254, 30, 168, 244, 237, 67, 127, 165, 27, 24, 191, 229, 102, 254, 211, 72, 210, 211, 108, 177, 76, 144, 104, 5, 149, 61, 90, 69, 126, 74, 121, 75, 194, 29, 241, 174, 118, 110, 251, 8, 108, 147, 92, 0, 164, 144, 79, 171, 21, 34, 184, 26, 15, 121, 92, 139, 6, 240, 15, 17, 252, 85, 3, 248, 239, 16, 252, 69, 3, 248, 45, 181, 154, 40, 89, 204, 120, 241, 66, 64, 95, 62, 4, 198, 242, 247, 235, 180, 215, 218, 151, 157, 104, 209, 137, 174, 58, 209, 133, 51, 152, 110, 215, 61, 170, 80, 219, 117, 63, 252, 119, 126, 202, 190, 125, 120, 185, 109, 83, 102, 104, 218, 244, 208, 228, 55, 3, 33, 138, 130, 208, 222, 161, 8, 66, 123, 7, 34, 8, 237, 25, 6, 46, 146, 181, 131, 176, 170, 118, 28, 134, 157, 236, 78, 36, 94, 154, 6, 249, 76, 113, 18, 144, 75, 248, 103, 246, 163, 41, 237, 245, 147, 43, 166, 29, 179, 210, 150, 86, 138, 115, 169, 165, 117, 25, 227, 69, 254, 141, 151, 224, 26, 83, 180, 20, 227, 67, 35, 200, 250, 234, 214, 118, 67, 94, 126, 191, 166, 222, 225, 3, 40, 96, 16, 41, 97, 34, 153, 47, 129, 5, 246, 122, 10, 85, 152, 103, 104, 68, 121, 25, 60, 203, 211, 100, 144, 139, 0, 37, 115, 196, 154, 111, 50, 233, 191, 97, 12, 46, 177, 49, 22, 181, 235, 173, 49, 170, 207, 181, 99, 5, 4, 223, 219, 8, 14, 120, 127, 87, 192, 176, 229, 236, 139, 71, 119, 120, 153, 62, 169, 96, 158, 94, 230, 69, 250, 13, 155, 140, 53, 60, 122, 29, 44, 237, 156, 130, 18, 61, 245, 129, 137, 46, 40, 56, 217, 33, 31, 160, 74, 140, 232, 161, 57, 214, 237, 107, 96, 203, 100, 152, 166, 93, 33, 97, 166, 11, 181, 216, 49, 157, 233, 160, 230, 60, 107, 89, 202, 124, 225, 180, 196, 182, 181, 12, 36, 114, 20, 72, 136, 172, 231, 203, 238, 151, 87, 131, 101, 189, 40, 248, 150, 170, 96, 236, 98, 80, 204, 237, 133, 35, 190, 168, 160, 96, 212, 42, 48, 100, 123, 101, 158, 44, 88, 73, 111, 210, 218, 196, 41, 35, 56, 27, 161, 196, 226, 6, 58, 3, 213, 15, 197, 125, 249, 75, 172, 137, 110, 168, 19, 61, 176, 16, 104, 121, 19, 12, 10, 107, 131, 87, 5, 44, 1, 67, 237, 24, 206, 155, 86, 212, 221, 191, 70, 100, 83, 115, 229, 42, 34, 102, 125, 87, 247, 205, 8, 220, 181, 153, 31, 253, 247, 163, 220, 96, 122, 89, 45, 138, 86, 33, 244, 199, 242, 212, 15, 171, 188, 72, 252, 65, 95, 96, 93, 181, 141, 36, 86, 92, 247, 252, 53, 191, 118, 58, 228, 218, 139, 22, 252, 161, 97, 104, 233, 150, 180, 241, 228, 201, 19, 95, 43, 223, 153, 41, 238, 156, 113, 102, 111, 18, 79, 103, 71, 249, 47, 188, 157, 117, 255, 244, 88, 219, 219, 232, 60, 20, 104, 232, 90, 158, 168, 179, 81, 15, 16, 151, 112, 15, 30, 4, 248, 180, 66, 51, 53, 156, 194, 45, 61, 244, 247, 199, 227, 77, 54, 238, 79, 40, 118, 54, 221, 88, 1, 187, 175, 27, 161, 64, 91, 52, 16, 10, 74, 58, 145, 53, 246, 158, 168, 27, 97, 240, 196, 14, 14, 134, 27, 56, 202, 55, 112, 12, 63, 122, 76, 46, 122, 237, 11, 202, 253, 94, 170, 29, 159, 170, 75, 225, 204, 72, 153, 183, 31, 255, 96, 234, 242, 61, 122, 238, 212, 230, 81, 195, 64, 184, 175, 168, 57, 167, 148, 248, 84, 50, 16, 252, 43, 12, 250, 60, 160, 69, 199, 13, 6, 180, 169, 51, 94, 59, 191, 243, 29, 49, 42, 20, 194, 217, 77, 219, 5, 168, 243, 107, 150, 123, 54, 171, 249, 54, 146, 92, 225, 28, 62, 255, 99, 158, 158, 171, 199, 136, 66, 19, 70, 34, 82, 63, 7, 173, 119, 166, 32, 84, 190, 163, 37, 86, 5, 162, 253, 14, 205, 134, 232, 46, 117, 148, 173, 106, 132, 230, 97, 24, 205, 195, 21, 208, 124, 23, 70, 243, 221, 10, 104, 216, 136, 5, 241, 108, 109, 27, 68, 40, 240, 194, 202, 24, 45, 49, 165, 54, 219, 24, 200, 82, 182, 5, 178, 57, 97, 52, 223, 133, 201, 220, 164, 100, 134, 51, 57, 29, 156, 96, 169, 35, 49, 156, 222, 9, 161, 112, 70, 53, 156, 243, 9, 161, 112, 70, 180, 38, 17, 20, 194, 17, 24, 77, 185, 93, 88, 241, 73, 88, 71, 100, 5, 247, 242, 236, 60, 41, 170, 167, 89, 127, 47, 159, 45, 142, 114, 80, 171, 74, 120, 219, 53, 94, 44, 0, 233, 181, 154, 37, 112, 7, 245, 142, 1, 251, 152, 168, 176, 14, 72, 91, 152, 175, 86, 241, 175, 31, 173, 0, 171, 43, 212, 228, 181, 112, 207, 195, 185, 45, 55, 171, 38, 121, 74, 57, 135, 249, 67, 98, 99, 52, 79, 163, 177, 122, 151, 164, 131, 133, 220, 37, 231, 31, 34, 241, 44, 56, 50, 226, 241, 65, 211, 197, 190, 102, 188, 91, 194, 48, 74, 82, 152, 101, 222, 213, 99, 210, 89, 187, 119, 180, 59, 206, 220, 206, 43, 241, 249, 46, 102, 81, 102, 213, 206, 171, 81, 82, 240, 66, 223, 36, 110, 118, 180, 226, 249, 91, 76, 137, 232, 124, 27, 33, 252, 27, 51, 187, 106, 60, 184, 21, 63, 54, 4, 17, 196, 41, 153, 129, 231, 108, 177, 47, 22, 85, 14, 57, 51, 234, 204, 92, 58, 77, 178, 18, 118, 179, 100, 201, 5, 47, 59, 210, 115, 99, 147, 174, 247, 241, 211, 134, 157, 167, 45, 249, 187, 79, 96, 130, 120, 249, 162, 242, 164, 175, 33, 213, 254, 92, 219, 91, 209, 151, 103, 203, 10, 114, 195, 170, 172, 130, 49, 184, 64, 228, 149, 123, 208, 145, 37, 14, 7, 198, 198, 223, 9, 124, 111, 151, 25, 255, 80, 15, 228, 214, 185, 229, 56, 17, 201, 28, 43, 248, 220, 15, 149, 118, 218, 212, 254, 109, 215, 234, 37, 217, 17, 228, 99, 0, 134, 77, 204, 119, 229, 38, 203, 173, 61, 180, 26, 30, 163, 71, 14, 16, 225, 91, 179, 110, 210, 22, 100, 55, 77, 47, 109, 220, 127, 219, 181, 104, 34, 189, 244, 145, 187, 164, 151, 48, 186, 54, 43, 33, 195, 102, 53, 172, 232, 241, 12, 167, 194, 66, 252, 104, 123, 244, 40, 33, 107, 75, 208, 53, 53, 174, 181, 219, 82, 240, 80, 169, 205, 243, 246, 43, 163, 89, 104, 243, 22, 78, 137, 28, 154, 104, 74, 144, 214, 40, 172, 108, 165, 98, 165, 169, 21, 72, 255, 180, 6, 106, 65, 208, 64, 108, 123, 32, 96, 97, 174, 213, 56, 58, 3, 162, 161, 79, 122, 251, 96, 51, 170, 61, 244, 61, 12, 80, 181, 189, 26, 45, 249, 33, 159, 109, 218, 119, 102, 65, 225, 250, 250, 253, 215, 207, 224, 254, 250, 74, 34, 55, 23, 216, 19, 187, 214, 151, 217, 220, 122, 11, 216, 192, 130, 233, 181, 52, 48, 93, 223, 220, 134, 164, 27, 132, 104, 80, 109, 102, 196, 16, 14, 184, 249, 4, 83, 129, 234, 85, 45, 191, 36, 88, 57, 240, 93, 109, 138, 229, 72, 162, 150, 214, 91, 237, 214, 58, 80, 191, 113, 185, 14, 93, 19, 223, 23, 232, 251, 21, 250, 126, 177, 222, 90, 227, 98, 202, 79, 94, 208, 38, 182, 87, 110, 98, 141, 190, 145, 193, 9, 194, 183, 58, 98, 116, 229, 251, 183, 30, 81, 113, 108, 20, 58, 111, 65, 236, 204, 103, 141, 70, 224, 219, 206, 85, 246, 156, 168, 27, 200, 119, 173, 115, 36, 15, 227, 88, 162, 212, 226, 57, 159, 214, 218, 10, 125, 20, 38, 205, 34, 89, 236, 152, 29, 229, 69, 53, 138, 179, 126, 123, 99, 99, 131, 185, 59, 229, 221, 250, 117, 106, 59, 63, 35, 13, 41, 222, 235, 188, 63, 159, 36, 109, 45, 127, 208, 174, 57, 185, 167, 142, 224, 4, 208, 241, 35, 235, 104, 91, 46, 89, 17, 193, 152, 224, 168, 53, 43, 20, 135, 234, 211, 110, 87, 96, 85, 213, 97, 7, 238, 209, 134, 54, 251, 109, 94, 222, 225, 8, 212, 218, 7, 62, 35, 111, 189, 181, 36, 47, 226, 33, 115, 154, 177, 133, 202, 153, 99, 174, 121, 103, 92, 89, 112, 215, 209, 241, 47, 197, 64, 119, 99, 95, 174, 29, 123, 179, 159, 119, 62, 153, 60, 114, 170, 30, 21, 113, 86, 194, 123, 214, 178, 202, 169, 175, 227, 145, 154, 96, 2, 55, 20, 136, 35, 224, 179, 212, 104, 2, 30, 12, 124, 186, 168, 146, 87, 202, 224, 52, 51, 106, 15, 52, 2, 217, 227, 63, 163, 211, 249, 96, 144, 20, 59, 184, 201, 31, 121, 17, 111, 207, 137, 198, 76, 163, 59, 232, 187, 23, 76, 196, 56, 20, 179, 46, 22, 195, 119, 77, 23, 181, 28, 9, 180, 130, 133, 166, 2, 0, 199, 155, 124, 147, 154, 25, 9, 55, 112, 115, 249, 188, 92, 151, 240, 164, 206, 38, 27, 77, 144, 156, 186, 91, 57, 175, 209, 218, 209, 205, 109, 8, 28, 40, 25, 212, 82, 29, 172, 135, 18, 57, 101, 12, 131, 223, 17, 41, 96, 196, 11, 32, 49, 140, 116, 97, 48, 144, 118, 22, 48, 220, 211, 108, 65, 19, 47, 206, 35, 247, 45, 41, 40, 110, 131, 223, 63, 206, 211, 73, 191, 29, 119, 78, 59, 189, 187, 183, 104, 227, 148, 159, 191, 111, 244, 10, 245, 77, 50, 50, 80, 19, 185, 91, 15, 205, 142, 14, 57, 102, 49, 159, 132, 204, 40, 88, 22, 129, 119, 139, 247, 247, 48, 249, 99, 127, 58, 155, 200, 86, 248, 84, 169, 150, 214, 217, 100, 22, 75, 197, 210, 243, 47, 69, 47, 135, 6, 228, 82, 62, 56, 189, 131, 118, 227, 78, 116, 218, 168, 93, 133, 220, 195, 94, 242, 122, 79, 207, 115, 146, 138, 243, 198, 4, 75, 4, 131, 159, 236, 3, 74, 59, 209, 71, 144, 12, 22, 255, 64, 155, 251, 240, 82, 52, 38, 33, 6, 109, 28, 118, 109, 99, 61, 15, 226, 218, 100, 55, 194, 229, 108, 146, 246, 210, 234, 89, 50, 19, 123, 148, 78, 229, 125, 53, 102, 114, 160, 32, 108, 92, 241, 185, 117, 123, 94, 121, 29, 207, 126, 17, 77, 169, 209, 140, 245, 129, 126, 123, 0, 17, 158, 211, 21, 240, 156, 54, 26, 77, 242, 142, 111, 111, 79, 169, 120, 42, 249, 185, 121, 191, 143, 44, 106, 79, 111, 218, 113, 27, 209, 150, 175, 231, 46, 137, 183, 16, 253, 90, 209, 134, 91, 5, 185, 254, 221, 113, 110, 138, 199, 41, 12, 162, 81, 254, 72, 59, 29, 188, 156, 185, 28, 218, 187, 82, 40, 234, 147, 96, 55, 51, 82, 129, 49, 17, 163, 33, 5, 82, 68, 246, 98, 104, 2, 182, 105, 21, 172, 91, 24, 171, 208, 223, 101, 190, 130, 149, 88, 132, 123, 239, 254, 229, 199, 43, 118, 79, 113, 107, 158, 222, 243, 240, 52, 148, 163, 228, 47, 136, 148, 20, 171, 150, 237, 12, 148, 186, 131, 150, 179, 206, 51, 36, 6, 137, 158, 136, 106, 70, 64, 190, 196, 6, 158, 255, 75, 143, 128, 100, 55, 244, 163, 41, 199, 37, 35, 5, 7, 60, 140, 228, 60, 33, 204, 123, 243, 203, 235, 247, 63, 254, 126, 244, 252, 176, 125, 154, 86, 37, 61, 233, 3, 37, 209, 122, 244, 119, 216, 143, 18, 61, 160, 81, 210, 243, 163, 247, 63, 238, 31, 181, 251, 113, 21, 195, 123, 66, 225, 6, 7, 189, 127, 227, 34, 47, 192, 133, 226, 165, 178, 170, 58, 91, 90, 242, 11, 144, 163, 199, 143, 225, 117, 172, 240, 24, 118, 234, 136, 184, 4, 80, 29, 67, 213, 147, 232, 195, 46, 7, 37, 45, 190, 188, 251, 22, 85, 63, 81, 203, 95, 243, 26, 60, 253, 179, 73, 154, 119, 157, 164, 216, 100, 21, 153, 147, 4, 105, 138, 25, 56, 74, 70, 236, 116, 4, 204, 167, 47, 120, 217, 60, 72, 93, 91, 252, 150, 91, 162, 96, 92, 251, 102, 238, 8, 136, 220, 159, 226, 0, 57, 163, 227, 112, 22, 195, 37, 7, 195, 34, 78, 51, 153, 209, 204, 171, 120, 34, 34, 163, 235, 104, 151, 99, 129, 87, 117, 207, 171, 100, 159, 84, 194, 244, 74, 82, 213, 17, 85, 113, 251, 148, 188, 90, 6, 98, 103, 13, 218, 214, 207, 25, 239, 210, 158, 231, 24, 188, 44, 215, 17, 190, 192, 237, 196, 238, 114, 55, 163, 120, 136, 55, 51, 202, 250, 252, 157, 188, 226, 50, 14, 149, 116, 215, 125, 132, 67, 185, 140, 225, 252, 186, 21, 172, 79, 135, 41, 148, 114, 50, 53, 172, 166, 183, 108, 180, 67, 19, 241, 241, 1, 54, 89, 122, 123, 230, 166, 238, 9, 234, 5, 98, 57, 233, 139, 184, 202, 32, 41, 14, 230, 213, 219, 60, 205, 170, 196, 10, 175, 241, 129, 59, 212, 178, 96, 125, 199, 180, 91, 127, 157, 139, 239, 227, 242, 106, 77, 5, 98, 214, 203, 50, 185, 17, 122, 15, 39, 255, 59, 138, 247, 29, 139, 104, 180, 85, 213, 185, 149, 16, 80, 168, 31, 239, 227, 217, 108, 178, 16, 18, 79, 174, 226, 144, 120, 151, 144, 144, 207, 108, 72, 254, 146, 78, 235, 26, 7, 229, 50, 25, 46, 194, 122, 113, 45, 247, 228, 118, 87, 213, 143, 253, 172, 87, 240, 217, 198, 210, 7, 221, 123, 170, 92, 178, 150, 17, 130, 245, 93, 46, 76, 74, 52, 165, 209, 16, 205, 62, 162, 59, 5, 154, 171, 29, 73, 102, 106, 29, 35, 53, 93, 85, 35, 143, 61, 26, 71, 1, 142, 55, 79, 196, 93, 56, 122, 226, 50, 98, 75, 31, 104, 110, 170, 139, 1, 124, 115, 52, 81, 231, 173, 160, 58, 203, 179, 18, 206, 148, 103, 52, 148, 238, 228, 231, 229, 175, 248, 33, 126, 13, 98, 221, 49, 97, 117, 139, 91, 10, 85, 77, 193, 224, 190, 253, 205, 125, 140, 123, 104, 154, 177, 122, 104, 133, 44, 150, 53, 86, 167, 151, 172, 77, 209, 127, 90, 228, 237, 88, 191, 137, 46, 235, 166, 119, 144, 5, 199, 0, 168, 27, 59, 164, 79, 215, 68, 210, 86, 21, 233, 149, 143, 45, 251, 62, 74, 90, 196, 11, 220, 137, 213, 239, 146, 165, 78, 124, 25, 138, 50, 239, 217, 9, 232, 209, 150, 209, 115, 244, 224, 177, 61, 194, 217, 137, 239, 188, 204, 157, 116, 128, 80, 164, 111, 127, 18, 87, 77, 225, 219, 171, 112, 195, 89, 212, 149, 164, 147, 40, 205, 13, 144, 211, 140, 203, 59, 245, 87, 167, 241, 226, 52, 121, 65, 156, 2, 49, 111, 114, 88, 114, 61, 136, 40, 90, 211, 32, 214, 21, 117, 6, 147, 227, 44, 120, 110, 167, 67, 208, 106, 228, 184, 39, 185, 107, 158, 160, 9, 79, 155, 5, 67, 58, 210, 66, 9, 244, 18, 105, 17, 130, 179, 212, 200, 241, 38, 12, 104, 27, 207, 177, 191, 100, 105, 213, 236, 28, 5, 110, 218, 154, 168, 95, 174, 224, 36, 96, 240, 37, 147, 189, 229, 182, 104, 6, 57, 182, 46, 101, 3, 169, 198, 146, 26, 54, 243, 182, 14, 249, 216, 58, 77, 148, 243, 36, 82, 192, 59, 160, 61, 190, 185, 131, 224, 229, 16, 73, 236, 243, 179, 81, 76, 22, 212, 27, 67, 152, 237, 232, 68, 88, 144, 53, 116, 205, 236, 236, 204, 207, 203, 232, 183, 166, 232, 107, 119, 202, 69, 2, 178, 124, 230, 181, 3, 121, 77, 191, 156, 113, 177, 62, 14, 28, 85, 188, 59, 63, 61, 205, 76, 172, 235, 174, 33, 96, 253, 246, 120, 245, 59, 105, 182, 31, 114, 243, 217, 51, 205, 207, 122, 47, 95, 147, 160, 59, 47, 155, 197, 151, 10, 209, 128, 24, 225, 14, 37, 71, 49, 200, 13, 35, 149, 29, 42, 19, 141, 195, 22, 60, 118, 206, 112, 226, 78, 45, 137, 55, 208, 204, 179, 239, 56, 20, 48, 188, 142, 155, 225, 77, 231, 61, 244, 71, 76, 62, 27, 135, 152, 214, 192, 58, 97, 22, 223, 165, 129, 75, 51, 106, 223, 188, 70, 143, 3, 150, 60, 137, 185, 37, 16, 42, 191, 18, 102, 197, 45, 126, 133, 95, 219, 219, 31, 38, 47, 97, 35, 35, 218, 71, 134, 222, 106, 227, 7, 166, 106, 59, 203, 172, 63, 106, 214, 50, 254, 183, 68, 236, 155, 86, 204, 154, 127, 142, 140, 138, 192, 173, 111, 57, 245, 73, 90, 40, 38, 180, 39, 3, 106, 240, 105, 231, 180, 217, 82, 99, 182, 246, 159, 9, 195, 63, 97, 68, 70, 172, 215, 119, 145, 136, 169, 167, 117, 83, 137, 117, 237, 152, 154, 78, 68, 113, 221, 248, 11, 217, 50, 147, 11, 26, 114, 21, 4, 11, 100, 55, 151, 19, 205, 92, 227, 51, 80, 254, 226, 91, 116, 26, 112, 89, 208, 35, 66, 166, 226, 54, 157, 127, 67, 174, 248, 165, 201, 160, 95, 248, 254, 55, 117, 16, 120, 166, 47, 174, 251, 183, 86, 147, 184, 24, 2, 127, 142, 101, 151, 180, 235, 183, 52, 155, 98, 39, 105, 0, 145, 157, 40, 89, 146, 215, 81, 85, 60, 58, 185, 98, 82, 231, 60, 79, 251, 112, 54, 4, 48, 126, 249, 122, 125, 211, 92, 79, 141, 74, 112, 135, 129, 204, 45, 100, 5, 233, 86, 86, 130, 250, 157, 230, 29, 37, 97, 220, 109, 178, 247, 221, 167, 164, 214, 113, 8, 103, 61, 37, 118, 3, 84, 65, 83, 202, 56, 153, 170, 171, 238, 113, 82, 74, 190, 146, 225, 92, 221, 218, 40, 92, 40, 14, 108, 189, 160, 128, 191, 122, 197, 120, 154, 76, 219, 183, 60, 190, 226, 67, 221, 239, 160, 19, 35, 73, 35, 123, 171, 188, 77, 26, 235, 81, 202, 247, 59, 108, 25, 65, 71, 23, 93, 234, 247, 52, 90, 9, 30, 85, 107, 211, 174, 133, 220, 57, 197, 24, 44, 42, 196, 178, 113, 36, 194, 186, 225, 238, 35, 175, 211, 139, 151, 155, 249, 115, 113, 52, 41, 150, 182, 126, 205, 93, 56, 191, 77, 223, 45, 199, 84, 177, 98, 109, 53, 94, 200, 214, 239, 164, 255, 222, 78, 91, 241, 157, 148, 52, 36, 199, 158, 229, 44, 41, 199, 110, 48, 167, 47, 95, 13, 134, 97, 177, 14, 194, 194, 11, 116, 26, 67, 125, 180, 147, 144, 93, 246, 193, 212, 41, 207, 189, 76, 226, 33, 158, 48, 191, 19, 243, 165, 89, 126, 195, 188, 51, 41, 27, 245, 54, 13, 237, 165, 55, 91, 215, 72, 240, 166, 251, 7, 193, 88, 192, 55, 67, 187, 99, 136, 38, 254, 115, 119, 106, 38, 77, 197, 222, 137, 153, 223, 223, 193, 71, 158, 59, 51, 98, 0, 140, 252, 170, 181, 68, 206, 34, 6, 97, 38, 57, 232, 60, 189, 47, 29, 79, 15, 56, 43, 191, 124, 255, 156, 9, 217, 228, 112, 217, 155, 68, 73, 71, 204, 219, 65, 156, 80, 207, 32, 105, 123, 226, 206, 78, 116, 214, 128, 151, 192, 145, 151, 118, 183, 77, 238, 145, 59, 109, 103, 235, 235, 39, 218, 128, 137, 220, 81, 125, 22, 228, 183, 36, 30, 179, 57, 247, 125, 220, 239, 11, 29, 195, 35, 119, 24, 133, 223, 202, 112, 168, 222, 84, 32, 49, 176, 2, 255, 235, 53, 58, 81, 75, 130, 192, 86, 101, 214, 142, 217, 252, 126, 40, 71, 109, 194, 198, 134, 113, 177, 72, 233, 149, 68, 193, 205, 115, 146, 22, 89, 229, 198, 205, 210, 14, 157, 147, 59, 147, 40, 50, 117, 123, 145, 227, 1, 65, 19, 108, 70, 125, 207, 221, 18, 73, 143, 156, 247, 58, 252, 238, 19, 89, 81, 157, 191, 247, 242, 190, 55, 73, 226, 162, 253, 145, 25, 207, 27, 249, 28, 172, 15, 52, 28, 100, 37, 135, 215, 204, 164, 252, 243, 159, 36, 81, 124, 236, 51, 109, 172, 146, 79, 34, 198, 162, 169, 207, 193, 206, 80, 203, 245, 194, 236, 190, 86, 195, 247, 18, 17, 60, 12, 162, 29, 91, 168, 215, 188, 140, 31, 197, 229, 39, 225, 58, 107, 231, 115, 176, 220, 219, 236, 93, 243, 155, 53, 226, 99, 246, 255, 7, 98, 234, 46, 220 }; + static const unsigned char data[] = { 120, 156, 237, 125, 105, 119, 219, 70, 178, 232, 247, 252, 10, 152, 231, 221, 152, 26, 145, 186, 146, 236, 108, 178, 229, 28, 71, 94, 162, 27, 111, 207, 114, 38, 147, 209, 243, 245, 129, 72, 144, 132, 72, 2, 12, 0, 74, 162, 98, 253, 247, 215, 213, 107, 85, 47, 32, 40, 81, 182, 39, 51, 60, 199, 22, 217, 168, 174, 174, 174, 174, 170, 174, 170, 94, 112, 22, 23, 81, 89, 245, 63, 60, 46, 138, 120, 241, 33, 205, 250, 201, 197, 235, 65, 180, 31, 241, 223, 135, 226, 231, 131, 175, 206, 36, 212, 81, 85, 164, 217, 240, 67, 57, 63, 41, 249, 55, 6, 104, 23, 25, 224, 223, 146, 120, 252, 50, 158, 49, 24, 249, 13, 227, 201, 103, 135, 85, 82, 196, 85, 154, 103, 28, 9, 250, 253, 224, 171, 193, 60, 235, 241, 39, 47, 210, 178, 106, 111, 68, 127, 126, 21, 177, 79, 53, 74, 203, 173, 73, 146, 13, 171, 17, 171, 178, 253, 224, 171, 171, 175, 224, 193, 36, 169, 56, 220, 155, 34, 175, 114, 246, 0, 240, 191, 62, 57, 77, 122, 213, 135, 94, 145, 196, 85, 210, 206, 230, 147, 201, 198, 3, 6, 170, 193, 182, 76, 71, 157, 206, 83, 192, 211, 60, 205, 8, 20, 20, 80, 144, 217, 188, 28, 17, 16, 40, 160, 32, 229, 36, 237, 37, 4, 134, 151, 88, 64, 121, 81, 81, 24, 86, 0, 32, 47, 227, 113, 114, 144, 103, 140, 191, 115, 198, 150, 147, 73, 210, 134, 74, 29, 83, 117, 3, 120, 161, 153, 246, 54, 233, 229, 69, 95, 179, 173, 72, 170, 121, 145, 133, 217, 114, 245, 149, 139, 95, 160, 232, 68, 127, 94, 109, 160, 209, 248, 57, 46, 89, 115, 179, 164, 168, 22, 237, 188, 19, 205, 172, 22, 102, 17, 99, 85, 78, 72, 121, 151, 255, 148, 231, 147, 36, 206, 218, 103, 22, 240, 157, 59, 103, 22, 228, 171, 249, 244, 36, 41, 28, 192, 77, 10, 119, 48, 74, 122, 99, 209, 143, 131, 60, 41, 122, 156, 92, 93, 39, 29, 68, 237, 179, 104, 127, 127, 63, 154, 179, 193, 28, 164, 89, 210, 143, 62, 126, 140, 68, 17, 239, 47, 7, 131, 207, 187, 81, 145, 159, 63, 45, 138, 188, 104, 239, 236, 118, 88, 251, 66, 144, 25, 174, 78, 212, 202, 121, 3, 173, 13, 139, 196, 23, 92, 248, 76, 115, 12, 47, 43, 61, 204, 170, 100, 200, 41, 127, 128, 136, 120, 200, 68, 212, 180, 38, 59, 179, 253, 192, 30, 146, 151, 113, 53, 250, 48, 77, 25, 131, 58, 209, 246, 197, 206, 0, 127, 104, 243, 71, 241, 52, 249, 123, 60, 153, 39, 255, 76, 138, 188, 125, 209, 137, 22, 22, 167, 46, 120, 47, 23, 208, 225, 246, 69, 116, 135, 125, 191, 136, 190, 254, 154, 21, 192, 215, 5, 69, 246, 60, 169, 94, 38, 213, 40, 239, 183, 95, 119, 162, 55, 10, 209, 170, 31, 209, 27, 80, 106, 64, 204, 153, 33, 134, 166, 253, 122, 227, 248, 205, 123, 195, 14, 241, 216, 30, 22, 93, 74, 71, 70, 118, 71, 131, 26, 52, 119, 14, 203, 131, 120, 50, 137, 97, 204, 161, 242, 134, 119, 56, 127, 232, 68, 213, 98, 150, 228, 3, 222, 192, 6, 97, 57, 148, 16, 70, 28, 106, 145, 254, 37, 89, 180, 227, 98, 56, 159, 38, 89, 165, 7, 152, 117, 13, 112, 177, 174, 73, 148, 10, 130, 96, 21, 32, 172, 35, 45, 97, 3, 91, 208, 59, 84, 184, 152, 158, 228, 147, 150, 61, 0, 194, 224, 49, 130, 153, 180, 117, 162, 41, 31, 14, 44, 200, 170, 41, 99, 244, 24, 178, 29, 211, 101, 81, 131, 145, 102, 6, 147, 99, 2, 177, 74, 37, 238, 13, 51, 68, 170, 136, 85, 232, 49, 38, 62, 147, 148, 180, 5, 154, 78, 196, 234, 110, 16, 86, 203, 161, 212, 168, 188, 204, 190, 255, 3, 210, 29, 3, 74, 216, 163, 138, 169, 56, 207, 146, 94, 154, 148, 218, 238, 40, 62, 176, 65, 143, 231, 147, 10, 149, 227, 193, 56, 96, 212, 51, 176, 173, 158, 121, 108, 104, 62, 160, 34, 134, 165, 90, 82, 226, 34, 23, 181, 175, 136, 136, 33, 146, 14, 54, 48, 22, 108, 51, 182, 59, 209, 19, 102, 37, 167, 179, 116, 146, 60, 46, 134, 237, 157, 78, 116, 160, 250, 125, 133, 123, 127, 64, 186, 141, 231, 213, 118, 153, 196, 69, 111, 244, 116, 146, 192, 48, 71, 184, 159, 175, 177, 50, 193, 196, 135, 6, 146, 73, 3, 123, 218, 110, 191, 150, 114, 177, 17, 61, 122, 244, 136, 25, 27, 195, 8, 14, 177, 239, 51, 64, 221, 29, 131, 7, 176, 56, 50, 246, 40, 218, 137, 126, 68, 70, 77, 3, 28, 239, 188, 223, 136, 246, 148, 5, 131, 86, 178, 232, 209, 62, 16, 83, 219, 200, 216, 130, 71, 36, 141, 89, 251, 153, 120, 156, 76, 202, 4, 241, 25, 158, 64, 31, 54, 213, 115, 133, 98, 28, 61, 196, 24, 20, 236, 182, 61, 138, 76, 171, 227, 222, 56, 233, 115, 110, 51, 99, 132, 7, 113, 192, 84, 160, 253, 32, 2, 84, 172, 13, 246, 101, 115, 211, 182, 127, 128, 226, 245, 241, 248, 61, 103, 34, 25, 36, 218, 54, 234, 243, 216, 16, 42, 200, 184, 178, 59, 213, 176, 221, 49, 204, 162, 175, 193, 112, 223, 152, 2, 44, 133, 48, 38, 182, 24, 30, 85, 204, 223, 234, 41, 97, 156, 112, 159, 194, 47, 146, 94, 115, 244, 16, 27, 35, 164, 26, 247, 190, 103, 83, 89, 39, 186, 203, 219, 80, 158, 214, 93, 36, 192, 131, 34, 159, 242, 86, 253, 2, 184, 203, 4, 208, 136, 221, 238, 123, 35, 117, 178, 47, 196, 124, 97, 133, 234, 68, 158, 78, 116, 76, 123, 27, 46, 15, 94, 196, 101, 245, 47, 172, 142, 128, 186, 27, 5, 245, 77, 61, 223, 112, 84, 75, 87, 226, 82, 42, 192, 31, 218, 218, 73, 116, 16, 0, 125, 186, 219, 88, 223, 64, 251, 217, 223, 110, 247, 211, 170, 91, 109, 179, 159, 92, 219, 176, 188, 221, 146, 198, 77, 76, 19, 94, 173, 123, 16, 110, 128, 41, 31, 25, 58, 159, 162, 50, 141, 124, 224, 103, 185, 163, 41, 208, 193, 141, 7, 228, 121, 189, 182, 216, 141, 18, 73, 37, 28, 118, 109, 0, 226, 236, 53, 236, 192, 211, 179, 164, 88, 180, 1, 235, 9, 147, 228, 65, 182, 70, 245, 247, 250, 113, 219, 13, 6, 114, 6, 177, 29, 184, 145, 91, 9, 80, 119, 151, 186, 103, 218, 19, 54, 68, 7, 253, 97, 226, 169, 176, 6, 112, 21, 211, 149, 119, 97, 11, 132, 237, 14, 51, 59, 103, 121, 218, 87, 70, 153, 43, 26, 55, 61, 145, 208, 180, 192, 4, 103, 212, 205, 167, 136, 119, 200, 128, 26, 250, 152, 119, 217, 225, 186, 217, 137, 198, 236, 203, 70, 80, 33, 7, 49, 19, 198, 90, 165, 100, 62, 93, 18, 82, 75, 49, 252, 66, 106, 60, 66, 16, 208, 198, 221, 6, 131, 184, 214, 161, 219, 89, 113, 232, 60, 83, 41, 30, 186, 160, 46, 113, 118, 40, 45, 34, 131, 225, 81, 157, 163, 124, 154, 124, 177, 154, 83, 50, 226, 254, 210, 138, 115, 83, 189, 17, 90, 81, 163, 54, 82, 177, 252, 122, 195, 199, 254, 54, 212, 102, 157, 227, 246, 169, 180, 6, 152, 209, 84, 105, 158, 229, 197, 211, 184, 55, 250, 98, 245, 102, 32, 232, 251, 203, 170, 78, 67, 181, 169, 213, 12, 69, 145, 61, 182, 47, 227, 217, 23, 59, 174, 211, 120, 246, 133, 142, 41, 212, 124, 204, 106, 190, 74, 206, 159, 36, 89, 153, 136, 88, 2, 242, 11, 235, 26, 115, 168, 204, 250, 63, 75, 250, 60, 143, 106, 39, 194, 154, 72, 1, 124, 126, 205, 202, 120, 144, 188, 153, 87, 210, 177, 44, 219, 143, 57, 28, 66, 93, 47, 55, 143, 67, 214, 20, 4, 231, 54, 140, 233, 26, 7, 253, 83, 217, 82, 198, 138, 198, 254, 7, 231, 157, 50, 168, 183, 193, 191, 53, 27, 195, 53, 243, 208, 101, 158, 228, 69, 83, 6, 190, 77, 250, 243, 222, 151, 235, 194, 21, 156, 188, 91, 183, 90, 40, 145, 8, 63, 227, 94, 111, 62, 157, 79, 98, 146, 97, 246, 89, 53, 108, 101, 80, 37, 18, 50, 239, 132, 66, 230, 64, 122, 200, 230, 204, 183, 200, 160, 44, 203, 183, 184, 116, 48, 75, 182, 185, 249, 30, 153, 36, 155, 10, 205, 130, 55, 69, 82, 66, 22, 98, 223, 14, 167, 224, 211, 32, 129, 169, 232, 243, 219, 223, 32, 117, 239, 31, 120, 225, 16, 57, 212, 75, 37, 64, 155, 155, 254, 7, 39, 69, 18, 143, 221, 71, 87, 95, 133, 127, 113, 217, 82, 173, 218, 30, 115, 104, 60, 176, 129, 95, 194, 162, 48, 107, 40, 75, 140, 128, 182, 209, 131, 230, 142, 9, 145, 93, 191, 185, 148, 58, 111, 27, 136, 53, 24, 203, 245, 170, 171, 215, 86, 6, 115, 87, 26, 99, 112, 94, 17, 253, 246, 89, 70, 108, 97, 55, 60, 41, 207, 149, 81, 6, 45, 237, 219, 116, 56, 170, 254, 99, 110, 247, 221, 236, 245, 95, 203, 232, 118, 187, 107, 52, 186, 193, 52, 182, 34, 239, 83, 218, 220, 110, 247, 211, 216, 220, 21, 173, 174, 151, 71, 95, 156, 209, 21, 234, 127, 107, 150, 151, 163, 255, 178, 205, 47, 39, 241, 22, 108, 112, 0, 175, 47, 253, 146, 102, 253, 246, 140, 49, 44, 237, 197, 85, 178, 162, 1, 54, 203, 113, 218, 16, 175, 55, 247, 194, 136, 11, 13, 160, 166, 185, 177, 249, 69, 53, 174, 19, 162, 91, 219, 112, 154, 70, 225, 28, 68, 133, 217, 212, 220, 56, 89, 75, 77, 33, 143, 187, 69, 45, 111, 218, 82, 173, 189, 113, 8, 207, 162, 16, 162, 213, 55, 224, 124, 237, 231, 63, 163, 126, 139, 163, 190, 100, 104, 131, 249, 104, 186, 168, 186, 116, 41, 245, 32, 159, 45, 126, 75, 217, 40, 101, 237, 138, 17, 158, 192, 82, 31, 251, 194, 254, 36, 25, 108, 136, 114, 182, 0, 221, 112, 132, 1, 164, 72, 152, 105, 79, 207, 146, 119, 188, 65, 2, 45, 104, 64, 176, 124, 31, 170, 85, 225, 97, 180, 205, 216, 108, 182, 252, 197, 23, 109, 177, 188, 78, 225, 58, 182, 71, 226, 251, 236, 209, 173, 131, 54, 6, 147, 51, 195, 132, 31, 1, 135, 8, 221, 156, 103, 214, 10, 49, 34, 92, 84, 88, 74, 247, 145, 224, 124, 3, 178, 67, 116, 75, 12, 126, 178, 159, 242, 17, 229, 227, 74, 246, 15, 254, 8, 240, 203, 155, 212, 45, 155, 110, 51, 92, 184, 211, 105, 22, 79, 80, 175, 161, 189, 165, 125, 102, 64, 55, 25, 40, 94, 157, 118, 183, 151, 207, 51, 181, 251, 87, 195, 11, 218, 186, 124, 96, 58, 210, 91, 230, 187, 125, 161, 22, 223, 87, 9, 35, 246, 16, 228, 237, 235, 175, 225, 255, 135, 178, 104, 83, 224, 163, 251, 48, 196, 232, 226, 231, 198, 249, 134, 15, 151, 90, 246, 159, 247, 225, 249, 136, 217, 22, 166, 220, 252, 9, 196, 28, 158, 165, 40, 142, 154, 59, 90, 14, 99, 94, 31, 87, 249, 123, 62, 19, 0, 144, 229, 123, 146, 41, 94, 125, 250, 201, 36, 97, 54, 154, 87, 180, 92, 98, 134, 193, 118, 64, 171, 220, 46, 225, 148, 226, 194, 192, 62, 145, 47, 172, 99, 118, 54, 163, 202, 237, 146, 80, 199, 176, 209, 124, 237, 155, 0, 39, 19, 102, 194, 249, 212, 90, 74, 83, 176, 253, 41, 12, 230, 114, 187, 51, 246, 25, 29, 210, 231, 107, 24, 160, 181, 219, 154, 213, 109, 136, 51, 244, 215, 178, 41, 205, 108, 8, 202, 60, 113, 106, 156, 169, 89, 236, 111, 138, 206, 2, 174, 147, 71, 98, 14, 179, 222, 100, 222, 79, 202, 118, 104, 27, 13, 72, 208, 170, 2, 35, 183, 208, 123, 92, 168, 240, 46, 57, 20, 22, 171, 141, 114, 102, 36, 240, 166, 30, 45, 80, 6, 171, 220, 116, 106, 109, 46, 205, 48, 7, 214, 185, 249, 84, 90, 19, 233, 31, 217, 174, 17, 221, 194, 111, 177, 21, 6, 200, 239, 22, 209, 64, 92, 39, 60, 151, 172, 210, 31, 240, 83, 30, 98, 36, 229, 6, 236, 199, 149, 216, 99, 61, 78, 65, 120, 50, 60, 118, 98, 143, 118, 210, 23, 163, 135, 7, 210, 236, 11, 183, 118, 145, 191, 74, 206, 9, 246, 182, 4, 19, 235, 99, 71, 73, 245, 150, 197, 244, 197, 89, 210, 63, 154, 228, 102, 27, 57, 119, 126, 105, 107, 13, 235, 237, 0, 201, 205, 64, 119, 69, 31, 27, 108, 67, 247, 112, 201, 240, 200, 58, 96, 177, 148, 163, 219, 158, 72, 83, 65, 30, 246, 217, 40, 167, 213, 194, 62, 151, 3, 122, 18, 174, 245, 42, 185, 48, 7, 160, 232, 190, 124, 174, 96, 112, 208, 128, 21, 81, 250, 249, 19, 207, 238, 119, 226, 153, 67, 108, 65, 170, 137, 35, 4, 135, 131, 223, 10, 190, 142, 217, 225, 148, 117, 106, 93, 156, 150, 67, 107, 139, 236, 128, 231, 41, 61, 38, 41, 98, 184, 158, 39, 149, 32, 253, 25, 83, 89, 50, 112, 188, 37, 157, 199, 228, 98, 38, 237, 139, 174, 233, 129, 223, 161, 98, 57, 253, 37, 229, 230, 91, 87, 97, 22, 226, 222, 110, 160, 173, 93, 98, 253, 203, 249, 4, 68, 254, 79, 97, 29, 81, 244, 211, 137, 250, 121, 198, 10, 184, 118, 69, 87, 198, 172, 8, 250, 30, 65, 50, 54, 182, 146, 177, 152, 241, 126, 57, 149, 228, 195, 129, 31, 116, 212, 199, 12, 22, 144, 179, 5, 13, 59, 73, 56, 57, 144, 2, 4, 115, 122, 73, 67, 130, 220, 77, 205, 50, 222, 5, 205, 178, 125, 203, 66, 74, 10, 206, 100, 244, 30, 31, 243, 234, 239, 27, 144, 225, 224, 221, 177, 115, 2, 179, 56, 45, 156, 69, 254, 93, 212, 125, 0, 56, 222, 134, 169, 42, 53, 155, 85, 245, 131, 157, 247, 1, 130, 8, 197, 0, 218, 128, 218, 85, 62, 202, 146, 144, 118, 16, 133, 86, 35, 182, 62, 115, 171, 95, 50, 107, 145, 53, 178, 41, 82, 33, 132, 193, 243, 227, 178, 237, 136, 207, 144, 25, 189, 114, 182, 187, 102, 85, 145, 174, 130, 100, 199, 131, 228, 151, 100, 177, 2, 134, 93, 95, 26, 142, 105, 39, 23, 152, 146, 111, 159, 24, 100, 251, 72, 245, 160, 218, 227, 98, 184, 239, 117, 78, 225, 32, 144, 48, 157, 170, 4, 182, 95, 136, 35, 169, 28, 19, 63, 245, 230, 57, 67, 166, 192, 88, 192, 132, 51, 44, 188, 78, 227, 156, 168, 132, 166, 153, 21, 73, 175, 41, 140, 171, 170, 40, 193, 67, 184, 216, 222, 141, 62, 194, 159, 29, 241, 231, 190, 129, 153, 151, 140, 152, 67, 51, 169, 154, 179, 92, 146, 45, 158, 211, 92, 208, 13, 90, 239, 78, 232, 220, 147, 218, 84, 99, 159, 106, 98, 174, 104, 150, 156, 71, 7, 109, 56, 68, 112, 140, 52, 201, 154, 227, 241, 49, 53, 73, 16, 105, 217, 218, 221, 141, 252, 33, 248, 72, 111, 8, 108, 152, 111, 47, 78, 198, 38, 12, 208, 34, 137, 107, 43, 227, 147, 29, 141, 118, 232, 156, 7, 16, 158, 173, 140, 104, 176, 190, 217, 241, 32, 128, 106, 220, 160, 250, 86, 71, 30, 155, 51, 197, 158, 165, 11, 179, 101, 7, 151, 94, 121, 251, 162, 210, 157, 188, 57, 228, 104, 99, 56, 186, 255, 72, 9, 227, 143, 214, 145, 60, 16, 47, 173, 0, 29, 131, 156, 57, 26, 48, 96, 250, 55, 197, 255, 225, 9, 23, 128, 39, 113, 21, 235, 115, 186, 176, 53, 105, 115, 147, 108, 78, 234, 8, 193, 12, 172, 102, 220, 202, 7, 233, 4, 223, 170, 159, 142, 19, 236, 98, 114, 201, 10, 7, 11, 186, 142, 27, 100, 214, 202, 54, 247, 192, 247, 110, 188, 161, 140, 164, 178, 57, 169, 36, 157, 125, 237, 81, 29, 227, 33, 29, 91, 227, 25, 26, 203, 154, 145, 20, 227, 135, 196, 25, 58, 131, 109, 51, 223, 119, 6, 228, 246, 153, 137, 126, 151, 78, 147, 103, 121, 49, 141, 171, 131, 184, 55, 18, 98, 123, 174, 207, 142, 163, 83, 223, 140, 132, 228, 67, 149, 191, 200, 89, 103, 18, 121, 208, 19, 91, 226, 11, 123, 31, 29, 88, 44, 94, 137, 171, 0, 156, 198, 64, 177, 32, 168, 163, 56, 233, 253, 33, 45, 95, 197, 175, 218, 23, 27, 78, 188, 215, 58, 204, 88, 213, 180, 207, 155, 110, 33, 177, 224, 36, 148, 254, 36, 246, 54, 73, 98, 111, 123, 146, 216, 128, 34, 159, 1, 137, 1, 20, 203, 243, 224, 46, 243, 80, 252, 170, 168, 35, 169, 3, 54, 205, 232, 54, 67, 70, 26, 106, 123, 134, 100, 139, 150, 133, 235, 195, 199, 116, 236, 93, 254, 68, 214, 123, 45, 202, 218, 242, 89, 39, 106, 197, 217, 162, 5, 127, 38, 147, 150, 101, 38, 155, 180, 207, 252, 157, 106, 242, 225, 9, 41, 85, 221, 238, 40, 10, 28, 179, 226, 98, 103, 152, 150, 55, 39, 101, 218, 142, 206, 111, 210, 207, 155, 244, 135, 4, 219, 188, 158, 128, 87, 181, 173, 1, 236, 68, 214, 33, 35, 162, 69, 240, 227, 63, 154, 244, 137, 53, 105, 125, 90, 4, 216, 90, 234, 111, 67, 61, 250, 132, 58, 180, 6, 253, 9, 245, 240, 139, 209, 32, 128, 251, 143, 6, 125, 66, 13, 170, 214, 58, 15, 1, 182, 150, 250, 219, 64, 131, 170, 79, 55, 11, 85, 235, 153, 129, 66, 61, 252, 156, 26, 196, 35, 36, 125, 89, 199, 6, 191, 61, 137, 11, 226, 201, 41, 138, 166, 105, 196, 5, 217, 95, 161, 20, 120, 11, 3, 235, 24, 255, 198, 123, 40, 17, 194, 119, 174, 101, 162, 187, 60, 40, 138, 167, 224, 88, 194, 133, 29, 240, 21, 138, 101, 81, 91, 252, 165, 146, 244, 163, 194, 202, 151, 90, 36, 153, 0, 168, 17, 78, 203, 161, 196, 55, 77, 202, 50, 30, 114, 148, 162, 176, 205, 255, 56, 8, 9, 46, 6, 178, 161, 186, 168, 9, 104, 181, 54, 48, 95, 25, 140, 2, 81, 24, 45, 8, 213, 21, 244, 51, 218, 140, 90, 123, 81, 139, 253, 225, 213, 241, 53, 39, 85, 92, 60, 79, 50, 148, 205, 101, 60, 178, 18, 186, 71, 243, 114, 150, 48, 170, 251, 4, 216, 77, 227, 134, 243, 191, 164, 34, 126, 238, 93, 79, 112, 242, 192, 164, 186, 63, 15, 12, 169, 81, 54, 192, 78, 119, 176, 108, 243, 37, 14, 151, 18, 22, 159, 77, 242, 50, 233, 215, 80, 20, 206, 188, 66, 238, 64, 37, 94, 85, 27, 26, 255, 97, 249, 118, 158, 101, 48, 178, 30, 212, 72, 94, 119, 191, 33, 26, 84, 21, 11, 55, 57, 14, 153, 187, 105, 66, 185, 47, 123, 125, 23, 226, 109, 181, 53, 231, 138, 177, 175, 234, 141, 162, 118, 226, 12, 205, 138, 125, 215, 160, 71, 73, 133, 161, 76, 103, 43, 232, 66, 164, 151, 235, 66, 98, 197, 123, 250, 23, 146, 43, 222, 159, 27, 10, 150, 96, 29, 67, 254, 153, 69, 135, 211, 241, 5, 203, 206, 91, 222, 133, 191, 144, 240, 136, 14, 173, 217, 44, 241, 118, 62, 149, 65, 226, 107, 82, 103, 124, 67, 65, 19, 2, 106, 133, 176, 16, 82, 216, 131, 158, 126, 89, 82, 248, 34, 25, 198, 189, 197, 178, 169, 209, 35, 86, 86, 69, 191, 96, 133, 132, 202, 170, 92, 43, 86, 30, 10, 91, 40, 114, 240, 210, 17, 226, 22, 31, 123, 114, 203, 101, 27, 161, 90, 46, 63, 62, 217, 89, 199, 52, 230, 145, 129, 149, 250, 181, 102, 41, 8, 205, 100, 95, 144, 24, 144, 201, 105, 101, 57, 176, 166, 165, 79, 56, 248, 190, 137, 232, 11, 27, 125, 94, 29, 246, 245, 20, 89, 60, 105, 251, 214, 106, 86, 251, 184, 171, 57, 107, 253, 220, 18, 250, 155, 74, 134, 95, 8, 144, 99, 77, 103, 131, 213, 71, 157, 151, 194, 127, 140, 160, 26, 45, 124, 176, 108, 164, 27, 236, 107, 249, 4, 74, 110, 235, 55, 39, 237, 218, 250, 45, 104, 17, 149, 9, 65, 117, 162, 174, 147, 82, 248, 94, 82, 120, 54, 99, 232, 144, 99, 3, 227, 218, 30, 38, 153, 84, 105, 188, 67, 105, 144, 23, 189, 68, 215, 73, 10, 60, 187, 140, 213, 102, 140, 86, 198, 167, 176, 175, 40, 223, 108, 121, 49, 13, 72, 120, 31, 166, 74, 88, 193, 230, 168, 42, 219, 167, 95, 89, 35, 150, 183, 209, 83, 67, 135, 248, 88, 229, 143, 143, 14, 14, 15, 127, 101, 67, 94, 28, 196, 76, 232, 202, 107, 219, 21, 147, 118, 211, 59, 134, 90, 45, 107, 41, 49, 21, 75, 137, 105, 244, 48, 82, 201, 60, 246, 203, 93, 79, 132, 203, 114, 203, 227, 212, 58, 21, 211, 138, 91, 112, 111, 112, 15, 50, 117, 61, 248, 214, 186, 108, 81, 107, 219, 243, 37, 56, 229, 21, 220, 85, 174, 123, 217, 137, 122, 136, 213, 146, 220, 77, 86, 213, 147, 72, 50, 91, 86, 196, 198, 43, 33, 147, 76, 150, 38, 111, 147, 225, 211, 139, 89, 25, 213, 220, 169, 237, 1, 223, 154, 103, 105, 47, 239, 39, 34, 81, 251, 244, 162, 74, 178, 146, 81, 122, 148, 252, 49, 79, 178, 94, 242, 246, 105, 36, 46, 252, 245, 87, 158, 196, 217, 112, 30, 15, 147, 119, 241, 112, 9, 100, 127, 62, 155, 240, 163, 37, 127, 143, 139, 52, 206, 170, 166, 224, 71, 140, 87, 147, 164, 202, 179, 37, 21, 210, 242, 183, 132, 49, 58, 47, 166, 73, 255, 96, 94, 20, 140, 248, 197, 1, 235, 215, 146, 106, 61, 9, 250, 36, 29, 166, 85, 137, 128, 181, 72, 14, 147, 234, 215, 37, 44, 178, 119, 213, 92, 139, 205, 31, 63, 98, 209, 105, 95, 111, 168, 172, 176, 104, 152, 92, 204, 62, 232, 203, 119, 63, 100, 249, 135, 146, 159, 108, 44, 219, 173, 238, 188, 221, 61, 142, 187, 151, 219, 221, 31, 222, 255, 185, 219, 249, 254, 106, 99, 179, 181, 65, 149, 177, 72, 166, 249, 89, 34, 59, 175, 155, 43, 101, 190, 19, 39, 242, 19, 253, 208, 104, 93, 3, 185, 106, 196, 90, 129, 81, 238, 77, 105, 155, 150, 248, 6, 110, 222, 193, 228, 34, 233, 225, 190, 45, 107, 185, 35, 151, 3, 88, 96, 124, 71, 93, 103, 141, 20, 94, 60, 172, 209, 220, 34, 153, 77, 98, 126, 132, 156, 67, 118, 80, 247, 143, 183, 223, 119, 32, 1, 105, 180, 121, 25, 49, 230, 254, 71, 123, 115, 178, 20, 38, 209, 8, 25, 24, 198, 183, 23, 88, 241, 200, 212, 188, 84, 77, 157, 169, 120, 105, 13, 51, 168, 143, 95, 188, 249, 25, 54, 113, 180, 64, 116, 30, 119, 255, 249, 30, 173, 177, 60, 57, 124, 126, 248, 142, 63, 3, 153, 66, 15, 226, 201, 108, 20, 103, 115, 56, 206, 210, 106, 255, 184, 7, 121, 87, 129, 103, 51, 106, 125, 132, 95, 162, 38, 251, 181, 209, 194, 86, 123, 56, 159, 196, 133, 172, 20, 23, 85, 119, 146, 159, 158, 196, 217, 199, 94, 50, 233, 14, 227, 249, 36, 45, 71, 31, 179, 188, 123, 146, 143, 225, 79, 182, 200, 62, 94, 142, 186, 195, 121, 190, 152, 195, 151, 81, 60, 30, 199, 240, 101, 154, 102, 242, 79, 55, 139, 249, 215, 11, 102, 127, 134, 184, 169, 180, 160, 141, 37, 89, 247, 249, 79, 221, 60, 233, 127, 76, 187, 241, 52, 101, 255, 159, 100, 25, 251, 95, 222, 71, 205, 190, 37, 89, 222, 27, 49, 52, 236, 43, 107, 136, 253, 63, 158, 48, 201, 200, 225, 247, 100, 126, 193, 254, 159, 194, 79, 246, 55, 139, 207, 226, 83, 248, 50, 59, 135, 135, 85, 156, 243, 255, 23, 240, 127, 57, 255, 88, 14, 179, 238, 79, 79, 187, 207, 222, 170, 111, 175, 94, 240, 111, 7, 63, 119, 159, 60, 197, 36, 14, 139, 56, 235, 15, 226, 106, 148, 20, 73, 31, 49, 210, 144, 174, 152, 137, 126, 99, 4, 179, 34, 61, 99, 230, 116, 94, 38, 178, 246, 5, 251, 103, 44, 192, 14, 183, 0, 184, 66, 169, 12, 47, 106, 77, 15, 212, 199, 227, 199, 221, 223, 126, 239, 254, 51, 238, 158, 47, 186, 151, 239, 113, 61, 173, 12, 168, 158, 193, 181, 201, 203, 186, 80, 168, 197, 130, 149, 73, 11, 132, 241, 156, 137, 121, 2, 97, 33, 21, 190, 97, 21, 62, 90, 100, 17, 128, 123, 87, 27, 150, 56, 81, 154, 180, 8, 254, 185, 123, 69, 165, 16, 170, 98, 70, 244, 138, 116, 86, 121, 107, 222, 191, 178, 122, 14, 106, 227, 133, 188, 119, 165, 186, 141, 203, 54, 254, 220, 238, 236, 18, 28, 74, 239, 252, 132, 118, 12, 26, 213, 24, 12, 243, 143, 31, 109, 178, 172, 2, 96, 150, 221, 74, 21, 3, 165, 186, 61, 52, 48, 178, 195, 28, 179, 42, 147, 236, 195, 101, 30, 183, 76, 141, 25, 128, 253, 13, 81, 42, 5, 130, 20, 35, 129, 228, 88, 61, 76, 120, 199, 73, 108, 253, 175, 100, 132, 162, 90, 73, 58, 197, 192, 139, 168, 154, 0, 222, 255, 211, 34, 254, 232, 114, 243, 184, 194, 36, 138, 234, 49, 171, 159, 218, 243, 39, 51, 211, 79, 28, 175, 103, 169, 173, 118, 29, 165, 70, 6, 219, 173, 246, 89, 172, 246, 154, 181, 214, 238, 21, 26, 28, 47, 126, 48, 35, 93, 230, 201, 32, 233, 108, 181, 1, 10, 75, 102, 151, 62, 14, 162, 249, 27, 129, 251, 127, 255, 111, 199, 170, 119, 199, 174, 183, 209, 64, 214, 124, 126, 112, 99, 129, 179, 43, 215, 8, 28, 242, 155, 155, 139, 28, 170, 180, 154, 208, 161, 138, 159, 69, 236, 174, 59, 89, 185, 29, 192, 18, 214, 109, 59, 147, 215, 114, 225, 217, 188, 109, 193, 33, 17, 209, 234, 162, 163, 171, 91, 194, 3, 43, 64, 80, 105, 94, 48, 183, 23, 78, 45, 164, 125, 228, 100, 90, 78, 255, 58, 62, 94, 123, 175, 226, 2, 203, 191, 69, 201, 47, 217, 201, 42, 41, 67, 182, 24, 123, 248, 182, 20, 163, 195, 137, 206, 237, 7, 248, 149, 89, 112, 238, 180, 132, 75, 11, 140, 151, 223, 186, 232, 182, 92, 132, 230, 196, 15, 247, 178, 242, 178, 38, 112, 72, 245, 125, 231, 10, 101, 23, 112, 26, 114, 160, 58, 132, 36, 93, 116, 113, 252, 210, 112, 68, 191, 203, 203, 224, 133, 27, 37, 114, 188, 69, 219, 27, 125, 7, 230, 167, 58, 237, 112, 234, 17, 51, 67, 132, 56, 52, 80, 46, 41, 122, 180, 162, 175, 191, 198, 210, 181, 20, 3, 106, 92, 227, 160, 199, 5, 227, 44, 103, 241, 23, 147, 229, 203, 228, 118, 133, 25, 73, 244, 210, 1, 131, 93, 113, 231, 42, 243, 163, 201, 14, 138, 164, 204, 228, 140, 226, 242, 245, 121, 166, 118, 151, 119, 184, 210, 176, 190, 188, 20, 187, 215, 203, 176, 200, 91, 128, 199, 2, 238, 61, 50, 158, 243, 19, 230, 86, 213, 9, 110, 201, 24, 94, 97, 177, 109, 33, 41, 73, 77, 224, 42, 163, 116, 158, 79, 19, 72, 245, 105, 59, 154, 80, 19, 79, 33, 85, 37, 192, 156, 220, 154, 40, 39, 175, 82, 130, 52, 27, 67, 13, 251, 245, 62, 126, 212, 24, 32, 189, 121, 209, 178, 86, 57, 172, 139, 173, 252, 8, 239, 219, 227, 175, 137, 106, 148, 178, 19, 208, 44, 224, 223, 240, 186, 194, 81, 212, 68, 97, 5, 18, 115, 56, 18, 62, 87, 230, 37, 31, 41, 183, 6, 219, 208, 115, 151, 254, 221, 117, 208, 239, 221, 121, 183, 170, 24, 30, 113, 84, 70, 18, 37, 234, 32, 125, 110, 157, 99, 241, 204, 189, 128, 109, 21, 82, 100, 72, 212, 156, 14, 171, 130, 34, 98, 107, 86, 36, 131, 132, 5, 214, 125, 154, 255, 22, 35, 162, 101, 177, 174, 118, 42, 222, 238, 166, 164, 27, 132, 196, 181, 53, 78, 191, 228, 123, 4, 71, 233, 160, 82, 196, 151, 158, 28, 124, 106, 95, 158, 113, 229, 25, 67, 163, 88, 90, 203, 144, 66, 208, 67, 233, 124, 243, 31, 108, 66, 156, 248, 132, 199, 188, 72, 177, 19, 36, 25, 94, 143, 168, 73, 22, 103, 198, 55, 108, 59, 65, 82, 119, 112, 122, 69, 188, 46, 114, 137, 229, 48, 178, 207, 251, 114, 71, 234, 187, 101, 79, 52, 110, 117, 125, 69, 234, 233, 236, 10, 237, 224, 187, 11, 169, 12, 96, 100, 118, 186, 99, 29, 172, 163, 61, 177, 249, 8, 31, 195, 71, 254, 46, 75, 147, 18, 37, 155, 84, 16, 20, 188, 152, 18, 79, 235, 50, 110, 254, 85, 164, 130, 90, 102, 234, 241, 216, 109, 221, 42, 169, 180, 142, 142, 186, 2, 210, 83, 179, 53, 200, 7, 23, 71, 67, 26, 234, 142, 217, 183, 109, 168, 51, 85, 55, 247, 1, 43, 78, 59, 148, 252, 173, 160, 237, 22, 245, 180, 116, 119, 48, 62, 231, 110, 44, 137, 213, 219, 102, 160, 93, 131, 25, 13, 153, 125, 179, 12, 238, 170, 93, 193, 122, 113, 142, 4, 84, 43, 62, 249, 164, 127, 84, 45, 38, 216, 157, 81, 22, 8, 118, 32, 241, 186, 173, 89, 220, 125, 243, 11, 108, 113, 101, 95, 30, 23, 241, 9, 252, 18, 119, 3, 180, 46, 71, 221, 131, 87, 240, 136, 125, 249, 57, 206, 74, 248, 101, 30, 253, 252, 139, 121, 84, 193, 47, 243, 232, 232, 57, 174, 197, 126, 153, 71, 239, 126, 195, 181, 216, 175, 175, 174, 240, 137, 52, 145, 53, 21, 25, 112, 178, 255, 95, 248, 19, 175, 7, 240, 46, 160, 183, 137, 124, 161, 106, 139, 103, 95, 157, 189, 251, 236, 201, 219, 121, 6, 251, 179, 45, 124, 40, 52, 104, 22, 194, 184, 14, 146, 67, 134, 229, 200, 45, 113, 35, 87, 247, 221, 106, 6, 209, 227, 199, 105, 58, 106, 170, 81, 175, 142, 115, 163, 221, 203, 39, 252, 126, 75, 181, 112, 45, 219, 79, 147, 114, 43, 62, 139, 83, 126, 146, 90, 112, 177, 108, 111, 40, 4, 212, 7, 207, 248, 121, 11, 177, 65, 253, 6, 104, 232, 78, 247, 149, 16, 73, 62, 120, 150, 103, 66, 195, 214, 96, 245, 228, 48, 184, 88, 216, 100, 41, 175, 102, 169, 177, 193, 34, 94, 221, 66, 101, 227, 104, 190, 117, 252, 191, 60, 161, 226, 196, 240, 126, 220, 109, 181, 204, 137, 181, 79, 188, 1, 86, 238, 181, 215, 0, 70, 239, 132, 9, 102, 34, 15, 235, 13, 206, 194, 124, 15, 137, 189, 129, 84, 134, 18, 166, 233, 123, 117, 193, 247, 146, 32, 113, 201, 24, 117, 16, 113, 148, 3, 113, 191, 255, 218, 213, 146, 178, 109, 11, 24, 102, 132, 86, 43, 125, 72, 8, 169, 46, 163, 4, 169, 238, 171, 120, 202, 132, 179, 70, 15, 237, 195, 202, 104, 135, 129, 213, 76, 221, 126, 3, 10, 138, 53, 95, 212, 117, 194, 37, 187, 123, 199, 117, 166, 130, 34, 123, 255, 158, 78, 105, 97, 84, 82, 179, 241, 45, 39, 244, 74, 93, 171, 102, 56, 20, 231, 207, 185, 251, 55, 161, 195, 17, 60, 13, 229, 200, 146, 237, 64, 242, 168, 50, 225, 39, 209, 237, 71, 128, 84, 190, 15, 24, 227, 86, 111, 255, 181, 13, 45, 8, 128, 180, 62, 37, 138, 146, 233, 171, 242, 36, 217, 104, 150, 170, 189, 236, 219, 186, 109, 33, 124, 239, 20, 7, 52, 151, 58, 227, 119, 120, 191, 134, 179, 231, 116, 216, 245, 69, 204, 158, 235, 26, 2, 215, 185, 170, 170, 119, 20, 75, 20, 156, 245, 62, 100, 189, 193, 75, 60, 223, 168, 191, 199, 225, 222, 238, 183, 86, 160, 2, 36, 136, 40, 75, 219, 152, 177, 253, 178, 21, 77, 203, 146, 137, 155, 7, 113, 245, 205, 127, 211, 137, 104, 52, 11, 31, 209, 126, 104, 246, 118, 225, 121, 158, 128, 201, 144, 122, 11, 42, 7, 225, 156, 233, 238, 184, 237, 115, 72, 238, 129, 135, 226, 104, 255, 221, 95, 80, 143, 168, 198, 79, 204, 248, 61, 166, 186, 227, 40, 180, 201, 145, 173, 57, 103, 229, 253, 184, 49, 231, 191, 114, 27, 50, 182, 232, 167, 224, 134, 104, 199, 129, 104, 162, 125, 231, 137, 215, 166, 106, 28, 150, 197, 52, 174, 186, 120, 76, 3, 196, 250, 148, 240, 4, 191, 6, 83, 99, 176, 67, 62, 149, 27, 246, 137, 162, 255, 253, 235, 184, 214, 163, 253, 104, 151, 239, 81, 83, 232, 143, 161, 180, 27, 237, 138, 55, 168, 66, 83, 4, 35, 127, 202, 234, 24, 76, 152, 123, 13, 178, 90, 168, 31, 36, 19, 77, 119, 19, 231, 249, 120, 62, 123, 9, 27, 155, 225, 77, 185, 142, 188, 23, 176, 69, 166, 172, 146, 190, 103, 198, 70, 41, 71, 99, 123, 237, 10, 106, 130, 213, 96, 86, 27, 118, 132, 1, 78, 133, 217, 229, 244, 194, 21, 146, 84, 152, 107, 96, 165, 133, 42, 124, 200, 86, 123, 171, 54, 117, 100, 10, 119, 27, 230, 53, 106, 183, 95, 153, 218, 14, 49, 13, 45, 138, 219, 42, 22, 57, 55, 69, 164, 247, 81, 210, 171, 71, 2, 250, 18, 190, 229, 72, 94, 203, 165, 89, 227, 29, 23, 133, 117, 98, 144, 121, 232, 245, 204, 123, 235, 221, 119, 134, 49, 235, 140, 2, 23, 218, 53, 236, 63, 171, 105, 66, 100, 39, 81, 115, 199, 219, 239, 107, 192, 213, 6, 50, 90, 103, 203, 186, 147, 13, 49, 223, 219, 80, 61, 164, 211, 198, 33, 197, 30, 184, 177, 215, 30, 109, 111, 248, 30, 216, 226, 74, 166, 200, 103, 105, 181, 186, 181, 144, 40, 87, 54, 54, 164, 117, 136, 46, 39, 103, 97, 69, 178, 107, 235, 147, 224, 240, 104, 146, 156, 197, 89, 165, 197, 0, 110, 97, 83, 2, 0, 215, 5, 97, 179, 54, 21, 244, 129, 191, 47, 234, 75, 174, 73, 186, 141, 193, 42, 248, 9, 106, 5, 13, 54, 124, 194, 59, 104, 25, 242, 31, 87, 239, 55, 169, 191, 119, 13, 190, 27, 34, 7, 57, 211, 126, 99, 205, 182, 240, 180, 107, 203, 122, 177, 101, 9, 161, 43, 221, 56, 53, 234, 38, 75, 69, 201, 11, 100, 241, 73, 214, 48, 108, 140, 28, 241, 46, 182, 66, 2, 110, 183, 182, 124, 193, 76, 215, 8, 230, 112, 9, 225, 88, 185, 142, 72, 30, 118, 5, 83, 172, 46, 164, 100, 178, 165, 153, 143, 134, 2, 47, 251, 205, 102, 121, 193, 134, 238, 41, 26, 136, 86, 119, 142, 55, 70, 250, 231, 89, 143, 72, 19, 58, 233, 124, 233, 132, 55, 201, 34, 132, 133, 76, 138, 150, 12, 129, 178, 104, 231, 13, 126, 180, 209, 67, 251, 98, 187, 100, 65, 42, 89, 104, 142, 217, 115, 171, 33, 117, 71, 36, 169, 73, 108, 174, 159, 101, 143, 251, 253, 180, 146, 172, 107, 121, 80, 190, 201, 75, 58, 159, 57, 66, 20, 148, 76, 212, 153, 55, 126, 31, 82, 36, 214, 245, 174, 2, 87, 63, 88, 85, 79, 136, 35, 17, 202, 141, 6, 158, 128, 2, 1, 109, 70, 59, 108, 20, 3, 226, 74, 151, 235, 245, 199, 134, 62, 54, 184, 208, 194, 202, 174, 27, 81, 249, 67, 27, 33, 239, 210, 204, 168, 160, 182, 174, 13, 127, 100, 161, 134, 163, 1, 35, 137, 16, 116, 172, 198, 3, 135, 98, 128, 103, 186, 137, 48, 107, 9, 53, 216, 45, 244, 92, 16, 136, 63, 245, 146, 199, 23, 30, 64, 175, 54, 229, 119, 207, 117, 134, 234, 227, 222, 27, 232, 125, 215, 213, 205, 217, 214, 130, 160, 42, 116, 134, 200, 199, 174, 165, 204, 18, 24, 93, 132, 161, 23, 86, 93, 17, 125, 148, 115, 170, 146, 32, 249, 211, 178, 5, 64, 23, 1, 36, 234, 105, 75, 252, 138, 28, 193, 136, 55, 156, 94, 255, 233, 40, 170, 67, 200, 153, 168, 233, 142, 212, 25, 237, 84, 64, 152, 26, 155, 175, 16, 27, 197, 220, 194, 89, 22, 237, 219, 66, 230, 153, 79, 54, 247, 107, 218, 12, 198, 26, 98, 187, 134, 93, 13, 91, 14, 251, 150, 226, 34, 193, 115, 88, 131, 8, 21, 77, 8, 60, 70, 165, 83, 190, 53, 155, 176, 0, 182, 186, 9, 254, 32, 114, 234, 33, 145, 94, 108, 122, 217, 73, 73, 161, 174, 51, 245, 178, 157, 25, 63, 236, 91, 11, 31, 241, 72, 53, 167, 86, 96, 86, 139, 200, 27, 134, 224, 192, 162, 68, 249, 45, 118, 250, 182, 113, 130, 52, 28, 84, 219, 55, 125, 222, 44, 176, 246, 36, 13, 214, 20, 92, 47, 13, 151, 137, 42, 10, 174, 137, 92, 163, 67, 100, 32, 201, 40, 170, 240, 69, 247, 182, 117, 165, 180, 116, 234, 111, 48, 226, 36, 170, 186, 14, 30, 122, 233, 210, 234, 8, 204, 93, 91, 110, 248, 100, 220, 127, 117, 247, 87, 208, 197, 194, 151, 131, 169, 27, 180, 156, 27, 201, 26, 69, 101, 170, 73, 5, 92, 235, 213, 25, 140, 230, 174, 43, 81, 228, 241, 214, 48, 70, 21, 230, 65, 10, 138, 148, 159, 48, 230, 68, 131, 180, 106, 45, 73, 147, 127, 215, 137, 220, 150, 240, 125, 194, 68, 73, 73, 136, 105, 38, 193, 143, 31, 35, 18, 123, 214, 52, 206, 66, 208, 27, 72, 155, 139, 110, 239, 70, 50, 7, 24, 188, 103, 127, 133, 178, 4, 22, 228, 124, 119, 251, 138, 26, 157, 40, 237, 200, 202, 44, 114, 169, 127, 249, 130, 190, 78, 124, 103, 155, 255, 217, 221, 38, 169, 143, 186, 70, 90, 130, 176, 86, 135, 18, 26, 110, 143, 181, 245, 189, 183, 45, 98, 28, 136, 14, 194, 219, 30, 184, 116, 155, 75, 241, 102, 122, 187, 2, 172, 20, 117, 196, 116, 207, 202, 7, 242, 77, 141, 88, 245, 44, 255, 227, 88, 213, 69, 251, 17, 206, 92, 119, 202, 206, 184, 67, 59, 82, 164, 242, 124, 146, 196, 153, 37, 81, 170, 149, 119, 249, 79, 226, 185, 64, 138, 227, 106, 181, 103, 208, 160, 114, 86, 249, 40, 38, 169, 127, 62, 68, 181, 227, 105, 62, 212, 0, 8, 46, 57, 110, 227, 82, 87, 81, 113, 87, 186, 120, 190, 76, 63, 209, 228, 239, 241, 0, 217, 212, 203, 113, 14, 189, 61, 72, 141, 160, 45, 1, 226, 118, 205, 176, 28, 76, 211, 44, 157, 206, 167, 96, 67, 46, 196, 23, 91, 22, 174, 247, 185, 201, 82, 208, 167, 88, 70, 138, 240, 74, 210, 205, 69, 221, 8, 159, 96, 184, 35, 124, 206, 77, 167, 82, 40, 152, 225, 21, 117, 31, 170, 161, 48, 69, 143, 212, 160, 212, 200, 204, 253, 160, 164, 232, 119, 86, 13, 216, 12, 67, 9, 90, 46, 57, 162, 139, 218, 108, 229, 76, 100, 88, 67, 170, 199, 62, 203, 38, 65, 58, 232, 37, 11, 242, 149, 11, 247, 55, 236, 171, 13, 202, 151, 241, 76, 58, 140, 191, 37, 241, 152, 253, 34, 247, 157, 51, 62, 84, 41, 95, 219, 133, 77, 54, 232, 26, 204, 181, 45, 148, 162, 116, 152, 34, 41, 170, 185, 99, 129, 15, 160, 2, 220, 18, 118, 8, 54, 226, 21, 64, 102, 203, 126, 62, 137, 47, 23, 50, 73, 37, 238, 29, 160, 143, 83, 180, 85, 170, 36, 48, 142, 61, 145, 204, 249, 32, 230, 37, 196, 188, 78, 196, 223, 121, 164, 139, 232, 92, 160, 139, 201, 144, 150, 112, 2, 71, 80, 166, 55, 49, 149, 106, 34, 80, 52, 139, 43, 73, 111, 235, 243, 233, 214, 135, 175, 91, 207, 59, 138, 234, 107, 64, 16, 224, 143, 205, 105, 119, 63, 28, 230, 57, 17, 129, 219, 101, 249, 26, 121, 126, 11, 163, 183, 76, 55, 200, 239, 165, 122, 134, 134, 96, 26, 47, 78, 146, 186, 65, 184, 5, 174, 127, 162, 105, 107, 57, 9, 226, 252, 157, 102, 142, 203, 49, 51, 173, 25, 189, 87, 40, 212, 94, 44, 109, 149, 110, 145, 80, 219, 94, 109, 89, 227, 141, 6, 52, 45, 15, 245, 172, 208, 247, 79, 11, 190, 91, 168, 176, 21, 29, 197, 165, 199, 138, 58, 199, 109, 13, 114, 37, 63, 165, 120, 195, 92, 111, 18, 151, 37, 236, 22, 100, 206, 18, 191, 201, 10, 190, 223, 218, 230, 29, 255, 20, 21, 236, 220, 208, 59, 69, 92, 239, 182, 167, 107, 81, 138, 15, 34, 219, 215, 158, 51, 143, 198, 178, 154, 224, 72, 105, 126, 250, 95, 210, 189, 187, 19, 96, 57, 42, 110, 50, 239, 13, 19, 58, 144, 183, 100, 111, 191, 224, 241, 90, 217, 42, 220, 9, 154, 133, 160, 170, 218, 157, 183, 138, 229, 84, 73, 165, 192, 52, 104, 66, 187, 3, 185, 193, 29, 5, 119, 246, 164, 80, 136, 165, 251, 3, 107, 43, 124, 105, 25, 51, 79, 220, 72, 47, 121, 95, 222, 196, 19, 239, 30, 119, 95, 67, 203, 48, 189, 242, 108, 186, 71, 120, 4, 39, 86, 244, 27, 188, 178, 175, 45, 39, 223, 33, 39, 89, 244, 75, 178, 112, 14, 148, 140, 179, 189, 232, 79, 21, 232, 236, 69, 173, 108, 62, 77, 138, 180, 215, 18, 30, 225, 158, 137, 215, 175, 68, 82, 98, 60, 160, 240, 189, 184, 76, 158, 165, 69, 89, 153, 26, 165, 186, 133, 94, 4, 190, 123, 209, 113, 107, 14, 155, 203, 225, 94, 250, 9, 28, 143, 133, 47, 124, 195, 120, 235, 253, 21, 57, 79, 82, 59, 164, 170, 116, 125, 254, 105, 64, 92, 155, 134, 0, 28, 120, 107, 94, 138, 59, 93, 108, 18, 197, 3, 211, 132, 122, 6, 214, 32, 120, 122, 195, 128, 107, 152, 242, 40, 103, 143, 178, 97, 176, 9, 153, 11, 97, 80, 206, 201, 26, 169, 227, 14, 46, 63, 239, 126, 212, 52, 242, 115, 101, 102, 121, 203, 15, 190, 135, 192, 249, 11, 116, 77, 5, 67, 135, 119, 47, 0, 236, 220, 85, 85, 189, 0, 168, 62, 28, 14, 34, 123, 116, 116, 77, 247, 96, 73, 93, 154, 206, 225, 157, 147, 119, 94, 165, 114, 62, 171, 234, 224, 253, 91, 131, 234, 26, 48, 155, 134, 124, 50, 102, 86, 69, 156, 173, 54, 227, 100, 225, 164, 139, 58, 234, 197, 103, 214, 118, 143, 206, 26, 182, 121, 52, 221, 226, 33, 55, 26, 8, 225, 236, 229, 45, 219, 65, 82, 36, 71, 252, 49, 176, 150, 97, 177, 214, 41, 245, 74, 250, 86, 47, 231, 136, 64, 9, 225, 93, 18, 242, 182, 46, 120, 165, 4, 60, 67, 249, 110, 223, 122, 183, 121, 31, 165, 199, 16, 90, 139, 196, 22, 105, 178, 234, 150, 42, 10, 208, 231, 65, 130, 222, 112, 185, 181, 44, 249, 137, 113, 157, 153, 45, 251, 214, 146, 56, 58, 196, 141, 133, 195, 100, 169, 220, 117, 91, 207, 78, 207, 210, 103, 71, 138, 248, 252, 8, 198, 177, 74, 207, 82, 213, 73, 190, 84, 91, 255, 122, 26, 199, 174, 56, 199, 177, 97, 116, 229, 253, 60, 173, 37, 163, 196, 111, 165, 192, 59, 155, 10, 180, 209, 201, 230, 59, 6, 117, 119, 15, 153, 103, 150, 35, 5, 244, 208, 138, 204, 116, 89, 253, 54, 236, 186, 242, 104, 34, 2, 135, 105, 194, 167, 172, 233, 48, 203, 139, 228, 13, 204, 105, 243, 88, 174, 192, 59, 28, 119, 128, 124, 152, 78, 96, 165, 87, 191, 159, 198, 218, 6, 30, 154, 240, 177, 195, 235, 78, 164, 60, 130, 113, 195, 22, 219, 95, 173, 139, 127, 140, 111, 70, 48, 221, 78, 136, 17, 158, 162, 151, 135, 246, 110, 64, 34, 19, 45, 10, 185, 159, 133, 206, 56, 212, 249, 154, 174, 251, 124, 27, 78, 28, 186, 140, 88, 5, 191, 138, 20, 173, 132, 106, 231, 169, 187, 128, 122, 61, 158, 115, 190, 5, 162, 109, 213, 166, 255, 2, 236, 123, 187, 248, 165, 222, 88, 172, 188, 25, 93, 141, 140, 198, 39, 88, 93, 150, 121, 100, 100, 31, 153, 57, 235, 183, 228, 96, 154, 168, 187, 212, 49, 240, 108, 70, 112, 215, 161, 3, 39, 218, 204, 42, 244, 159, 242, 69, 17, 36, 74, 88, 186, 72, 205, 119, 189, 71, 251, 190, 53, 188, 22, 119, 254, 192, 149, 214, 254, 246, 177, 240, 2, 161, 136, 59, 100, 45, 184, 177, 148, 23, 133, 250, 170, 156, 215, 185, 105, 46, 159, 249, 55, 159, 250, 124, 32, 177, 86, 98, 234, 154, 213, 111, 31, 193, 100, 109, 221, 34, 92, 174, 126, 119, 208, 146, 243, 123, 252, 67, 18, 193, 144, 209, 37, 122, 110, 8, 172, 45, 212, 50, 132, 81, 187, 193, 124, 164, 152, 40, 71, 79, 201, 228, 102, 118, 51, 148, 102, 168, 9, 218, 192, 150, 14, 171, 105, 242, 243, 199, 232, 46, 204, 232, 119, 153, 211, 114, 151, 71, 63, 119, 77, 159, 198, 153, 5, 141, 2, 1, 21, 96, 213, 245, 7, 71, 97, 152, 173, 193, 184, 203, 237, 34, 39, 99, 192, 115, 21, 184, 65, 67, 72, 25, 104, 27, 77, 137, 184, 245, 128, 233, 57, 110, 157, 48, 252, 252, 157, 160, 189, 94, 146, 113, 130, 123, 178, 68, 121, 10, 30, 234, 150, 184, 44, 102, 46, 230, 38, 103, 22, 32, 213, 153, 117, 131, 2, 192, 217, 20, 106, 219, 55, 195, 167, 51, 109, 247, 125, 203, 43, 120, 210, 116, 130, 89, 251, 190, 249, 201, 7, 245, 248, 67, 105, 237, 134, 120, 61, 208, 135, 104, 201, 57, 231, 155, 191, 193, 208, 14, 167, 106, 227, 84, 79, 240, 117, 115, 75, 44, 103, 191, 27, 237, 25, 178, 83, 30, 46, 233, 58, 243, 65, 99, 220, 61, 93, 229, 136, 148, 119, 228, 168, 210, 0, 23, 65, 91, 79, 4, 252, 7, 155, 230, 61, 30, 187, 136, 135, 238, 51, 53, 248, 109, 251, 70, 4, 19, 202, 3, 221, 240, 114, 130, 45, 7, 53, 141, 185, 20, 75, 137, 250, 145, 27, 19, 80, 5, 117, 41, 161, 31, 49, 185, 180, 32, 116, 250, 62, 37, 226, 234, 138, 197, 134, 90, 107, 238, 200, 49, 246, 4, 142, 144, 33, 98, 33, 34, 211, 146, 113, 214, 122, 79, 50, 66, 254, 65, 241, 220, 83, 174, 99, 72, 185, 94, 52, 49, 180, 28, 232, 71, 116, 139, 93, 96, 223, 198, 60, 147, 215, 52, 25, 148, 157, 8, 57, 27, 234, 149, 84, 154, 57, 189, 124, 15, 195, 154, 80, 57, 131, 142, 9, 131, 171, 246, 76, 191, 23, 204, 32, 26, 31, 18, 37, 171, 151, 222, 118, 143, 129, 176, 247, 75, 218, 52, 143, 145, 173, 222, 51, 246, 182, 134, 166, 131, 124, 58, 139, 139, 228, 93, 254, 83, 154, 245, 219, 23, 157, 136, 220, 51, 241, 15, 188, 207, 230, 2, 153, 128, 223, 241, 131, 133, 227, 221, 130, 196, 112, 188, 2, 162, 148, 175, 70, 249, 71, 39, 250, 189, 206, 24, 246, 68, 37, 72, 154, 183, 107, 34, 23, 215, 75, 151, 47, 24, 145, 245, 91, 158, 200, 64, 6, 91, 146, 172, 112, 216, 11, 205, 61, 67, 166, 145, 240, 231, 1, 1, 59, 241, 190, 34, 66, 253, 248, 112, 194, 42, 116, 162, 103, 228, 61, 184, 156, 170, 0, 69, 12, 31, 142, 229, 157, 68, 60, 134, 174, 225, 161, 140, 100, 250, 234, 69, 169, 215, 225, 163, 133, 163, 69, 108, 191, 60, 189, 100, 88, 38, 164, 120, 15, 175, 52, 136, 221, 211, 26, 130, 123, 161, 24, 128, 23, 4, 164, 214, 64, 161, 98, 3, 235, 204, 205, 184, 134, 243, 80, 73, 190, 33, 223, 159, 174, 172, 153, 9, 107, 18, 152, 190, 13, 137, 181, 185, 55, 223, 133, 33, 171, 156, 169, 66, 153, 43, 43, 249, 246, 35, 206, 178, 69, 123, 193, 76, 152, 39, 211, 101, 237, 1, 18, 35, 140, 115, 142, 154, 191, 38, 21, 85, 127, 246, 83, 58, 233, 53, 215, 15, 105, 1, 154, 160, 41, 23, 87, 249, 55, 159, 112, 241, 58, 210, 13, 38, 221, 108, 110, 77, 183, 75, 151, 169, 240, 19, 124, 198, 245, 230, 159, 107, 46, 198, 4, 28, 78, 31, 173, 78, 100, 111, 7, 189, 222, 74, 36, 192, 197, 79, 69, 196, 86, 35, 196, 129, 213, 24, 140, 99, 203, 60, 169, 91, 243, 32, 85, 86, 95, 247, 88, 202, 136, 149, 17, 44, 89, 255, 32, 244, 126, 162, 197, 16, 10, 36, 70, 134, 153, 181, 163, 5, 235, 231, 148, 67, 103, 40, 203, 17, 148, 145, 18, 20, 206, 135, 145, 63, 64, 57, 95, 157, 30, 111, 169, 11, 182, 200, 26, 7, 173, 172, 64, 66, 173, 170, 231, 174, 27, 64, 235, 63, 73, 203, 217, 36, 94, 138, 70, 130, 97, 35, 76, 241, 201, 61, 166, 32, 179, 195, 164, 16, 111, 72, 10, 33, 245, 193, 250, 24, 36, 225, 158, 21, 49, 183, 37, 141, 144, 82, 96, 47, 86, 177, 245, 181, 33, 86, 31, 176, 25, 175, 150, 108, 245, 136, 249, 1, 233, 128, 133, 163, 89, 37, 64, 90, 172, 81, 47, 198, 79, 114, 39, 79, 104, 208, 67, 212, 46, 225, 169, 3, 31, 196, 47, 184, 213, 28, 127, 0, 62, 44, 102, 243, 50, 121, 94, 228, 243, 153, 89, 190, 118, 144, 34, 144, 91, 89, 2, 241, 79, 101, 235, 88, 6, 193, 152, 63, 197, 82, 136, 17, 149, 47, 101, 57, 196, 207, 219, 207, 182, 36, 130, 201, 105, 227, 169, 121, 125, 75, 35, 116, 28, 234, 150, 72, 112, 251, 107, 89, 38, 33, 8, 233, 82, 137, 173, 84, 183, 189, 92, 178, 212, 159, 248, 220, 75, 38, 117, 107, 24, 62, 95, 230, 203, 92, 199, 8, 38, 220, 193, 19, 177, 218, 237, 39, 189, 116, 26, 79, 160, 148, 41, 150, 78, 172, 43, 207, 4, 104, 80, 32, 117, 188, 176, 156, 28, 117, 151, 168, 119, 201, 65, 225, 198, 132, 44, 89, 66, 233, 185, 135, 167, 238, 132, 47, 54, 13, 169, 205, 61, 243, 82, 77, 78, 160, 61, 205, 215, 186, 101, 156, 138, 128, 232, 217, 45, 221, 219, 70, 166, 28, 248, 16, 188, 42, 53, 200, 80, 228, 248, 245, 16, 46, 61, 209, 30, 144, 151, 84, 106, 124, 102, 191, 70, 175, 191, 132, 255, 210, 215, 179, 228, 1, 206, 63, 243, 162, 197, 244, 36, 231, 98, 145, 177, 153, 73, 44, 8, 138, 162, 141, 26, 134, 53, 235, 147, 113, 70, 123, 104, 29, 97, 154, 165, 146, 100, 255, 153, 179, 150, 207, 159, 100, 4, 238, 116, 34, 216, 234, 187, 83, 39, 158, 1, 183, 21, 154, 196, 4, 12, 250, 242, 38, 42, 200, 131, 184, 210, 240, 163, 230, 255, 30, 190, 2, 7, 234, 53, 33, 156, 122, 151, 45, 126, 201, 192, 46, 251, 135, 218, 109, 208, 7, 199, 161, 133, 234, 136, 152, 11, 141, 172, 201, 56, 33, 112, 105, 252, 249, 145, 47, 230, 177, 181, 1, 111, 71, 245, 120, 195, 88, 84, 140, 81, 153, 140, 21, 16, 110, 111, 120, 140, 243, 210, 90, 247, 54, 104, 23, 235, 249, 237, 243, 230, 91, 130, 209, 146, 229, 23, 205, 88, 30, 136, 33, 160, 58, 30, 255, 178, 31, 153, 147, 224, 245, 222, 180, 232, 0, 173, 80, 235, 30, 243, 253, 96, 208, 194, 29, 231, 192, 245, 133, 93, 138, 13, 150, 164, 106, 169, 84, 186, 49, 141, 163, 82, 98, 132, 150, 161, 11, 244, 130, 243, 189, 236, 11, 140, 187, 59, 203, 236, 94, 77, 236, 2, 104, 150, 213, 14, 71, 38, 208, 1, 219, 70, 14, 131, 59, 36, 116, 108, 17, 92, 232, 229, 215, 131, 214, 200, 14, 13, 97, 134, 245, 139, 188, 86, 72, 224, 141, 40, 245, 10, 37, 177, 252, 58, 225, 249, 211, 207, 79, 246, 162, 123, 34, 61, 242, 211, 225, 51, 102, 161, 228, 247, 223, 223, 234, 239, 7, 47, 158, 161, 239, 111, 244, 247, 39, 255, 99, 202, 15, 255, 175, 193, 243, 252, 21, 42, 63, 250, 69, 127, 255, 159, 215, 6, 230, 127, 222, 252, 174, 203, 127, 121, 105, 224, 127, 121, 251, 155, 249, 254, 155, 129, 127, 241, 187, 249, 254, 250, 229, 91, 253, 253, 205, 239, 207, 53, 252, 219, 223, 12, 158, 119, 175, 12, 252, 175, 207, 255, 161, 203, 127, 253, 253, 80, 127, 255, 59, 192, 168, 239, 191, 254, 93, 127, 255, 199, 99, 131, 231, 31, 175, 209, 247, 55, 240, 157, 36, 48, 97, 217, 194, 122, 239, 115, 147, 123, 223, 157, 119, 69, 55, 184, 237, 221, 125, 191, 180, 149, 2, 174, 185, 227, 253, 127, 143, 225, 142, 247, 63, 239, 93, 253, 31, 251, 158, 119, 219, 33, 176, 110, 119, 191, 222, 231, 150, 2, 82, 228, 216, 53, 124, 59, 2, 229, 153, 249, 237, 190, 185, 129, 66, 30, 171, 159, 239, 73, 84, 184, 235, 46, 116, 145, 204, 248, 103, 221, 61, 177, 242, 11, 22, 190, 156, 29, 20, 110, 2, 69, 103, 82, 125, 239, 8, 151, 219, 177, 45, 72, 181, 14, 111, 165, 98, 233, 26, 188, 164, 248, 88, 15, 191, 31, 151, 73, 23, 183, 226, 34, 62, 225, 219, 135, 216, 223, 228, 130, 135, 57, 39, 140, 43, 34, 228, 18, 134, 190, 159, 156, 197, 104, 3, 82, 107, 192, 66, 223, 243, 84, 56, 196, 195, 249, 105, 33, 254, 22, 115, 248, 59, 138, 153, 243, 152, 240, 157, 96, 227, 209, 180, 192, 213, 198, 89, 63, 230, 123, 167, 226, 60, 23, 127, 43, 190, 97, 104, 146, 78, 57, 5, 211, 201, 98, 138, 225, 167, 185, 104, 126, 186, 152, 242, 38, 242, 98, 193, 235, 87, 241, 116, 34, 155, 168, 146, 201, 28, 87, 169, 70, 49, 167, 188, 74, 79, 228, 50, 252, 123, 194, 127, 255, 146, 215, 178, 141, 1, 217, 124, 175, 110, 228, 196, 236, 25, 108, 71, 252, 47, 87, 252, 201, 181, 100, 48, 214, 23, 81, 224, 118, 3, 188, 186, 47, 80, 72, 32, 177, 104, 124, 225, 89, 216, 39, 186, 58, 16, 127, 150, 172, 237, 251, 83, 78, 114, 93, 122, 32, 207, 115, 5, 151, 247, 85, 34, 113, 217, 234, 126, 136, 29, 214, 2, 255, 96, 29, 11, 252, 58, 185, 121, 50, 88, 190, 190, 47, 128, 151, 112, 114, 133, 37, 254, 90, 118, 174, 123, 153, 223, 50, 4, 24, 52, 11, 169, 59, 79, 72, 144, 69, 127, 40, 48, 207, 125, 17, 33, 6, 247, 61, 119, 106, 211, 192, 192, 83, 157, 2, 160, 250, 190, 192, 130, 212, 247, 1, 224, 141, 15, 218, 185, 164, 219, 31, 116, 177, 210, 85, 197, 120, 97, 165, 173, 165, 114, 99, 61, 81, 86, 198, 45, 211, 153, 2, 100, 178, 130, 193, 3, 130, 9, 69, 4, 202, 94, 193, 31, 239, 91, 81, 28, 90, 235, 246, 57, 204, 116, 40, 133, 107, 56, 103, 138, 154, 122, 26, 200, 45, 159, 89, 111, 152, 8, 109, 104, 32, 59, 25, 150, 108, 97, 176, 87, 204, 151, 28, 17, 69, 207, 214, 189, 172, 116, 131, 117, 115, 74, 152, 56, 27, 83, 251, 50, 167, 192, 138, 54, 197, 211, 112, 77, 219, 170, 180, 250, 170, 182, 203, 214, 149, 215, 181, 61, 40, 4, 241, 175, 235, 215, 183, 45, 218, 63, 203, 10, 55, 148, 51, 7, 165, 224, 96, 189, 216, 7, 178, 108, 17, 124, 201, 113, 43, 138, 12, 222, 207, 246, 207, 60, 83, 71, 93, 61, 172, 83, 16, 166, 1, 49, 33, 191, 214, 155, 43, 60, 149, 52, 136, 169, 53, 139, 43, 104, 152, 231, 91, 225, 138, 185, 195, 131, 95, 223, 136, 34, 116, 178, 171, 99, 144, 123, 121, 104, 144, 200, 111, 183, 178, 186, 24, 82, 251, 117, 172, 47, 90, 167, 212, 63, 209, 10, 227, 151, 182, 198, 24, 226, 240, 167, 89, 101, 148, 90, 194, 41, 128, 189, 149, 76, 186, 51, 113, 46, 194, 100, 108, 206, 147, 100, 220, 143, 23, 124, 63, 85, 92, 20, 249, 57, 207, 187, 143, 228, 73, 156, 9, 4, 5, 114, 7, 110, 82, 196, 13, 160, 22, 73, 92, 0, 216, 110, 183, 15, 243, 108, 11, 29, 87, 145, 16, 44, 208, 168, 70, 1, 16, 158, 233, 175, 197, 47, 105, 13, 163, 31, 229, 243, 101, 4, 164, 217, 188, 74, 234, 97, 202, 164, 151, 103, 253, 122, 24, 101, 49, 64, 166, 1, 210, 162, 23, 50, 59, 222, 17, 104, 240, 34, 183, 192, 168, 225, 251, 197, 204, 194, 45, 21, 178, 54, 157, 6, 215, 191, 122, 108, 180, 172, 110, 5, 153, 82, 177, 150, 53, 100, 11, 37, 93, 69, 118, 141, 243, 109, 175, 35, 55, 152, 193, 131, 43, 201, 120, 29, 88, 225, 81, 1, 143, 201, 0, 199, 25, 95, 24, 131, 126, 227, 240, 69, 79, 242, 193, 85, 226, 58, 207, 32, 218, 55, 8, 108, 156, 122, 57, 87, 143, 214, 90, 86, 144, 61, 226, 230, 172, 41, 107, 154, 156, 149, 101, 242, 219, 80, 92, 93, 162, 213, 11, 58, 121, 243, 251, 82, 46, 195, 171, 18, 188, 170, 179, 26, 170, 119, 252, 87, 151, 27, 56, 224, 53, 200, 90, 191, 190, 59, 104, 213, 92, 99, 248, 3, 139, 150, 47, 137, 231, 189, 196, 215, 0, 50, 46, 253, 14, 71, 227, 129, 197, 149, 60, 238, 72, 42, 246, 58, 163, 56, 7, 197, 56, 174, 105, 10, 197, 56, 128, 3, 185, 216, 166, 130, 179, 139, 251, 44, 124, 208, 15, 144, 96, 113, 9, 88, 57, 190, 17, 219, 115, 138, 78, 246, 64, 244, 80, 0, 69, 228, 210, 128, 43, 139, 147, 205, 196, 153, 192, 90, 226, 124, 18, 151, 242, 156, 229, 106, 210, 12, 52, 140, 138, 157, 221, 192, 210, 14, 204, 81, 59, 187, 43, 157, 223, 228, 216, 2, 231, 54, 53, 83, 182, 4, 98, 214, 42, 128, 215, 175, 246, 56, 14, 90, 32, 216, 35, 78, 163, 227, 210, 202, 44, 137, 239, 214, 235, 114, 204, 2, 139, 138, 92, 225, 95, 158, 167, 240, 130, 42, 197, 137, 45, 233, 124, 168, 90, 112, 146, 82, 59, 0, 123, 38, 139, 162, 16, 193, 171, 172, 159, 194, 7, 221, 198, 112, 194, 236, 250, 248, 1, 170, 46, 102, 225, 80, 237, 186, 154, 124, 218, 174, 105, 182, 133, 165, 204, 238, 10, 243, 142, 86, 233, 198, 115, 248, 92, 187, 27, 181, 53, 235, 186, 17, 110, 213, 223, 43, 240, 230, 104, 183, 148, 59, 20, 104, 96, 177, 168, 35, 77, 57, 80, 161, 202, 171, 144, 198, 221, 200, 149, 104, 123, 249, 242, 6, 180, 213, 215, 173, 29, 235, 151, 240, 185, 246, 88, 47, 169, 91, 55, 218, 225, 118, 253, 44, 117, 84, 113, 9, 67, 251, 253, 27, 48, 52, 80, 215, 216, 112, 176, 101, 71, 178, 198, 193, 136, 167, 10, 90, 167, 234, 158, 90, 179, 19, 79, 217, 188, 218, 11, 188, 41, 44, 157, 194, 125, 237, 140, 16, 113, 206, 141, 219, 190, 10, 63, 215, 26, 7, 168, 177, 10, 103, 157, 22, 54, 157, 162, 107, 51, 190, 17, 162, 128, 202, 241, 192, 105, 37, 17, 153, 78, 111, 32, 34, 129, 186, 126, 226, 68, 196, 182, 18, 113, 101, 121, 3, 226, 2, 117, 253, 196, 225, 80, 145, 146, 88, 171, 251, 151, 215, 213, 252, 75, 246, 169, 163, 15, 47, 119, 201, 12, 19, 155, 242, 149, 88, 232, 89, 93, 225, 164, 78, 64, 77, 200, 2, 209, 78, 90, 128, 15, 35, 87, 66, 111, 30, 109, 222, 48, 89, 228, 216, 139, 229, 59, 119, 205, 173, 177, 171, 237, 221, 53, 0, 110, 220, 233, 238, 242, 205, 146, 68, 237, 254, 226, 135, 245, 244, 171, 203, 129, 214, 182, 98, 164, 216, 225, 198, 35, 64, 216, 109, 69, 139, 33, 62, 220, 192, 175, 126, 178, 29, 43, 119, 187, 22, 158, 216, 157, 167, 134, 211, 100, 146, 13, 99, 113, 154, 216, 176, 79, 36, 90, 189, 228, 71, 187, 177, 32, 122, 58, 11, 186, 178, 82, 103, 193, 162, 133, 105, 20, 70, 171, 65, 95, 133, 1, 89, 67, 127, 8, 204, 215, 95, 71, 237, 190, 174, 64, 6, 147, 22, 199, 147, 73, 107, 3, 107, 139, 125, 123, 189, 142, 30, 96, 236, 112, 58, 10, 133, 71, 193, 58, 124, 32, 87, 173, 212, 23, 27, 115, 173, 42, 141, 59, 170, 7, 242, 186, 29, 133, 129, 93, 185, 163, 124, 184, 87, 173, 37, 198, 62, 216, 89, 105, 46, 37, 60, 125, 103, 17, 68, 136, 207, 112, 8, 105, 208, 138, 184, 140, 4, 69, 252, 21, 79, 241, 228, 77, 194, 162, 48, 126, 210, 121, 103, 119, 219, 68, 148, 177, 124, 3, 153, 121, 140, 159, 130, 193, 127, 145, 148, 165, 121, 250, 61, 125, 248, 18, 142, 135, 235, 135, 223, 162, 117, 104, 152, 100, 104, 85, 251, 41, 173, 123, 15, 173, 103, 144, 181, 218, 150, 180, 44, 192, 42, 22, 242, 180, 144, 56, 106, 17, 235, 211, 85, 90, 53, 138, 102, 100, 12, 183, 241, 188, 216, 122, 111, 26, 61, 83, 73, 234, 149, 179, 196, 6, 7, 196, 228, 71, 189, 156, 223, 62, 208, 61, 204, 6, 144, 83, 92, 208, 199, 106, 67, 130, 42, 243, 190, 86, 84, 142, 226, 10, 47, 18, 29, 168, 53, 25, 89, 213, 201, 145, 148, 146, 172, 237, 7, 158, 122, 111, 116, 186, 6, 62, 122, 105, 250, 84, 164, 109, 78, 89, 147, 102, 72, 116, 186, 230, 148, 166, 107, 172, 193, 131, 33, 53, 149, 142, 79, 61, 111, 108, 150, 66, 251, 70, 228, 121, 114, 255, 59, 68, 16, 81, 146, 80, 119, 225, 73, 125, 86, 89, 238, 30, 200, 84, 181, 106, 110, 195, 189, 97, 146, 52, 41, 126, 132, 168, 67, 147, 189, 128, 183, 79, 109, 32, 92, 181, 175, 195, 226, 202, 193, 135, 170, 187, 111, 235, 38, 109, 242, 202, 108, 74, 199, 45, 59, 231, 69, 112, 47, 26, 183, 76, 109, 134, 183, 97, 183, 182, 53, 166, 234, 37, 194, 129, 91, 99, 156, 183, 253, 160, 170, 158, 229, 58, 42, 172, 171, 226, 54, 53, 3, 168, 251, 201, 68, 167, 239, 245, 22, 124, 243, 35, 205, 218, 118, 227, 93, 167, 175, 157, 104, 119, 163, 19, 117, 119, 61, 77, 192, 40, 201, 38, 216, 24, 120, 94, 247, 74, 184, 111, 25, 86, 23, 157, 30, 121, 131, 51, 240, 202, 80, 141, 211, 182, 184, 141, 144, 134, 94, 68, 74, 177, 34, 43, 223, 12, 107, 147, 254, 7, 145, 250, 222, 197, 201, 79, 102, 240, 218, 143, 140, 13, 182, 5, 28, 27, 103, 14, 252, 192, 121, 252, 140, 90, 209, 7, 190, 166, 188, 175, 24, 196, 150, 221, 125, 187, 96, 179, 185, 90, 162, 106, 50, 187, 187, 251, 217, 104, 22, 245, 243, 238, 227, 93, 182, 41, 230, 139, 222, 201, 139, 151, 79, 67, 29, 136, 124, 151, 173, 208, 74, 255, 230, 215, 173, 88, 226, 120, 131, 91, 206, 98, 225, 133, 89, 215, 174, 132, 152, 189, 244, 6, 176, 120, 207, 190, 235, 76, 238, 10, 210, 155, 125, 241, 246, 203, 85, 119, 5, 83, 178, 200, 190, 96, 178, 253, 155, 129, 249, 117, 110, 155, 232, 220, 118, 64, 231, 96, 230, 107, 11, 36, 116, 70, 255, 145, 207, 95, 192, 252, 15, 89, 126, 206, 218, 220, 51, 91, 143, 1, 62, 184, 243, 88, 141, 87, 205, 222, 99, 107, 72, 155, 237, 62, 14, 109, 70, 89, 247, 254, 227, 48, 227, 169, 187, 251, 5, 237, 64, 182, 248, 185, 194, 30, 228, 37, 76, 93, 247, 46, 100, 181, 111, 14, 195, 168, 178, 155, 237, 85, 86, 161, 24, 6, 86, 101, 74, 187, 4, 99, 121, 143, 208, 26, 160, 129, 151, 169, 196, 142, 236, 27, 149, 112, 122, 131, 86, 218, 155, 203, 250, 144, 132, 126, 151, 235, 53, 96, 205, 144, 167, 123, 145, 137, 54, 121, 201, 243, 61, 25, 116, 242, 95, 112, 7, 160, 136, 61, 249, 207, 151, 123, 58, 4, 229, 191, 95, 88, 191, 251, 123, 56, 50, 29, 237, 169, 192, 148, 255, 252, 153, 254, 28, 211, 159, 191, 208, 159, 211, 61, 19, 204, 242, 130, 114, 207, 196, 180, 188, 224, 114, 207, 10, 109, 69, 241, 153, 191, 248, 239, 118, 177, 239, 82, 43, 196, 111, 155, 203, 215, 201, 176, 122, 99, 94, 20, 213, 202, 54, 164, 49, 180, 181, 188, 103, 54, 38, 30, 51, 31, 204, 222, 228, 44, 242, 60, 119, 157, 183, 38, 4, 241, 67, 104, 164, 241, 189, 23, 123, 45, 238, 122, 94, 51, 160, 253, 61, 111, 65, 240, 98, 254, 30, 83, 125, 144, 171, 157, 7, 215, 34, 7, 122, 211, 243, 113, 217, 33, 7, 62, 188, 49, 251, 193, 149, 67, 147, 245, 174, 3, 248, 168, 69, 3, 167, 45, 145, 239, 127, 142, 146, 253, 166, 244, 169, 183, 244, 210, 91, 122, 230, 45, 253, 187, 85, 202, 123, 198, 223, 143, 0, 92, 123, 184, 31, 221, 243, 71, 8, 106, 163, 135, 204, 192, 212, 68, 27, 146, 255, 140, 141, 247, 151, 160, 226, 57, 28, 63, 166, 250, 138, 50, 31, 228, 86, 69, 75, 31, 180, 219, 11, 47, 51, 250, 222, 210, 145, 183, 244, 103, 111, 233, 212, 91, 90, 122, 75, 199, 222, 210, 95, 106, 7, 164, 38, 102, 213, 220, 80, 57, 179, 107, 113, 82, 38, 217, 86, 96, 229, 75, 111, 55, 94, 44, 237, 70, 32, 160, 109, 64, 138, 71, 180, 110, 198, 19, 11, 217, 95, 77, 228, 101, 6, 158, 14, 8, 53, 74, 43, 29, 79, 9, 77, 222, 112, 183, 137, 219, 223, 192, 145, 149, 32, 150, 227, 222, 123, 247, 133, 192, 154, 70, 49, 187, 140, 248, 202, 130, 252, 241, 139, 103, 174, 8, 52, 106, 246, 94, 161, 179, 243, 152, 233, 184, 145, 159, 113, 35, 227, 235, 52, 130, 111, 98, 55, 60, 191, 146, 78, 208, 132, 197, 106, 44, 78, 115, 94, 107, 214, 138, 139, 170, 59, 201, 79, 79, 226, 172, 197, 156, 131, 211, 147, 92, 186, 10, 173, 94, 50, 233, 14, 217, 80, 166, 229, 8, 158, 224, 159, 18, 34, 201, 186, 195, 147, 110, 158, 244, 225, 57, 251, 241, 252, 39, 254, 67, 62, 77, 187, 241, 52, 133, 39, 240, 71, 151, 157, 100, 188, 29, 248, 163, 203, 212, 91, 138, 246, 240, 15, 253, 52, 201, 242, 222, 40, 21, 228, 161, 95, 250, 249, 40, 30, 195, 35, 248, 163, 203, 198, 19, 214, 201, 156, 87, 169, 38, 35, 83, 62, 153, 95, 64, 217, 228, 196, 20, 77, 1, 82, 224, 22, 95, 245, 147, 140, 5, 139, 167, 252, 81, 118, 102, 74, 103, 231, 28, 45, 252, 209, 101, 85, 204, 193, 224, 15, 42, 91, 136, 178, 5, 42, 43, 231, 188, 172, 84, 103, 91, 91, 167, 113, 23, 78, 203, 118, 71, 201, 236, 132, 121, 174, 240, 151, 249, 225, 124, 36, 226, 238, 11, 120, 18, 79, 226, 73, 239, 135, 239, 84, 133, 44, 239, 158, 228, 188, 195, 217, 9, 42, 203, 22, 156, 42, 195, 213, 114, 152, 117, 79, 146, 238, 160, 128, 226, 114, 112, 98, 149, 103, 19, 40, 63, 27, 86, 164, 156, 3, 159, 92, 150, 184, 176, 55, 98, 67, 194, 145, 12, 135, 164, 156, 119, 185, 87, 146, 22, 5, 228, 176, 36, 144, 125, 78, 110, 191, 156, 224, 194, 164, 228, 56, 203, 25, 46, 20, 212, 14, 40, 228, 240, 132, 83, 53, 72, 73, 97, 33, 26, 34, 164, 166, 188, 245, 148, 182, 158, 10, 185, 42, 19, 92, 120, 58, 227, 44, 166, 13, 77, 185, 108, 76, 7, 4, 103, 198, 69, 56, 235, 209, 194, 137, 232, 17, 193, 153, 9, 81, 161, 56, 103, 188, 245, 89, 89, 224, 194, 82, 240, 243, 156, 64, 206, 57, 67, 98, 138, 243, 50, 22, 195, 167, 91, 191, 28, 117, 123, 83, 62, 212, 240, 135, 20, 50, 69, 200, 74, 249, 164, 251, 51, 124, 119, 30, 87, 232, 113, 133, 30, 15, 133, 118, 13, 99, 140, 114, 56, 207, 23, 115, 183, 37, 166, 103, 227, 216, 82, 56, 86, 204, 116, 7, 10, 229, 55, 82, 206, 20, 73, 8, 39, 65, 127, 62, 231, 200, 225, 143, 41, 188, 96, 154, 61, 228, 200, 75, 12, 187, 152, 115, 142, 193, 159, 142, 58, 235, 0, 6, 237, 104, 126, 82, 249, 108, 218, 79, 191, 2, 248, 203, 151, 10, 197, 147, 39, 240, 251, 201, 83, 245, 251, 217, 63, 224, 247, 179, 183, 234, 247, 187, 55, 240, 251, 221, 11, 245, 251, 119, 14, 255, 187, 134, 255, 231, 91, 248, 125, 240, 68, 253, 142, 23, 92, 84, 178, 185, 38, 242, 228, 148, 27, 193, 126, 161, 199, 180, 215, 251, 3, 74, 138, 177, 150, 219, 222, 41, 23, 218, 105, 62, 213, 37, 130, 147, 61, 125, 156, 158, 125, 229, 202, 114, 209, 211, 70, 171, 95, 112, 243, 59, 30, 141, 77, 201, 57, 23, 170, 66, 143, 240, 48, 62, 227, 173, 39, 218, 78, 141, 138, 66, 24, 17, 77, 79, 122, 194, 37, 57, 159, 197, 186, 68, 152, 84, 99, 176, 57, 218, 145, 22, 192, 83, 94, 97, 161, 233, 63, 229, 207, 79, 117, 27, 227, 161, 160, 108, 170, 219, 152, 136, 62, 207, 102, 186, 63, 83, 174, 20, 133, 182, 139, 211, 146, 203, 224, 180, 208, 86, 113, 186, 176, 75, 202, 158, 144, 175, 137, 166, 164, 18, 154, 93, 196, 186, 199, 213, 152, 243, 169, 58, 215, 188, 172, 38, 156, 188, 243, 68, 183, 85, 101, 3, 139, 79, 139, 147, 190, 53, 38, 139, 41, 111, 107, 82, 20, 70, 178, 146, 139, 10, 132, 203, 51, 85, 114, 51, 255, 231, 172, 72, 6, 73, 81, 36, 16, 230, 67, 17, 44, 38, 178, 201, 249, 2, 126, 22, 234, 245, 159, 173, 248, 100, 228, 0, 159, 140, 130, 192, 103, 46, 240, 89, 8, 184, 55, 117, 128, 89, 81, 8, 248, 15, 23, 248, 143, 32, 240, 185, 11, 124, 30, 4, 190, 112, 129, 47, 130, 192, 11, 23, 120, 17, 2, 238, 15, 28, 96, 86, 20, 2, 46, 93, 224, 18, 3, 51, 67, 106, 160, 147, 19, 7, 58, 57, 9, 161, 78, 122, 46, 112, 47, 8, 220, 119, 129, 251, 97, 58, 50, 23, 58, 11, 66, 15, 92, 170, 7, 65, 170, 7, 67, 23, 120, 24, 68, 205, 167, 66, 10, 125, 58, 11, 161, 158, 185, 12, 153, 5, 25, 50, 115, 25, 50, 235, 135, 128, 11, 183, 135, 69, 176, 135, 133, 43, 211, 69, 80, 166, 11, 87, 62, 138, 50, 8, 236, 138, 105, 17, 20, 211, 226, 210, 5, 190, 12, 1, 243, 73, 159, 2, 243, 87, 143, 248, 71, 165, 116, 53, 160, 28, 132, 161, 221, 49, 44, 103, 97, 104, 151, 123, 229, 31, 97, 104, 215, 36, 148, 231, 65, 232, 185, 203, 146, 121, 144, 37, 103, 19, 7, 248, 108, 18, 2, 94, 184, 230, 116, 17, 52, 167, 11, 23, 243, 34, 140, 217, 85, 197, 69, 22, 4, 118, 89, 189, 8, 169, 203, 201, 137, 195, 13, 40, 10, 0, 15, 82, 7, 24, 124, 94, 63, 163, 79, 6, 99, 23, 122, 28, 132, 62, 117, 186, 8, 69, 8, 122, 90, 26, 224, 220, 177, 31, 80, 20, 66, 253, 135, 139, 250, 143, 160, 33, 59, 249, 195, 209, 47, 40, 10, 65, 87, 167, 14, 52, 43, 10, 144, 125, 230, 40, 24, 20, 133, 128, 29, 1, 57, 161, 162, 71, 232, 56, 155, 187, 208, 243, 16, 234, 75, 199, 222, 240, 136, 202, 143, 186, 215, 119, 92, 10, 40, 66, 208, 151, 35, 4, 236, 160, 238, 133, 167, 186, 222, 169, 195, 107, 40, 10, 160, 158, 58, 195, 200, 221, 126, 63, 112, 30, 59, 192, 121, 28, 224, 71, 111, 230, 184, 9, 80, 20, 192, 92, 58, 19, 12, 20, 133, 122, 88, 58, 51, 12, 20, 5, 161, 29, 1, 233, 133, 45, 112, 207, 181, 192, 189, 176, 5, 238, 149, 142, 206, 64, 81, 16, 218, 145, 62, 40, 10, 66, 187, 67, 83, 6, 53, 172, 231, 90, 247, 94, 216, 186, 247, 202, 194, 133, 46, 130, 208, 151, 142, 13, 134, 162, 192, 80, 94, 186, 146, 125, 25, 146, 236, 126, 238, 80, 13, 69, 1, 58, 250, 238, 72, 246, 195, 35, 217, 119, 185, 221, 15, 115, 187, 63, 119, 12, 124, 127, 62, 11, 200, 118, 210, 115, 20, 18, 138, 2, 168, 19, 151, 144, 36, 76, 72, 226, 14, 123, 18, 30, 246, 164, 116, 184, 13, 69, 33, 232, 202, 25, 200, 164, 26, 133, 160, 7, 110, 47, 7, 225, 94, 14, 220, 193, 25, 132, 7, 103, 224, 242, 100, 16, 230, 201, 160, 116, 41, 41, 131, 148, 136, 172, 6, 129, 230, 25, 14, 175, 4, 14, 93, 219, 58, 12, 219, 214, 97, 238, 132, 96, 80, 132, 160, 199, 249, 216, 64, 187, 44, 25, 134, 89, 50, 116, 109, 201, 48, 108, 75, 134, 165, 75, 73, 57, 13, 67, 187, 189, 172, 97, 224, 220, 133, 158, 7, 161, 71, 177, 227, 194, 143, 226, 147, 48, 180, 99, 95, 161, 40, 8, 237, 184, 59, 60, 253, 228, 29, 202, 145, 59, 148, 163, 240, 80, 142, 78, 29, 191, 11, 138, 252, 10, 63, 26, 187, 168, 199, 97, 212, 185, 11, 157, 135, 161, 103, 46, 244, 44, 12, 93, 58, 42, 12, 69, 65, 104, 71, 205, 70, 97, 53, 27, 185, 166, 103, 84, 134, 20, 39, 237, 57, 168, 161, 40, 128, 58, 157, 56, 157, 132, 162, 16, 116, 230, 226, 206, 194, 184, 51, 23, 119, 22, 198, 237, 42, 101, 26, 86, 202, 212, 85, 202, 52, 172, 148, 169, 59, 173, 166, 225, 105, 245, 212, 21, 238, 83, 42, 220, 72, 2, 79, 99, 199, 157, 130, 162, 0, 176, 107, 185, 79, 195, 150, 251, 116, 228, 66, 143, 194, 208, 238, 72, 158, 134, 71, 242, 212, 85, 133, 211, 176, 42, 156, 186, 226, 122, 26, 22, 215, 83, 215, 76, 157, 134, 205, 212, 120, 232, 232, 59, 20, 133, 160, 51, 71, 21, 160, 40, 96, 232, 199, 103, 142, 9, 132, 34, 255, 224, 140, 207, 156, 97, 135, 162, 16, 33, 103, 142, 72, 65, 81, 0, 245, 133, 227, 24, 67, 145, 31, 120, 114, 226, 176, 15, 138, 2, 116, 76, 122, 142, 218, 64, 81, 0, 117, 207, 49, 242, 80, 20, 0, 78, 157, 76, 3, 20, 5, 128, 93, 233, 155, 132, 165, 111, 226, 106, 239, 36, 172, 189, 19, 87, 250, 38, 97, 233, 155, 184, 158, 215, 36, 236, 121, 77, 220, 84, 205, 36, 156, 170, 153, 240, 108, 185, 5, 93, 133, 161, 157, 128, 15, 138, 66, 208, 149, 203, 147, 138, 240, 100, 114, 102, 128, 207, 92, 118, 159, 149, 33, 96, 55, 76, 152, 4, 195, 132, 169, 107, 207, 166, 65, 123, 54, 237, 59, 35, 3, 69, 129, 30, 78, 19, 103, 100, 160, 40, 128, 122, 224, 196, 180, 80, 20, 2, 118, 244, 124, 58, 8, 233, 57, 44, 40, 58, 192, 65, 81, 21, 235, 104, 20, 58, 13, 165, 106, 166, 153, 35, 77, 80, 20, 224, 244, 31, 206, 128, 67, 81, 0, 115, 225, 168, 57, 20, 133, 136, 118, 3, 241, 105, 56, 16, 159, 150, 142, 5, 134, 162, 0, 33, 238, 84, 58, 13, 79, 165, 211, 185, 139, 122, 30, 68, 125, 233, 36, 27, 160, 40, 132, 250, 210, 101, 223, 101, 208, 134, 76, 47, 29, 93, 132, 162, 0, 116, 230, 70, 42, 89, 48, 82, 201, 92, 123, 157, 133, 237, 117, 230, 186, 0, 89, 216, 5, 200, 220, 161, 201, 202, 224, 228, 152, 185, 134, 50, 11, 27, 202, 204, 53, 125, 89, 216, 244, 101, 238, 184, 103, 225, 113, 207, 220, 164, 91, 22, 78, 186, 229, 99, 135, 110, 40, 10, 65, 23, 206, 224, 64, 145, 95, 166, 114, 119, 181, 33, 167, 171, 13, 8, 120, 150, 56, 116, 64, 81, 0, 120, 232, 24, 40, 40, 242, 39, 147, 103, 110, 156, 50, 11, 199, 41, 179, 194, 165, 163, 8, 242, 99, 230, 46, 123, 204, 232, 178, 7, 129, 118, 51, 122, 179, 112, 70, 111, 230, 26, 146, 89, 216, 144, 204, 92, 23, 126, 86, 134, 124, 145, 153, 235, 3, 204, 194, 62, 192, 204, 21, 237, 89, 88, 180, 103, 174, 15, 48, 11, 251, 0, 51, 87, 17, 102, 97, 69, 152, 185, 138, 48, 11, 43, 194, 108, 225, 14, 252, 34, 56, 240, 197, 212, 129, 134, 162, 16, 180, 107, 28, 138, 176, 113, 40, 92, 14, 22, 97, 14, 150, 238, 204, 94, 134, 103, 246, 210, 157, 127, 203, 65, 48, 213, 80, 186, 19, 112, 25, 158, 128, 97, 155, 147, 13, 61, 12, 202, 73, 57, 116, 220, 23, 40, 10, 65, 143, 156, 133, 4, 40, 242, 235, 112, 57, 113, 156, 103, 40, 10, 161, 118, 29, 226, 50, 236, 16, 151, 127, 56, 177, 7, 20, 5, 161, 93, 220, 127, 132, 113, 187, 153, 137, 178, 12, 173, 212, 149, 174, 46, 148, 97, 93, 40, 93, 93, 40, 195, 186, 80, 186, 1, 86, 25, 14, 176, 202, 115, 199, 80, 65, 17, 134, 62, 71, 192, 110, 31, 207, 71, 65, 96, 87, 182, 207, 195, 178, 189, 112, 60, 7, 40, 10, 64, 87, 83, 39, 104, 130, 34, 191, 9, 172, 92, 123, 89, 133, 83, 30, 149, 155, 89, 172, 194, 153, 197, 202, 93, 119, 168, 194, 235, 14, 149, 155, 135, 172, 194, 121, 200, 202, 13, 108, 170, 112, 96, 83, 93, 58, 19, 37, 20, 5, 160, 231, 67, 103, 118, 135, 162, 32, 180, 67, 9, 20, 133, 160, 93, 63, 99, 30, 246, 51, 230, 238, 156, 61, 15, 207, 217, 243, 194, 17, 110, 40, 242, 15, 252, 252, 210, 237, 228, 37, 233, 228, 252, 18, 1, 187, 116, 80, 95, 10, 1, 195, 14, 82, 11, 152, 111, 42, 245, 19, 125, 54, 118, 136, 134, 34, 63, 209, 103, 99, 23, 245, 184, 10, 1, 187, 115, 211, 89, 120, 110, 58, 115, 231, 166, 179, 240, 220, 116, 86, 58, 155, 179, 160, 40, 0, 45, 246, 53, 18, 104, 190, 199, 209, 235, 203, 95, 140, 29, 178, 161, 40, 128, 250, 98, 234, 144, 13, 69, 65, 104, 71, 127, 161, 200, 207, 191, 11, 215, 19, 184, 8, 123, 2, 11, 55, 195, 190, 8, 103, 216, 23, 46, 183, 23, 97, 110, 139, 189, 158, 20, 122, 158, 4, 248, 119, 153, 58, 142, 0, 20, 5, 80, 95, 78, 28, 142, 64, 145, 159, 35, 151, 83, 103, 104, 160, 40, 0, 236, 118, 241, 50, 220, 197, 75, 215, 178, 94, 150, 30, 58, 240, 217, 188, 195, 42, 41, 226, 42, 47, 14, 251, 73, 86, 165, 213, 194, 126, 89, 27, 28, 2, 85, 7, 30, 95, 36, 195, 184, 183, 80, 53, 126, 43, 98, 184, 120, 248, 101, 60, 147, 119, 253, 194, 105, 139, 223, 146, 120, 204, 74, 218, 248, 154, 111, 90, 237, 85, 114, 81, 181, 227, 98, 72, 14, 164, 86, 252, 154, 100, 231, 236, 134, 196, 6, 167, 130, 59, 193, 214, 201, 233, 218, 170, 88, 160, 147, 103, 234, 188, 182, 56, 137, 177, 199, 219, 217, 202, 20, 1, 157, 168, 207, 15, 139, 242, 3, 14, 234, 108, 232, 21, 163, 130, 31, 97, 75, 236, 11, 24, 147, 40, 205, 202, 42, 206, 122, 73, 62, 224, 125, 61, 170, 242, 153, 32, 135, 209, 235, 61, 15, 175, 27, 70, 55, 227, 138, 54, 225, 228, 134, 106, 146, 211, 13, 151, 48, 71, 250, 30, 174, 171, 16, 247, 248, 101, 205, 237, 228, 34, 251, 108, 236, 171, 52, 5, 255, 138, 252, 107, 67, 31, 204, 245, 133, 1, 134, 149, 203, 25, 214, 225, 220, 160, 231, 218, 69, 141, 231, 73, 38, 85, 234, 22, 219, 116, 213, 17, 95, 98, 95, 119, 111, 60, 185, 125, 223, 66, 65, 78, 137, 207, 154, 188, 197, 102, 38, 222, 115, 194, 84, 170, 57, 228, 150, 58, 255, 229, 26, 6, 7, 52, 225, 7, 232, 224, 98, 5, 122, 85, 29, 2, 233, 229, 217, 32, 29, 206, 21, 144, 185, 133, 16, 193, 156, 23, 105, 229, 127, 126, 12, 68, 167, 146, 132, 247, 141, 58, 65, 171, 232, 222, 216, 150, 52, 92, 163, 182, 83, 22, 108, 125, 239, 44, 96, 111, 55, 93, 81, 121, 83, 228, 85, 238, 237, 234, 243, 164, 34, 64, 213, 98, 150, 180, 55, 58, 162, 49, 201, 131, 151, 241, 56, 57, 80, 47, 213, 76, 89, 107, 109, 138, 221, 150, 98, 142, 136, 8, 129, 80, 214, 70, 242, 194, 65, 3, 2, 195, 109, 161, 11, 92, 47, 50, 2, 102, 137, 204, 8, 160, 37, 220, 116, 244, 252, 118, 216, 234, 52, 211, 169, 109, 95, 98, 11, 219, 134, 173, 148, 216, 9, 209, 51, 100, 196, 94, 37, 231, 30, 147, 217, 137, 206, 133, 49, 82, 22, 2, 44, 250, 157, 102, 173, 24, 211, 238, 53, 57, 228, 134, 5, 240, 35, 100, 75, 237, 144, 129, 85, 117, 143, 70, 233, 212, 118, 87, 92, 234, 245, 77, 167, 124, 154, 179, 133, 179, 145, 1, 191, 73, 67, 14, 50, 218, 34, 179, 237, 207, 242, 226, 105, 220, 27, 241, 147, 171, 39, 113, 111, 60, 200, 132, 181, 127, 92, 12, 35, 207, 53, 37, 32, 128, 47, 229, 93, 61, 230, 158, 147, 59, 135, 165, 108, 253, 101, 224, 69, 37, 157, 168, 197, 26, 107, 137, 91, 82, 160, 65, 246, 21, 228, 144, 77, 198, 47, 189, 179, 191, 51, 83, 193, 44, 53, 138, 217, 44, 164, 224, 253, 115, 253, 138, 141, 94, 225, 62, 28, 176, 54, 65, 229, 16, 51, 252, 221, 249, 161, 19, 61, 73, 122, 249, 116, 150, 78, 18, 198, 168, 246, 118, 39, 194, 85, 140, 174, 50, 91, 92, 136, 43, 150, 188, 253, 73, 181, 82, 41, 122, 228, 149, 10, 252, 104, 173, 117, 103, 132, 190, 238, 164, 22, 213, 7, 152, 113, 58, 170, 97, 235, 53, 28, 2, 197, 22, 56, 49, 212, 227, 177, 78, 60, 43, 210, 23, 226, 69, 76, 80, 199, 186, 110, 129, 208, 224, 145, 29, 65, 193, 226, 120, 231, 189, 250, 182, 253, 30, 13, 157, 244, 33, 240, 139, 54, 15, 216, 248, 36, 206, 123, 59, 140, 97, 224, 160, 31, 170, 92, 92, 143, 36, 223, 52, 98, 191, 154, 212, 97, 141, 172, 198, 169, 135, 43, 244, 144, 183, 73, 47, 159, 186, 246, 245, 69, 107, 184, 117, 12, 179, 193, 40, 149, 166, 205, 190, 3, 209, 127, 15, 179, 229, 239, 58, 172, 221, 194, 37, 161, 59, 156, 225, 179, 172, 166, 184, 105, 9, 191, 58, 84, 145, 138, 239, 30, 243, 97, 139, 246, 151, 32, 87, 138, 109, 93, 67, 114, 221, 246, 221, 171, 186, 201, 155, 105, 49, 90, 231, 150, 40, 41, 56, 105, 249, 12, 230, 18, 240, 13, 166, 120, 242, 145, 102, 132, 149, 138, 203, 86, 4, 46, 116, 192, 93, 182, 138, 60, 0, 53, 199, 176, 42, 93, 254, 63, 140, 193, 182, 191, 209, 87, 241, 171, 245, 182, 8, 85, 216, 95, 127, 107, 71, 241, 32, 145, 175, 72, 149, 76, 241, 55, 204, 30, 112, 68, 119, 197, 247, 187, 117, 109, 115, 147, 234, 97, 34, 32, 175, 171, 167, 110, 136, 26, 242, 160, 242, 93, 110, 209, 69, 239, 214, 26, 74, 130, 228, 195, 37, 228, 232, 11, 48, 227, 147, 82, 85, 223, 128, 251, 89, 126, 216, 222, 254, 110, 231, 135, 31, 118, 191, 185, 255, 221, 253, 237, 31, 126, 216, 113, 16, 25, 7, 140, 98, 70, 204, 124, 62, 201, 79, 226, 137, 25, 58, 196, 68, 53, 103, 227, 193, 213, 23, 153, 41, 150, 248, 145, 81, 190, 249, 209, 73, 152, 122, 140, 98, 122, 62, 226, 47, 160, 127, 92, 150, 233, 48, 107, 87, 204, 48, 65, 212, 55, 72, 139, 178, 58, 202, 231, 69, 143, 188, 168, 144, 59, 147, 198, 171, 224, 192, 136, 253, 142, 177, 123, 136, 47, 16, 81, 124, 203, 233, 53, 73, 242, 254, 160, 126, 110, 77, 109, 48, 103, 9, 10, 176, 25, 117, 94, 254, 138, 193, 246, 197, 53, 243, 112, 189, 132, 85, 28, 48, 107, 204, 233, 174, 210, 12, 207, 97, 252, 18, 214, 34, 159, 226, 126, 26, 92, 27, 20, 112, 156, 44, 74, 126, 29, 43, 131, 70, 87, 121, 192, 205, 126, 109, 64, 98, 129, 139, 155, 143, 117, 37, 114, 247, 49, 238, 181, 186, 250, 21, 93, 100, 220, 3, 55, 60, 122, 121, 120, 116, 116, 248, 234, 57, 28, 211, 189, 162, 213, 102, 73, 214, 103, 115, 223, 211, 139, 94, 194, 173, 29, 3, 145, 192, 6, 78, 186, 17, 166, 5, 231, 154, 101, 76, 4, 235, 4, 38, 246, 88, 87, 179, 238, 5, 166, 30, 26, 198, 210, 79, 202, 94, 84, 243, 82, 190, 39, 236, 121, 145, 206, 192, 81, 5, 102, 117, 84, 179, 193, 203, 101, 25, 58, 231, 238, 95, 40, 69, 241, 149, 255, 90, 152, 42, 63, 150, 184, 33, 156, 134, 198, 244, 111, 235, 70, 41, 175, 251, 136, 169, 112, 25, 189, 175, 89, 237, 111, 220, 51, 52, 214, 21, 85, 244, 202, 24, 205, 104, 114, 11, 23, 145, 121, 7, 229, 157, 16, 17, 34, 174, 181, 225, 213, 148, 42, 37, 98, 115, 19, 110, 235, 178, 149, 151, 70, 65, 160, 181, 142, 233, 120, 194, 199, 225, 40, 129, 235, 102, 218, 89, 60, 133, 151, 91, 240, 31, 228, 6, 86, 14, 106, 140, 4, 120, 91, 68, 85, 117, 129, 239, 253, 17, 188, 50, 92, 10, 200, 173, 159, 192, 66, 95, 32, 161, 32, 104, 176, 67, 226, 16, 237, 195, 75, 234, 188, 254, 251, 206, 247, 252, 18, 117, 0, 192, 23, 10, 142, 185, 30, 188, 203, 145, 114, 243, 174, 34, 16, 41, 232, 70, 96, 62, 124, 152, 65, 224, 251, 225, 3, 190, 2, 149, 83, 174, 5, 85, 228, 11, 59, 88, 193, 117, 248, 111, 63, 99, 68, 237, 73, 198, 138, 129, 19, 77, 35, 197, 178, 175, 224, 231, 165, 29, 160, 189, 195, 169, 243, 217, 125, 49, 120, 207, 241, 224, 13, 191, 232, 193, 27, 46, 27, 188, 225, 151, 56, 120, 67, 24, 188, 225, 186, 7, 239, 5, 127, 35, 36, 210, 60, 60, 104, 13, 122, 93, 203, 120, 103, 42, 94, 193, 146, 35, 234, 173, 56, 147, 119, 197, 178, 170, 171, 92, 143, 5, 245, 133, 134, 182, 60, 183, 97, 73, 51, 197, 167, 3, 6, 66, 13, 172, 124, 232, 185, 83, 255, 202, 21, 69, 218, 75, 157, 28, 123, 61, 144, 125, 219, 176, 172, 167, 172, 120, 71, 42, 69, 120, 180, 158, 255, 27, 142, 214, 112, 249, 104, 13, 191, 156, 209, 18, 9, 132, 15, 229, 252, 68, 188, 173, 146, 117, 48, 46, 120, 226, 68, 135, 210, 7, 163, 164, 55, 22, 13, 30, 228, 73, 209, 227, 105, 81, 43, 129, 192, 42, 243, 177, 82, 111, 62, 181, 210, 11, 220, 11, 100, 48, 196, 255, 147, 33, 206, 17, 52, 72, 98, 28, 78, 2, 125, 161, 239, 211, 12, 210, 164, 237, 36, 235, 187, 183, 35, 3, 246, 61, 84, 29, 72, 55, 149, 25, 88, 60, 81, 109, 144, 187, 255, 201, 91, 1, 20, 33, 157, 104, 123, 163, 195, 157, 68, 11, 133, 160, 160, 22, 193, 83, 184, 103, 216, 83, 157, 123, 122, 42, 6, 0, 137, 66, 52, 61, 212, 216, 177, 232, 73, 119, 220, 192, 25, 129, 224, 241, 136, 170, 19, 72, 20, 224, 234, 79, 241, 109, 201, 166, 50, 194, 74, 210, 3, 71, 74, 16, 126, 73, 138, 44, 153, 176, 62, 22, 29, 129, 239, 99, 180, 221, 97, 179, 97, 206, 162, 118, 238, 232, 67, 129, 95, 150, 120, 116, 69, 68, 10, 254, 48, 71, 201, 17, 173, 64, 4, 181, 227, 79, 158, 178, 9, 143, 145, 112, 87, 52, 179, 165, 241, 223, 165, 110, 27, 81, 87, 91, 188, 129, 8, 135, 152, 176, 66, 40, 109, 160, 183, 215, 222, 92, 33, 26, 137, 125, 153, 94, 38, 1, 181, 73, 132, 58, 72, 134, 185, 247, 133, 67, 77, 172, 18, 196, 189, 149, 89, 3, 37, 127, 219, 134, 217, 136, 44, 175, 114, 68, 155, 28, 117, 135, 15, 188, 162, 69, 36, 70, 95, 72, 82, 106, 52, 36, 81, 234, 193, 201, 235, 234, 214, 54, 52, 85, 4, 213, 195, 125, 76, 154, 28, 92, 245, 2, 216, 58, 105, 213, 212, 114, 137, 37, 56, 27, 137, 172, 45, 175, 116, 240, 111, 46, 178, 141, 229, 213, 17, 86, 61, 140, 158, 30, 76, 210, 94, 242, 215, 51, 222, 194, 144, 81, 113, 149, 23, 226, 107, 185, 2, 20, 155, 17, 54, 224, 12, 35, 17, 67, 243, 140, 90, 102, 110, 14, 165, 229, 174, 71, 45, 77, 187, 7, 49, 127, 66, 209, 150, 179, 56, 179, 85, 72, 219, 77, 163, 60, 205, 44, 46, 71, 182, 76, 110, 229, 224, 223, 138, 153, 5, 220, 77, 68, 22, 224, 26, 155, 215, 94, 222, 79, 222, 228, 140, 125, 143, 171, 246, 44, 47, 87, 145, 214, 163, 26, 89, 101, 168, 82, 153, 112, 48, 34, 5, 248, 29, 171, 122, 68, 164, 153, 103, 24, 84, 93, 38, 8, 16, 230, 233, 223, 143, 246, 121, 37, 199, 22, 121, 86, 48, 120, 14, 209, 183, 254, 162, 186, 61, 138, 139, 3, 214, 245, 199, 140, 53, 71, 29, 221, 4, 50, 204, 2, 3, 35, 225, 226, 201, 247, 219, 156, 14, 81, 242, 8, 74, 126, 122, 246, 140, 80, 182, 25, 237, 112, 141, 242, 146, 199, 235, 161, 126, 139, 183, 18, 174, 76, 27, 52, 130, 232, 147, 104, 56, 129, 7, 130, 64, 89, 196, 41, 124, 246, 236, 89, 29, 33, 234, 77, 36, 162, 79, 93, 217, 203, 141, 232, 111, 236, 219, 125, 134, 109, 83, 55, 208, 149, 13, 108, 176, 178, 237, 139, 157, 109, 246, 81, 59, 110, 122, 57, 139, 142, 153, 43, 191, 100, 165, 76, 118, 171, 72, 102, 73, 92, 137, 123, 148, 215, 37, 102, 84, 190, 4, 106, 195, 162, 140, 78, 170, 72, 181, 190, 253, 1, 199, 249, 12, 240, 111, 90, 14, 89, 157, 246, 78, 244, 240, 97, 180, 251, 253, 134, 63, 222, 255, 78, 13, 3, 180, 158, 69, 95, 71, 109, 93, 129, 113, 107, 7, 145, 247, 46, 50, 111, 74, 231, 175, 125, 123, 240, 192, 89, 22, 99, 245, 173, 91, 179, 223, 193, 11, 95, 143, 140, 175, 200, 4, 255, 17, 185, 234, 158, 215, 163, 117, 142, 172, 58, 206, 101, 211, 161, 23, 198, 190, 243, 25, 5, 181, 118, 219, 94, 215, 56, 41, 132, 12, 228, 85, 114, 46, 96, 244, 78, 1, 9, 247, 107, 86, 198, 3, 72, 232, 189, 77, 202, 164, 56, 75, 250, 71, 147, 188, 106, 155, 21, 105, 102, 10, 143, 26, 130, 238, 56, 182, 93, 61, 243, 244, 182, 233, 150, 207, 64, 53, 190, 129, 147, 108, 3, 209, 123, 15, 56, 19, 64, 49, 89, 145, 213, 103, 254, 8, 203, 130, 207, 160, 67, 246, 137, 214, 123, 153, 84, 163, 188, 127, 56, 16, 91, 225, 250, 114, 31, 156, 19, 226, 226, 79, 203, 37, 151, 190, 10, 83, 141, 157, 96, 234, 243, 164, 18, 21, 158, 177, 137, 143, 176, 87, 108, 185, 219, 38, 30, 133, 88, 43, 208, 53, 153, 34, 222, 219, 13, 84, 220, 89, 102, 250, 233, 27, 84, 130, 91, 29, 201, 246, 74, 225, 65, 3, 21, 106, 110, 32, 44, 213, 155, 11, 200, 158, 37, 196, 110, 245, 6, 19, 202, 13, 97, 128, 201, 27, 38, 174, 49, 167, 112, 186, 156, 9, 133, 209, 41, 103, 20, 120, 121, 160, 152, 99, 246, 213, 148, 194, 138, 68, 111, 96, 62, 121, 232, 116, 232, 58, 179, 135, 198, 103, 229, 88, 212, 92, 177, 175, 230, 15, 214, 182, 154, 82, 246, 213, 4, 98, 191, 181, 2, 241, 101, 215, 206, 140, 92, 213, 104, 166, 148, 0, 67, 141, 198, 164, 181, 212, 236, 233, 168, 233, 27, 138, 30, 85, 215, 234, 112, 90, 47, 169, 177, 173, 156, 88, 164, 135, 59, 234, 227, 2, 172, 89, 188, 182, 105, 233, 221, 40, 174, 232, 227, 24, 123, 226, 181, 155, 60, 154, 237, 208, 168, 221, 228, 177, 75, 80, 236, 6, 80, 168, 217, 219, 72, 232, 205, 54, 120, 16, 111, 96, 75, 253, 170, 95, 1, 245, 215, 16, 91, 35, 14, 228, 239, 186, 205, 28, 168, 142, 31, 89, 32, 47, 179, 106, 91, 238, 198, 13, 41, 54, 98, 128, 75, 221, 121, 46, 152, 239, 196, 112, 7, 163, 4, 136, 41, 14, 148, 223, 221, 214, 30, 56, 121, 173, 176, 202, 40, 56, 227, 139, 114, 15, 147, 132, 63, 144, 158, 23, 127, 107, 223, 6, 114, 54, 220, 37, 221, 200, 90, 127, 229, 47, 154, 69, 107, 126, 182, 177, 145, 27, 153, 141, 44, 249, 22, 98, 21, 224, 193, 27, 46, 242, 106, 209, 159, 149, 88, 38, 71, 2, 65, 250, 19, 109, 163, 224, 133, 124, 150, 164, 155, 77, 68, 185, 229, 25, 33, 39, 140, 185, 3, 90, 191, 20, 176, 183, 61, 25, 69, 200, 95, 143, 184, 19, 251, 140, 184, 199, 215, 71, 13, 150, 242, 153, 199, 82, 170, 161, 217, 154, 205, 203, 145, 194, 241, 192, 18, 126, 123, 249, 223, 100, 152, 105, 117, 246, 145, 237, 117, 149, 7, 190, 17, 253, 183, 112, 212, 69, 242, 143, 187, 230, 220, 129, 127, 16, 194, 225, 98, 248, 47, 237, 234, 75, 7, 223, 35, 234, 225, 119, 185, 49, 255, 99, 178, 232, 68, 200, 56, 115, 169, 150, 147, 79, 71, 44, 95, 105, 34, 234, 180, 161, 136, 207, 121, 102, 255, 136, 249, 104, 157, 104, 107, 75, 36, 103, 210, 106, 46, 20, 16, 233, 132, 216, 72, 242, 122, 112, 132, 1, 32, 190, 199, 191, 29, 53, 233, 229, 249, 152, 239, 237, 213, 11, 24, 170, 53, 156, 59, 139, 207, 9, 4, 175, 179, 197, 74, 177, 217, 230, 78, 228, 228, 40, 25, 42, 181, 123, 151, 139, 116, 86, 155, 1, 110, 57, 169, 61, 27, 190, 54, 143, 102, 188, 31, 193, 29, 20, 57, 4, 182, 102, 212, 236, 204, 228, 155, 70, 146, 33, 158, 130, 24, 133, 72, 123, 145, 156, 88, 141, 146, 159, 155, 10, 147, 43, 253, 135, 218, 79, 1, 3, 111, 117, 213, 123, 122, 5, 99, 118, 173, 7, 27, 211, 80, 35, 15, 253, 3, 111, 109, 18, 20, 56, 112, 151, 137, 88, 248, 59, 239, 132, 73, 6, 141, 226, 126, 67, 30, 97, 242, 157, 93, 20, 87, 97, 233, 167, 78, 8, 243, 110, 118, 120, 234, 102, 247, 38, 25, 35, 233, 242, 111, 17, 220, 173, 166, 14, 72, 51, 239, 161, 214, 1, 185, 71, 80, 220, 243, 160, 168, 201, 95, 17, 170, 57, 51, 36, 75, 58, 145, 103, 118, 118, 25, 123, 146, 14, 27, 199, 173, 74, 9, 31, 178, 74, 143, 90, 108, 44, 169, 71, 199, 10, 90, 15, 255, 155, 63, 243, 182, 52, 73, 179, 241, 53, 218, 130, 106, 225, 214, 196, 83, 111, 123, 249, 164, 127, 141, 230, 194, 77, 249, 155, 25, 164, 23, 201, 53, 218, 169, 170, 96, 67, 240, 200, 159, 103, 136, 39, 105, 175, 92, 189, 173, 52, 216, 84, 234, 111, 169, 156, 50, 73, 91, 189, 29, 94, 45, 216, 150, 124, 234, 159, 219, 138, 116, 156, 92, 163, 65, 94, 47, 220, 162, 124, 28, 88, 40, 187, 70, 123, 243, 176, 120, 240, 103, 254, 150, 102, 215, 105, 105, 86, 211, 210, 204, 106, 233, 105, 217, 139, 103, 44, 134, 101, 160, 39, 243, 42, 249, 59, 4, 134, 237, 51, 250, 214, 214, 217, 28, 108, 48, 182, 248, 103, 36, 57, 193, 158, 191, 224, 75, 38, 10, 212, 113, 13, 242, 121, 165, 145, 224, 217, 182, 55, 154, 103, 99, 181, 162, 178, 109, 185, 212, 226, 245, 154, 17, 108, 91, 83, 109, 60, 128, 23, 71, 218, 241, 144, 106, 85, 189, 112, 242, 110, 235, 174, 237, 38, 154, 246, 55, 27, 5, 189, 10, 101, 7, 81, 200, 74, 129, 139, 119, 191, 254, 99, 158, 87, 15, 238, 90, 78, 38, 238, 72, 10, 211, 181, 63, 108, 231, 241, 27, 130, 221, 247, 185, 42, 170, 245, 7, 190, 42, 134, 25, 104, 15, 214, 26, 186, 71, 197, 72, 99, 244, 137, 101, 156, 245, 70, 57, 221, 227, 114, 147, 64, 94, 54, 121, 247, 97, 28, 1, 202, 253, 214, 93, 198, 63, 175, 92, 138, 22, 217, 24, 180, 30, 1, 204, 145, 16, 234, 56, 96, 93, 153, 223, 207, 162, 197, 28, 210, 213, 236, 255, 181, 82, 10, 184, 35, 142, 182, 134, 90, 217, 172, 77, 46, 212, 13, 83, 12, 169, 168, 54, 206, 71, 173, 143, 94, 192, 90, 67, 174, 104, 180, 57, 181, 124, 78, 158, 23, 147, 53, 139, 192, 168, 72, 6, 53, 84, 242, 6, 151, 74, 64, 82, 93, 227, 116, 219, 81, 205, 233, 182, 163, 240, 233, 54, 214, 152, 247, 160, 153, 202, 161, 47, 57, 221, 198, 170, 139, 211, 109, 10, 190, 209, 233, 182, 101, 141, 222, 242, 233, 54, 158, 63, 244, 30, 110, 131, 238, 152, 245, 1, 69, 206, 181, 14, 183, 97, 84, 242, 112, 155, 104, 215, 202, 17, 172, 114, 182, 77, 37, 62, 175, 123, 182, 141, 87, 208, 127, 12, 183, 145, 240, 189, 99, 35, 209, 231, 219, 233, 159, 138, 163, 120, 228, 172, 218, 235, 26, 33, 123, 173, 214, 48, 12, 10, 86, 214, 100, 241, 194, 84, 112, 22, 46, 94, 51, 49, 113, 72, 162, 11, 19, 18, 235, 1, 63, 52, 204, 193, 244, 242, 201, 107, 177, 158, 224, 239, 222, 89, 82, 44, 154, 42, 88, 120, 193, 6, 117, 182, 241, 98, 77, 77, 127, 69, 10, 220, 55, 120, 68, 54, 236, 79, 203, 234, 151, 187, 116, 67, 71, 206, 236, 19, 49, 21, 101, 142, 226, 117, 221, 97, 25, 107, 198, 247, 196, 148, 255, 101, 16, 254, 215, 214, 76, 237, 117, 220, 74, 48, 85, 183, 166, 216, 239, 100, 39, 25, 144, 229, 141, 141, 133, 55, 54, 22, 9, 77, 246, 197, 205, 98, 142, 255, 46, 181, 235, 245, 241, 216, 202, 91, 86, 73, 9, 43, 21, 30, 77, 199, 35, 245, 174, 35, 113, 176, 191, 76, 114, 45, 61, 191, 99, 144, 120, 243, 30, 232, 208, 23, 145, 107, 231, 168, 185, 97, 240, 179, 148, 5, 43, 82, 153, 75, 229, 134, 118, 228, 102, 179, 207, 45, 195, 152, 46, 78, 83, 99, 9, 134, 110, 173, 65, 128, 133, 121, 158, 196, 85, 122, 150, 44, 223, 246, 52, 230, 102, 21, 67, 63, 140, 182, 9, 197, 222, 173, 70, 164, 74, 7, 235, 6, 124, 172, 221, 71, 22, 48, 221, 132, 164, 30, 138, 205, 87, 206, 222, 171, 102, 251, 102, 81, 39, 228, 22, 41, 135, 233, 181, 221, 144, 219, 166, 156, 74, 129, 142, 88, 123, 169, 196, 222, 5, 174, 100, 156, 26, 71, 205, 64, 179, 24, 137, 104, 222, 34, 162, 254, 58, 44, 231, 176, 43, 125, 5, 103, 232, 147, 207, 83, 215, 179, 219, 45, 187, 143, 174, 216, 127, 82, 43, 61, 32, 68, 124, 58, 51, 45, 142, 4, 240, 55, 77, 235, 203, 60, 248, 170, 219, 7, 244, 147, 249, 196, 100, 196, 76, 213, 3, 216, 36, 48, 75, 122, 204, 53, 64, 240, 48, 251, 187, 72, 177, 202, 39, 179, 202, 179, 40, 198, 35, 251, 120, 198, 68, 128, 47, 7, 216, 113, 253, 205, 102, 146, 50, 153, 48, 25, 148, 235, 12, 63, 229, 249, 36, 137, 179, 118, 227, 25, 197, 89, 156, 23, 200, 236, 124, 1, 116, 76, 172, 231, 136, 218, 246, 146, 146, 236, 156, 123, 120, 207, 136, 221, 99, 201, 152, 131, 182, 130, 118, 150, 12, 229, 82, 33, 227, 131, 2, 97, 191, 40, 51, 30, 31, 103, 160, 239, 64, 15, 251, 230, 209, 248, 199, 65, 141, 207, 250, 237, 25, 67, 153, 178, 104, 34, 249, 146, 92, 51, 151, 168, 21, 230, 181, 172, 255, 197, 57, 102, 3, 68, 148, 173, 240, 186, 175, 141, 245, 29, 213, 184, 93, 175, 204, 57, 231, 131, 198, 197, 209, 26, 159, 187, 37, 32, 60, 34, 137, 86, 32, 130, 162, 201, 87, 109, 254, 146, 242, 201, 123, 246, 69, 10, 41, 161, 236, 75, 147, 212, 37, 226, 8, 178, 91, 43, 140, 30, 57, 236, 238, 132, 4, 112, 181, 220, 208, 138, 242, 119, 43, 190, 79, 203, 161, 254, 182, 69, 204, 179, 63, 222, 32, 194, 146, 37, 200, 185, 219, 200, 229, 249, 52, 33, 105, 157, 55, 128, 4, 233, 26, 166, 139, 171, 208, 235, 1, 243, 27, 226, 162, 55, 122, 42, 54, 93, 136, 179, 11, 122, 211, 192, 103, 50, 91, 33, 146, 26, 27, 47, 217, 183, 53, 201, 21, 7, 241, 174, 110, 116, 209, 78, 79, 186, 173, 92, 211, 76, 182, 150, 63, 218, 231, 1, 82, 29, 154, 177, 5, 191, 141, 57, 10, 82, 146, 225, 62, 89, 91, 212, 224, 185, 136, 225, 50, 58, 59, 142, 233, 174, 118, 5, 187, 141, 113, 161, 128, 45, 100, 213, 68, 200, 6, 103, 22, 240, 8, 221, 220, 138, 253, 79, 14, 199, 205, 146, 89, 44, 174, 119, 251, 92, 82, 39, 219, 39, 89, 70, 32, 109, 77, 249, 7, 214, 0, 108, 60, 82, 205, 56, 33, 125, 171, 211, 226, 33, 189, 218, 138, 162, 249, 209, 68, 20, 241, 82, 164, 220, 65, 181, 205, 29, 165, 237, 247, 230, 193, 91, 188, 102, 161, 160, 108, 95, 126, 204, 119, 48, 215, 250, 97, 176, 188, 240, 22, 206, 19, 38, 51, 26, 209, 72, 156, 222, 104, 71, 110, 12, 180, 219, 71, 241, 8, 144, 119, 36, 55, 199, 120, 196, 231, 109, 64, 122, 248, 245, 53, 95, 80, 98, 26, 232, 89, 37, 43, 189, 27, 202, 74, 191, 136, 203, 170, 129, 153, 254, 220, 94, 230, 141, 205, 53, 234, 231, 39, 55, 217, 136, 145, 158, 12, 91, 55, 218, 33, 89, 54, 219, 172, 43, 125, 17, 166, 154, 164, 211, 152, 69, 203, 58, 10, 9, 156, 61, 36, 134, 89, 27, 219, 71, 98, 222, 239, 118, 63, 149, 173, 133, 171, 197, 255, 122, 201, 51, 44, 78, 112, 93, 230, 167, 73, 157, 221, 13, 132, 40, 211, 120, 214, 204, 137, 116, 49, 255, 171, 229, 205, 76, 110, 200, 74, 253, 54, 12, 233, 167, 124, 216, 255, 238, 59, 196, 177, 204, 221, 133, 207, 99, 145, 68, 70, 88, 86, 72, 44, 189, 77, 250, 243, 30, 30, 146, 232, 26, 22, 116, 189, 178, 222, 178, 169, 195, 91, 48, 63, 115, 224, 93, 16, 130, 110, 69, 176, 137, 237, 134, 83, 36, 94, 234, 3, 27, 89, 191, 181, 140, 242, 182, 249, 25, 247, 122, 243, 233, 92, 29, 223, 104, 118, 130, 134, 201, 219, 230, 230, 123, 103, 49, 195, 39, 200, 20, 125, 80, 136, 209, 185, 52, 84, 99, 89, 32, 135, 64, 107, 229, 248, 109, 58, 28, 85, 95, 182, 48, 115, 18, 191, 56, 137, 38, 84, 125, 249, 98, 45, 93, 138, 27, 11, 119, 183, 107, 11, 183, 215, 23, 249, 236, 178, 125, 150, 20, 101, 210, 224, 180, 236, 173, 248, 182, 45, 135, 146, 53, 5, 131, 211, 180, 223, 159, 136, 27, 98, 164, 199, 56, 152, 228, 57, 191, 236, 37, 250, 111, 30, 18, 232, 161, 17, 187, 239, 207, 249, 93, 178, 108, 124, 196, 87, 56, 17, 37, 112, 200, 18, 119, 106, 157, 195, 21, 240, 90, 96, 68, 181, 46, 222, 205, 169, 17, 155, 180, 58, 255, 105, 5, 110, 28, 143, 1, 225, 63, 17, 136, 170, 4, 30, 164, 134, 196, 143, 69, 5, 32, 68, 183, 229, 17, 133, 208, 122, 239, 81, 224, 110, 148, 207, 21, 232, 161, 61, 12, 88, 58, 56, 153, 43, 187, 157, 255, 217, 150, 240, 169, 183, 37, 136, 211, 93, 226, 60, 50, 105, 68, 208, 214, 5, 83, 133, 207, 171, 127, 78, 143, 26, 223, 77, 97, 86, 89, 249, 79, 185, 11, 81, 239, 174, 240, 44, 180, 210, 12, 204, 24, 47, 240, 102, 230, 144, 17, 82, 132, 144, 155, 124, 148, 79, 147, 191, 224, 222, 56, 232, 214, 23, 183, 184, 85, 34, 162, 254, 45, 119, 198, 45, 217, 24, 103, 110, 100, 32, 162, 235, 94, 101, 110, 216, 203, 219, 250, 162, 18, 132, 130, 162, 85, 82, 132, 214, 5, 82, 120, 165, 163, 55, 153, 247, 89, 247, 190, 248, 101, 156, 90, 181, 52, 57, 67, 186, 140, 35, 58, 119, 235, 73, 65, 235, 234, 254, 250, 165, 156, 207, 187, 74, 131, 12, 191, 117, 25, 58, 96, 56, 138, 167, 226, 112, 193, 63, 147, 34, 183, 101, 2, 20, 214, 191, 228, 76, 239, 57, 25, 123, 103, 135, 58, 21, 19, 87, 242, 195, 37, 46, 237, 146, 223, 59, 223, 129, 140, 204, 32, 195, 51, 196, 210, 76, 227, 129, 79, 41, 241, 196, 121, 16, 184, 74, 121, 219, 178, 123, 59, 157, 232, 0, 155, 187, 129, 72, 16, 13, 50, 131, 121, 224, 94, 204, 142, 45, 237, 160, 153, 129, 101, 13, 13, 84, 67, 234, 106, 36, 179, 182, 13, 220, 56, 32, 129, 145, 226, 205, 64, 243, 34, 164, 214, 188, 114, 207, 244, 157, 85, 144, 47, 58, 72, 171, 100, 90, 74, 246, 26, 52, 22, 27, 81, 205, 7, 181, 122, 215, 228, 115, 115, 12, 55, 71, 114, 147, 234, 40, 230, 98, 230, 73, 156, 198, 22, 210, 73, 100, 96, 73, 102, 117, 94, 162, 11, 145, 24, 192, 243, 164, 18, 102, 175, 45, 71, 4, 191, 70, 14, 25, 29, 90, 239, 78, 232, 50, 20, 126, 72, 144, 96, 199, 239, 14, 3, 252, 4, 17, 154, 52, 201, 25, 22, 123, 251, 34, 124, 2, 7, 86, 84, 93, 185, 44, 167, 26, 23, 239, 246, 180, 54, 9, 210, 137, 131, 95, 211, 225, 250, 223, 72, 81, 190, 217, 241, 32, 224, 111, 17, 116, 79, 182, 192, 199, 58, 221, 34, 72, 131, 46, 153, 187, 48, 196, 249, 22, 132, 246, 138, 112, 64, 152, 127, 89, 201, 247, 158, 10, 161, 63, 172, 7, 158, 124, 53, 124, 154, 186, 64, 10, 159, 118, 131, 68, 155, 196, 23, 82, 48, 52, 183, 173, 164, 239, 71, 58, 149, 74, 77, 198, 94, 17, 172, 22, 141, 173, 48, 26, 62, 186, 15, 129, 164, 55, 101, 139, 178, 237, 170, 146, 61, 135, 198, 124, 154, 76, 199, 73, 132, 46, 177, 224, 194, 134, 108, 167, 156, 86, 213, 157, 21, 186, 14, 189, 185, 98, 25, 131, 175, 229, 95, 234, 182, 28, 63, 115, 157, 124, 93, 198, 83, 122, 252, 194, 240, 178, 102, 30, 124, 61, 104, 71, 216, 34, 11, 30, 134, 239, 228, 225, 60, 199, 16, 52, 132, 188, 238, 156, 40, 143, 209, 29, 224, 0, 50, 57, 191, 198, 0, 153, 176, 145, 87, 23, 140, 226, 68, 179, 175, 152, 61, 226, 185, 203, 27, 33, 91, 204, 166, 241, 91, 218, 11, 65, 26, 148, 118, 162, 124, 48, 40, 147, 202, 86, 176, 101, 31, 32, 154, 51, 3, 161, 63, 44, 31, 87, 21, 220, 227, 212, 111, 43, 244, 146, 55, 56, 250, 186, 255, 29, 239, 113, 121, 158, 242, 115, 137, 75, 110, 161, 147, 228, 110, 75, 127, 184, 23, 51, 63, 110, 103, 207, 153, 239, 117, 255, 142, 152, 4, 198, 69, 176, 151, 15, 20, 142, 221, 26, 28, 111, 225, 205, 211, 73, 214, 75, 150, 163, 249, 166, 142, 148, 116, 218, 95, 142, 225, 254, 158, 250, 118, 175, 6, 215, 147, 164, 72, 25, 75, 106, 209, 93, 173, 52, 128, 214, 88, 46, 223, 178, 214, 152, 150, 21, 37, 137, 180, 255, 42, 57, 151, 232, 81, 155, 53, 189, 94, 70, 228, 225, 32, 84, 185, 3, 46, 154, 116, 0, 212, 181, 170, 252, 30, 189, 31, 87, 237, 46, 186, 37, 165, 142, 162, 215, 179, 248, 143, 121, 88, 164, 174, 197, 53, 176, 23, 57, 199, 171, 218, 138, 248, 85, 157, 175, 73, 153, 102, 33, 23, 19, 161, 163, 248, 25, 197, 16, 144, 46, 117, 148, 159, 192, 46, 237, 238, 26, 248, 191, 140, 111, 13, 217, 191, 196, 48, 112, 58, 248, 36, 202, 158, 68, 75, 47, 199, 148, 104, 190, 39, 166, 12, 170, 34, 51, 181, 77, 244, 249, 69, 30, 51, 239, 52, 171, 190, 111, 135, 109, 193, 142, 182, 5, 223, 187, 117, 231, 75, 42, 239, 122, 155, 219, 249, 182, 166, 202, 61, 127, 43, 181, 117, 238, 123, 155, 185, 183, 91, 83, 229, 27, 127, 51, 181, 117, 190, 117, 235, 12, 38, 121, 92, 95, 233, 187, 64, 165, 111, 239, 251, 43, 93, 195, 98, 174, 104, 43, 151, 79, 37, 159, 70, 240, 30, 103, 139, 101, 114, 71, 43, 168, 12, 213, 42, 226, 86, 202, 155, 24, 62, 15, 171, 107, 167, 219, 91, 224, 178, 100, 26, 32, 133, 20, 95, 173, 136, 194, 109, 125, 200, 9, 92, 52, 128, 191, 143, 224, 47, 27, 192, 127, 143, 224, 207, 27, 192, 239, 168, 5, 78, 201, 98, 198, 139, 103, 2, 250, 226, 62, 48, 150, 191, 242, 167, 189, 209, 190, 232, 68, 139, 78, 116, 217, 137, 206, 157, 177, 119, 187, 238, 81, 180, 218, 174, 251, 225, 191, 247, 83, 246, 237, 253, 139, 93, 155, 50, 67, 211, 182, 135, 38, 191, 145, 9, 81, 20, 132, 246, 14, 69, 16, 218, 59, 16, 65, 104, 207, 48, 112, 145, 172, 29, 132, 149, 149, 105, 21, 85, 58, 10, 7, 9, 157, 72, 188, 244, 13, 242, 177, 226, 36, 35, 87, 135, 207, 28, 7, 80, 218, 235, 231, 121, 76, 59, 230, 187, 45, 218, 20, 231, 82, 43, 238, 50, 198, 139, 252, 27, 47, 193, 53, 118, 107, 41, 198, 251, 70, 234, 245, 213, 179, 237, 134, 188, 252, 97, 67, 189, 131, 8, 80, 192, 32, 82, 194, 196, 98, 132, 4, 22, 216, 235, 41, 84, 97, 170, 161, 17, 229, 149, 176, 195, 65, 147, 89, 46, 2, 148, 140, 18, 107, 214, 201, 164, 255, 138, 49, 184, 196, 150, 91, 212, 174, 55, 221, 168, 62, 87, 165, 21, 16, 252, 96, 35, 120, 205, 251, 187, 2, 134, 29, 103, 95, 63, 186, 131, 204, 244, 73, 37, 35, 232, 101, 100, 164, 223, 176, 73, 90, 195, 163, 215, 217, 210, 206, 41, 40, 209, 83, 31, 152, 232, 130, 130, 147, 29, 242, 1, 170, 196, 142, 30, 154, 99, 221, 190, 6, 182, 76, 134, 105, 218, 21, 18, 102, 231, 80, 139, 29, 211, 153, 14, 106, 206, 179, 22, 167, 108, 29, 78, 171, 236, 90, 203, 88, 34, 199, 130, 132, 200, 122, 190, 236, 126, 124, 53, 88, 214, 139, 142, 111, 168, 10, 198, 46, 6, 197, 220, 94, 248, 226, 139, 34, 10, 70, 173, 98, 67, 182, 90, 230, 249, 130, 149, 244, 38, 179, 109, 156, 242, 130, 179, 29, 74, 44, 174, 161, 51, 80, 253, 72, 220, 247, 191, 196, 154, 232, 134, 58, 209, 61, 11, 129, 150, 55, 193, 160, 176, 54, 120, 85, 192, 18, 48, 212, 142, 225, 188, 105, 69, 221, 93, 108, 68, 54, 53, 87, 198, 34, 98, 54, 247, 117, 223, 140, 192, 93, 153, 249, 209, 127, 191, 203, 53, 166, 151, 213, 2, 122, 21, 205, 223, 86, 20, 112, 84, 229, 69, 226, 143, 63, 3, 235, 194, 109, 36, 177, 226, 186, 234, 175, 249, 181, 217, 161, 176, 65, 180, 224, 143, 82, 67, 75, 207, 164, 141, 71, 143, 30, 249, 90, 249, 222, 76, 113, 103, 140, 51, 7, 147, 120, 58, 123, 151, 255, 202, 219, 217, 244, 79, 143, 181, 189, 141, 206, 66, 65, 140, 174, 229, 9, 128, 27, 245, 0, 113, 9, 247, 224, 94, 128, 79, 43, 52, 83, 195, 41, 220, 210, 125, 127, 127, 60, 174, 103, 227, 254, 132, 194, 120, 211, 141, 21, 176, 251, 186, 17, 138, 249, 69, 3, 161, 8, 166, 19, 89, 99, 239, 73, 0, 32, 12, 158, 64, 195, 193, 112, 203, 33, 234, 245, 252, 200, 91, 79, 15, 8, 38, 249, 242, 3, 126, 167, 214, 142, 125, 213, 29, 120, 102, 96, 205, 203, 158, 127, 52, 117, 249, 150, 68, 119, 38, 244, 104, 109, 32, 243, 160, 168, 57, 163, 148, 248, 52, 56, 144, 135, 80, 24, 244, 241, 71, 139, 142, 219, 31, 255, 166, 174, 126, 173, 247, 192, 247, 11, 169, 64, 11, 167, 113, 109, 7, 163, 206, 107, 90, 238, 55, 173, 230, 57, 73, 114, 133, 235, 249, 244, 143, 121, 122, 166, 30, 35, 10, 77, 68, 139, 72, 253, 28, 180, 174, 77, 159, 168, 58, 68, 75, 108, 22, 36, 30, 58, 52, 49, 163, 187, 212, 81, 150, 176, 17, 154, 251, 97, 52, 247, 87, 64, 243, 125, 24, 205, 247, 43, 160, 97, 35, 22, 196, 179, 179, 107, 16, 161, 176, 14, 235, 110, 180, 196, 80, 219, 108, 99, 32, 75, 217, 22, 72, 44, 133, 209, 124, 31, 38, 115, 155, 146, 25, 78, 42, 117, 112, 174, 167, 142, 196, 112, 166, 41, 132, 194, 25, 213, 112, 250, 41, 132, 194, 25, 209, 154, 156, 84, 8, 71, 96, 52, 229, 102, 234, 189, 213, 13, 168, 48, 161, 228, 198, 211, 131, 60, 59, 75, 138, 234, 113, 214, 63, 200, 103, 139, 119, 57, 232, 96, 37, 28, 255, 26, 252, 0, 164, 87, 176, 150, 192, 189, 174, 247, 81, 216, 199, 4, 168, 117, 64, 218, 28, 125, 181, 138, 171, 255, 96, 85, 46, 105, 235, 234, 79, 177, 225, 158, 135, 211, 108, 110, 130, 79, 242, 148, 114, 14, 243, 135, 132, 233, 200, 7, 64, 99, 245, 54, 73, 7, 11, 185, 225, 208, 63, 68, 226, 89, 112, 100, 196, 227, 215, 77, 151, 64, 155, 241, 110, 9, 195, 40, 73, 97, 150, 121, 23, 226, 73, 103, 237, 222, 209, 238, 56, 142, 0, 175, 196, 39, 199, 152, 5, 188, 85, 59, 175, 70, 73, 193, 11, 125, 51, 190, 217, 28, 140, 39, 123, 49, 127, 162, 163, 130, 132, 240, 111, 204, 84, 172, 241, 224, 86, 252, 216, 16, 68, 16, 167, 100, 6, 158, 224, 197, 22, 99, 84, 57, 228, 249, 168, 227, 135, 233, 52, 201, 74, 216, 24, 148, 37, 231, 188, 236, 157, 158, 72, 155, 116, 189, 143, 159, 54, 236, 60, 109, 201, 223, 125, 2, 19, 196, 203, 151, 218, 39, 125, 13, 169, 182, 58, 219, 174, 141, 190, 135, 92, 86, 144, 123, 127, 101, 21, 140, 193, 5, 34, 111, 47, 132, 142, 44, 241, 78, 48, 54, 254, 122, 229, 59, 251, 108, 166, 8, 245, 64, 238, 66, 92, 142, 19, 145, 204, 177, 130, 63, 127, 95, 105, 167, 77, 237, 223, 246, 173, 94, 146, 205, 85, 62, 6, 96, 216, 196, 124, 87, 46, 184, 220, 37, 69, 171, 225, 49, 122, 224, 0, 17, 190, 53, 235, 38, 109, 65, 118, 211, 244, 210, 198, 253, 183, 125, 139, 38, 210, 75, 31, 185, 75, 122, 9, 163, 107, 179, 18, 146, 125, 86, 195, 138, 30, 207, 112, 42, 44, 196, 233, 182, 71, 143, 18, 178, 177, 4, 93, 83, 227, 90, 187, 89, 7, 15, 149, 58, 135, 96, 191, 125, 155, 197, 65, 111, 224, 192, 205, 145, 137, 212, 4, 105, 141, 66, 214, 86, 42, 86, 200, 90, 129, 76, 84, 107, 160, 22, 50, 13, 196, 174, 7, 2, 22, 20, 91, 171, 70, 126, 75, 67, 62, 232, 28, 244, 93, 239, 216, 108, 214, 59, 79, 63, 238, 7, 168, 223, 189, 29, 154, 243, 35, 62, 123, 181, 215, 102, 145, 225, 205, 2, 135, 47, 159, 192, 171, 5, 42, 137, 220, 188, 91, 128, 216, 201, 190, 76, 84, 215, 91, 212, 6, 22, 81, 47, 19, 130, 41, 252, 230, 38, 36, 93, 35, 62, 132, 106, 51, 35, 214, 112, 246, 208, 39, 232, 10, 84, 47, 216, 249, 37, 198, 74, 239, 239, 107, 211, 46, 71, 18, 181, 180, 217, 106, 183, 54, 129, 250, 173, 139, 77, 232, 154, 248, 190, 64, 223, 47, 209, 247, 243, 205, 214, 6, 23, 123, 126, 40, 134, 54, 177, 187, 114, 19, 27, 244, 101, 25, 78, 6, 96, 167, 35, 70, 119, 119, 197, 132, 120, 19, 153, 94, 43, 50, 217, 231, 70, 153, 132, 29, 72, 37, 240, 121, 177, 17, 248, 174, 243, 222, 3, 206, 166, 107, 104, 92, 173, 251, 39, 79, 110, 89, 194, 221, 226, 25, 179, 214, 198, 10, 125, 20, 70, 219, 34, 89, 108, 175, 30, 229, 69, 53, 138, 179, 126, 123, 107, 107, 139, 57, 116, 229, 122, 61, 87, 117, 246, 131, 145, 134, 76, 193, 203, 188, 63, 159, 36, 109, 173, 17, 208, 174, 57, 230, 169, 206, 107, 5, 208, 241, 251, 13, 208, 30, 110, 178, 252, 132, 49, 193, 185, 124, 86, 40, 110, 96, 72, 187, 93, 129, 85, 85, 135, 237, 218, 239, 182, 244, 196, 214, 230, 229, 29, 142, 64, 45, 52, 225, 11, 21, 172, 87, 220, 228, 69, 60, 100, 97, 1, 182, 153, 57, 11, 61, 52, 239, 140, 179, 14, 1, 9, 58, 43, 168, 24, 232, 110, 232, 204, 117, 232, 98, 54, 127, 207, 39, 147, 7, 78, 213, 119, 69, 156, 149, 240, 82, 190, 172, 114, 234, 107, 149, 172, 9, 151, 112, 67, 14, 159, 191, 211, 233, 128, 165, 102, 28, 240, 96, 224, 147, 69, 149, 188, 80, 38, 176, 153, 153, 189, 167, 17, 200, 30, 255, 25, 157, 204, 7, 131, 164, 216, 195, 77, 254, 196, 139, 120, 123, 78, 188, 105, 26, 221, 67, 223, 189, 96, 34, 138, 163, 152, 117, 177, 24, 190, 43, 186, 130, 232, 72, 160, 21, 14, 53, 21, 0, 56, 11, 231, 155, 102, 205, 72, 184, 161, 169, 203, 231, 229, 186, 132, 221, 17, 54, 253, 105, 130, 164, 211, 209, 202, 121, 141, 214, 158, 110, 110, 75, 224, 64, 185, 177, 150, 234, 96, 61, 148, 200, 200, 99, 24, 252, 66, 81, 1, 35, 222, 22, 138, 97, 164, 147, 134, 129, 180, 251, 130, 225, 30, 103, 11, 154, 135, 114, 30, 185, 175, 212, 65, 145, 41, 252, 254, 105, 158, 78, 250, 237, 184, 115, 210, 233, 173, 223, 162, 141, 83, 126, 89, 195, 18, 25, 55, 158, 178, 25, 25, 168, 137, 28, 197, 251, 102, 251, 140, 28, 179, 152, 79, 66, 102, 20, 44, 139, 192, 187, 197, 251, 123, 148, 252, 113, 56, 157, 77, 100, 43, 124, 242, 86, 251, 24, 216, 100, 22, 75, 197, 210, 30, 1, 69, 47, 135, 6, 228, 82, 62, 56, 89, 67, 187, 113, 39, 58, 105, 212, 174, 66, 238, 97, 47, 121, 23, 172, 231, 57, 201, 76, 122, 163, 158, 37, 130, 193, 143, 129, 2, 165, 157, 232, 22, 36, 131, 69, 120, 208, 230, 33, 188, 65, 143, 73, 136, 65, 27, 135, 157, 237, 88, 207, 131, 184, 54, 217, 250, 113, 49, 155, 164, 189, 180, 122, 146, 204, 196, 134, 176, 19, 121, 185, 145, 153, 28, 40, 8, 27, 87, 124, 201, 129, 61, 175, 188, 140, 103, 191, 138, 166, 212, 104, 198, 250, 246, 7, 123, 0, 17, 158, 147, 21, 240, 156, 52, 26, 77, 242, 66, 120, 111, 79, 169, 120, 42, 249, 185, 126, 191, 223, 89, 212, 158, 92, 183, 227, 54, 162, 29, 95, 207, 93, 18, 111, 32, 250, 181, 162, 13, 87, 80, 114, 253, 91, 115, 246, 141, 71, 78, 12, 162, 81, 134, 76, 59, 29, 188, 156, 185, 28, 218, 187, 82, 40, 234, 211, 124, 215, 51, 82, 129, 49, 17, 163, 33, 5, 82, 228, 46, 196, 208, 4, 108, 211, 42, 88, 119, 48, 86, 161, 191, 203, 124, 5, 43, 117, 10, 151, 36, 254, 203, 143, 87, 236, 30, 249, 215, 60, 189, 227, 225, 105, 40, 11, 203, 223, 38, 42, 41, 86, 45, 219, 57, 54, 117, 97, 49, 103, 157, 103, 72, 12, 18, 61, 17, 213, 140, 128, 124, 227, 17, 60, 255, 151, 30, 1, 201, 110, 232, 71, 83, 142, 75, 70, 10, 14, 120, 24, 201, 121, 66, 152, 247, 234, 215, 151, 31, 126, 250, 253, 221, 211, 163, 246, 73, 90, 149, 244, 132, 23, 148, 68, 155, 209, 119, 176, 249, 39, 186, 71, 163, 164, 167, 239, 62, 252, 116, 248, 174, 221, 143, 171, 24, 94, 42, 11, 215, 125, 232, 221, 47, 231, 121, 1, 46, 20, 47, 149, 85, 213, 65, 228, 146, 223, 150, 29, 61, 124, 8, 239, 238, 133, 199, 176, 45, 74, 196, 37, 128, 234, 24, 170, 190, 143, 62, 238, 115, 80, 210, 226, 243, 245, 183, 168, 250, 137, 90, 254, 154, 215, 224, 9, 169, 109, 210, 188, 235, 36, 197, 38, 111, 202, 156, 36, 72, 156, 204, 192, 81, 50, 98, 119, 187, 31, 29, 95, 243, 201, 241, 33, 92, 148, 200, 100, 186, 45, 126, 203, 221, 109, 32, 53, 125, 51, 51, 5, 4, 250, 79, 113, 151, 1, 235, 229, 209, 44, 134, 251, 54, 134, 69, 156, 102, 50, 35, 156, 87, 241, 68, 196, 93, 87, 209, 62, 199, 2, 111, 141, 159, 87, 201, 33, 169, 132, 185, 33, 25, 161, 78, 75, 139, 139, 208, 228, 45, 71, 16, 153, 107, 208, 182, 126, 206, 70, 38, 237, 121, 110, 100, 144, 229, 58, 127, 32, 112, 59, 153, 1, 185, 49, 85, 60, 196, 251, 82, 101, 125, 254, 122, 104, 113, 47, 140, 90, 180, 208, 125, 132, 243, 225, 108, 56, 249, 205, 63, 88, 91, 143, 82, 40, 229, 100, 106, 88, 77, 111, 217, 104, 179, 45, 226, 227, 61, 108, 16, 245, 78, 219, 109, 221, 19, 212, 11, 196, 114, 210, 23, 113, 171, 70, 82, 188, 158, 87, 111, 242, 52, 171, 18, 43, 120, 199, 199, 56, 81, 203, 130, 245, 29, 211, 110, 253, 205, 66, 190, 143, 203, 171, 13, 21, 230, 89, 239, 109, 229, 38, 238, 3, 92, 66, 209, 81, 188, 239, 88, 68, 163, 93, 199, 206, 5, 153, 128, 66, 253, 248, 16, 207, 102, 147, 133, 208, 39, 114, 43, 140, 196, 187, 132, 132, 124, 102, 67, 242, 247, 197, 90, 55, 138, 40, 135, 204, 112, 17, 214, 219, 107, 185, 39, 119, 46, 171, 126, 28, 102, 189, 130, 207, 101, 150, 62, 232, 222, 83, 229, 146, 181, 140, 16, 108, 238, 115, 97, 82, 162, 41, 77, 146, 104, 246, 1, 221, 105, 209, 92, 237, 62, 129, 249, 177, 63, 107, 205, 240, 222, 34, 114, 109, 113, 8, 31, 93, 195, 67, 30, 123, 236, 15, 5, 56, 222, 126, 47, 46, 169, 210, 78, 130, 81, 98, 250, 64, 203, 150, 186, 177, 195, 231, 15, 17, 227, 182, 19, 52, 110, 242, 16, 144, 227, 94, 24, 123, 69, 143, 168, 240, 242, 23, 252, 118, 13, 13, 98, 93, 254, 98, 117, 139, 219, 77, 85, 77, 193, 224, 190, 253, 205, 125, 140, 123, 104, 154, 177, 122, 104, 133, 135, 214, 220, 164, 206, 240, 89, 187, 253, 255, 180, 200, 219, 179, 126, 19, 203, 166, 155, 222, 67, 243, 25, 6, 64, 221, 216, 35, 125, 186, 34, 122, 183, 170, 130, 175, 69, 247, 148, 180, 100, 194, 169, 193, 115, 96, 151, 44, 156, 227, 91, 138, 212, 100, 151, 189, 7, 171, 178, 99, 172, 30, 122, 240, 208, 30, 225, 236, 189, 239, 32, 216, 90, 58, 64, 40, 210, 215, 178, 137, 59, 224, 240, 181, 114, 184, 225, 44, 234, 74, 210, 73, 68, 236, 38, 35, 210, 140, 203, 59, 141, 13, 166, 241, 226, 36, 121, 118, 27, 14, 216, 173, 155, 55, 217, 31, 114, 183, 144, 40, 218, 184, 205, 182, 117, 156, 130, 50, 34, 134, 139, 142, 219, 232, 185, 50, 19, 65, 43, 169, 229, 17, 203, 190, 121, 130, 92, 31, 109, 18, 205, 176, 33, 11, 36, 129, 158, 35, 11, 130, 224, 44, 19, 226, 248, 149, 6, 180, 141, 189, 173, 95, 179, 180, 106, 118, 56, 10, 55, 109, 185, 108, 207, 87, 112, 23, 49, 248, 18, 183, 207, 114, 96, 53, 131, 28, 59, 159, 50, 1, 81, 50, 66, 141, 186, 121, 133, 144, 124, 108, 29, 17, 204, 121, 178, 50, 224, 39, 210, 30, 95, 223, 85, 244, 114, 136, 44, 32, 241, 3, 143, 76, 22, 212, 107, 140, 152, 221, 236, 68, 88, 137, 53, 116, 141, 159, 230, 120, 106, 203, 232, 183, 156, 181, 43, 215, 249, 66, 2, 178, 220, 7, 179, 19, 70, 154, 126, 233, 123, 97, 91, 244, 169, 226, 64, 207, 231, 86, 141, 213, 103, 105, 117, 125, 193, 110, 154, 153, 116, 148, 187, 204, 135, 77, 174, 39, 52, 222, 75, 179, 195, 80, 172, 204, 158, 105, 81, 172, 15, 149, 53, 9, 90, 110, 100, 179, 248, 146, 56, 154, 179, 66, 184, 67, 235, 23, 24, 228, 154, 225, 254, 30, 85, 167, 198, 177, 63, 22, 123, 71, 19, 112, 167, 150, 4, 237, 200, 97, 57, 116, 252, 80, 24, 94, 199, 59, 245, 102, 220, 239, 251, 211, 14, 190, 233, 1, 49, 173, 129, 97, 199, 44, 94, 231, 220, 144, 102, 116, 106, 240, 206, 23, 28, 176, 228, 235, 12, 59, 2, 161, 10, 71, 192, 153, 218, 225, 87, 178, 182, 189, 253, 97, 242, 18, 182, 207, 162, 125, 52, 71, 90, 109, 252, 200, 84, 109, 111, 217, 196, 137, 154, 181, 230, 205, 27, 34, 246, 205, 200, 102, 91, 78, 142, 236, 177, 192, 173, 111, 173, 246, 73, 90, 40, 177, 98, 207, 163, 116, 174, 164, 157, 211, 22, 95, 141, 217, 198, 127, 230, 90, 255, 92, 27, 25, 177, 222, 220, 71, 34, 166, 158, 214, 205, 194, 214, 53, 146, 106, 38, 22, 197, 117, 227, 47, 100, 203, 204, 203, 104, 200, 85, 38, 73, 32, 187, 190, 156, 104, 230, 26, 119, 139, 242, 23, 95, 112, 214, 128, 203, 130, 30, 17, 105, 23, 55, 233, 252, 43, 114, 101, 59, 205, 168, 254, 202, 55, 225, 170, 139, 17, 102, 250, 34, 210, 127, 107, 53, 137, 139, 33, 240, 231, 88, 118, 73, 123, 205, 75, 83, 146, 118, 166, 19, 16, 217, 217, 198, 37, 201, 81, 85, 197, 163, 147, 43, 102, 70, 207, 242, 180, 15, 167, 217, 0, 227, 151, 175, 215, 215, 77, 152, 214, 168, 4, 119, 24, 200, 220, 66, 22, 121, 111, 100, 37, 168, 223, 105, 222, 57, 21, 198, 221, 38, 7, 112, 124, 74, 106, 157, 201, 114, 150, 60, 99, 55, 175, 33, 104, 74, 25, 39, 83, 250, 234, 146, 79, 243, 249, 212, 241, 132, 94, 101, 17, 239, 212, 57, 83, 183, 11, 11, 215, 144, 51, 193, 122, 145, 14, 127, 69, 152, 241, 160, 153, 21, 219, 241, 248, 192, 247, 245, 120, 6, 157, 51, 201, 114, 178, 173, 211, 219, 164, 25, 133, 82, 190, 135, 104, 199, 40, 48, 186, 144, 89, 191, 79, 216, 202, 119, 170, 90, 219, 118, 45, 228, 166, 170, 1, 199, 42, 64, 44, 54, 71, 34, 172, 54, 238, 62, 242, 166, 189, 120, 249, 244, 117, 38, 14, 137, 198, 114, 14, 219, 112, 247, 236, 220, 164, 239, 150, 195, 173, 88, 177, 177, 26, 47, 100, 235, 107, 233, 191, 183, 211, 86, 200, 47, 37, 13, 233, 167, 103, 37, 93, 234, 231, 231, 137, 239, 111, 65, 21, 149, 182, 233, 213, 8, 111, 100, 28, 235, 184, 56, 188, 173, 65, 99, 168, 15, 64, 19, 114, 250, 42, 184, 8, 194, 51, 137, 147, 120, 136, 125, 152, 239, 133, 11, 99, 54, 45, 224, 97, 55, 9, 72, 245, 194, 42, 29, 56, 53, 91, 175, 77, 240, 97, 172, 123, 193, 240, 204, 231, 52, 185, 226, 135, 124, 177, 51, 215, 91, 34, 77, 197, 94, 95, 137, 95, 49, 197, 133, 150, 251, 151, 98, 0, 140, 196, 169, 29, 24, 156, 69, 12, 194, 248, 29, 208, 121, 250, 74, 18, 60, 99, 227, 213, 198, 229, 187, 142, 77, 20, 45, 135, 203, 222, 90, 79, 58, 98, 94, 192, 229, 68, 223, 6, 73, 219, 147, 10, 232, 68, 167, 13, 120, 9, 28, 121, 110, 119, 219, 172, 34, 112, 63, 250, 116, 115, 243, 189, 182, 189, 34, 19, 90, 159, 211, 251, 45, 137, 199, 204, 13, 250, 16, 247, 251, 194, 60, 224, 145, 59, 138, 194, 47, 62, 58, 82, 47, 3, 146, 24, 88, 129, 255, 13, 86, 157, 168, 37, 65, 224, 128, 7, 107, 199, 28, 98, 58, 146, 163, 54, 97, 99, 195, 184, 88, 164, 244, 214, 188, 224, 150, 99, 73, 139, 172, 114, 237, 102, 105, 135, 206, 200, 181, 126, 20, 153, 186, 96, 207, 113, 74, 161, 9, 230, 228, 124, 224, 158, 162, 164, 71, 78, 217, 29, 126, 61, 151, 172, 168, 238, 124, 241, 242, 190, 55, 73, 226, 162, 125, 203, 140, 231, 141, 124, 14, 214, 7, 26, 14, 178, 146, 195, 107, 102, 82, 254, 249, 79, 4, 42, 62, 246, 153, 54, 86, 201, 39, 17, 99, 209, 212, 231, 96, 103, 168, 229, 122, 97, 118, 223, 92, 229, 123, 79, 23, 30, 6, 209, 142, 45, 212, 27, 94, 198, 143, 226, 242, 147, 112, 157, 181, 243, 57, 88, 238, 109, 118, 221, 252, 102, 141, 248, 152, 253, 255, 1, 71, 136, 183, 26 }; static const unsigned char * const compressedSources = reinterpret_cast(data); uint32_t GetCompressedSize() { - return 22509; + return 22722; } uint32_t GetRawScriptsSize() { - return 134371; + return 139556; } } // selfhosted } // js diff --git a/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp b/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp index d5c9ef1df7a..cd3a648c03f 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp +++ b/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp @@ -387,8 +387,10 @@ NativeObject::getChildPropertyOnDictionary(ExclusiveContext* cx, HandleNativeObj if (!shape) return nullptr; if (childRoot->hasSlot() && childRoot->slot() >= obj->lastProperty()->base()->slotSpan()) { - if (!obj->setSlotSpan(cx, childRoot->slot() + 1)) + if (!obj->setSlotSpan(cx, childRoot->slot() + 1)) { + new (shape) Shape(obj->lastProperty()->base()->unowned(), 0); return nullptr; + } } shape->initDictionaryShape(*childRoot, obj->numFixedSlots(), &obj->shape_); } diff --git a/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp b/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp index ef5501faed8..9564cc6a95a 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp +++ b/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp @@ -384,10 +384,11 @@ static void Discard(uint64_t* buffer, size_t nbytes, const JSStructuredCloneCallbacks* cb, void* cbClosure) { MOZ_ASSERT(nbytes % sizeof(uint64_t) == 0); - if (nbytes < sizeof(uint64_t)) + uint64_t* end = buffer + nbytes / sizeof(uint64_t); + uint64_t* point = buffer; + if (point == end) return; // Empty buffer - uint64_t* point = buffer; uint32_t tag, data; SCInput::getPair(point++, &tag, &data); if (tag != SCTAG_TRANSFER_MAP_HEADER) @@ -399,14 +400,24 @@ Discard(uint64_t* buffer, size_t nbytes, const JSStructuredCloneCallbacks* cb, v // freeTransfer should not GC JS::AutoSuppressGCAnalysis nogc; + if (point == end) + return; + uint64_t numTransferables = LittleEndian::readUint64(point++); while (numTransferables--) { + if (point == end) + return; + uint32_t ownership; SCInput::getPair(point++, &tag, &ownership); MOZ_ASSERT(tag >= SCTAG_TRANSFER_MAP_PENDING_ENTRY); + if (point == end) + return; void* content; SCInput::getPtr(point++, &content); + if (point == end) + return; uint64_t extraData = LittleEndian::readUint64(point++); diff --git a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp index f03c4b50798..5c4471378f1 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp +++ b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp @@ -3966,6 +3966,12 @@ JSScript::maybeSweepTypes(AutoClearTypeInferenceStateOnOOM* oom) for (unsigned i = 0; i < num; i++) typeArray[i].sweep(zone(), *oom); + if (oom->hadOOM()) { + // It's possible we OOM'd while copying freeze constraints, so they + // need to be regenerated. + hasFreezeConstraints_ = false; + } + // Update the recompile indexes in any IonScripts still on the script. if (hasIonScript()) ionScript()->recompileInfoRef().shouldSweep(types); diff --git a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h index ec63d159706..b8797955fa4 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h +++ b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h @@ -571,6 +571,9 @@ class AutoClearTypeInferenceStateOnOOM void setOOM() { oom = true; } + bool hadOOM() const { + return oom; + } }; /* Superclass common to stack and heap type sets. */ diff --git a/src/third_party/mozjs-38/extract/mfbt/Opaque.h b/src/third_party/mozjs-38/extract/mfbt/Opaque.h new file mode 100644 index 00000000000..d7239ee7c23 --- /dev/null +++ b/src/third_party/mozjs-38/extract/mfbt/Opaque.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* An opaque integral type supporting only comparison operators. */ + +#ifndef mozilla_Opaque_h +#define mozilla_Opaque_h + +#include "mozilla/TypeTraits.h" + +namespace mozilla { + +/** + * Opaque is a replacement for integral T in cases where only comparisons + * must be supported, and it's desirable to prevent accidental dependency on + * exact values. + */ +template +class Opaque final +{ + static_assert(mozilla::IsIntegral::value, + "mozilla::Opaque only supports integral types"); + + T mValue; + +public: + Opaque() {} + explicit Opaque(T aValue) : mValue(aValue) {} + + bool operator==(const Opaque& aOther) const { + return mValue == aOther.mValue; + } + + bool operator!=(const Opaque& aOther) const { + return !(*this == aOther); + } +}; + +} // namespace mozilla + +#endif /* mozilla_Opaque_h */ diff --git a/src/third_party/mozjs-38/extract/mfbt/moz.build b/src/third_party/mozjs-38/extract/mfbt/moz.build index 82b7e7e7fbe..51736cf7ed5 100644 --- a/src/third_party/mozjs-38/extract/mfbt/moz.build +++ b/src/third_party/mozjs-38/extract/mfbt/moz.build @@ -53,6 +53,7 @@ EXPORTS.mozilla = [ 'Move.h', 'NullPtr.h', 'NumericLimits.h', + 'Opaque.h', 'Pair.h', 'PodOperations.h', 'Poison.h', diff --git a/src/third_party/mozjs-38/get_sources.sh b/src/third_party/mozjs-38/get_sources.sh index 054eb941902..a589216380b 100755 --- a/src/third_party/mozjs-38/get_sources.sh +++ b/src/third_party/mozjs-38/get_sources.sh @@ -2,8 +2,8 @@ # how we got the last firefox sources -wget "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/38.6.1esr/source/firefox-38.6.1esr.source.tar.bz2" +wget "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/38.8.0esr/source/firefox-38.8.0esr.source.tar.bz2" -tar -jxf firefox-38.6.1esr.source.tar.bz2 +tar -jxf firefox-38.8.0esr.source.tar.bz2 mv mozilla-esr38 mozilla-release diff --git a/src/third_party/mozjs-38/include/js/HashTable.h b/src/third_party/mozjs-38/include/js/HashTable.h index 5fb5012c4a8..36f37d6852a 100644 --- a/src/third_party/mozjs-38/include/js/HashTable.h +++ b/src/third_party/mozjs-38/include/js/HashTable.h @@ -13,6 +13,7 @@ #include "mozilla/Casting.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Move.h" +#include "mozilla/Opaque.h" #include "mozilla/PodOperations.h" #include "mozilla/ReentrancyGuard.h" #include "mozilla/TemplateLib.h" @@ -32,6 +33,8 @@ namespace detail { /*****************************************************************************/ +using Generation = mozilla::Opaque; + // A JS-friendly, STL-like container providing a hash-based map from keys to // values. In particular, HashMap calls constructors and destructors of all // objects added so non-PODs may be used safely. @@ -205,7 +208,9 @@ class HashMap // If |generation()| is the same before and after a HashMap operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -436,7 +441,9 @@ class HashSet // If |generation()| is the same before and after a HashSet operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -771,7 +778,7 @@ class HashTable : private AllocPolicy Entry* entry_; #ifdef JS_DEBUG const HashTable* table_; - uint32_t generation; + Generation generation; #endif protected: @@ -878,7 +885,7 @@ class HashTable : private AllocPolicy #ifdef JS_DEBUG const HashTable* table_; uint64_t mutationCount; - uint32_t generation; + Generation generation; bool validEntry; #endif @@ -1017,8 +1024,8 @@ class HashTable : private AllocPolicy static const size_t CAP_BITS = 24; public: - Entry* table; // entry storage - uint32_t gen; // entry storage generation number + uint64_t gen; // entry storage generation number + Entry* table; // entry storage uint32_t entryCount; // number of entries in table uint32_t removedCount:CAP_BITS; // removed entry sentinels in table uint32_t hashShift:8; // multiplicative hash shift @@ -1102,8 +1109,8 @@ class HashTable : private AllocPolicy public: explicit HashTable(AllocPolicy ap) : AllocPolicy(ap) - , table(nullptr) , gen(0) + , table(nullptr) , entryCount(0) , removedCount(0) , hashShift(sHashBits) @@ -1529,10 +1536,10 @@ class HashTable : private AllocPolicy return JS_BIT(sHashBits - hashShift); } - uint32_t generation() const + Generation generation() const { MOZ_ASSERT(table); - return gen; + return Generation(gen); } size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const diff --git a/src/third_party/mozjs-38/include/jsapi.h b/src/third_party/mozjs-38/include/jsapi.h index 2b279b52092..0d8cd52b12b 100644 --- a/src/third_party/mozjs-38/include/jsapi.h +++ b/src/third_party/mozjs-38/include/jsapi.h @@ -275,10 +275,6 @@ class AutoHashMapRooter : protected AutoGCRooter return map.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return map.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { @@ -390,10 +386,6 @@ class AutoHashSetRooter : protected AutoGCRooter return set.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return set.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { diff --git a/src/third_party/mozjs-38/include/mozilla/Opaque.h b/src/third_party/mozjs-38/include/mozilla/Opaque.h new file mode 100644 index 00000000000..d7239ee7c23 --- /dev/null +++ b/src/third_party/mozjs-38/include/mozilla/Opaque.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* An opaque integral type supporting only comparison operators. */ + +#ifndef mozilla_Opaque_h +#define mozilla_Opaque_h + +#include "mozilla/TypeTraits.h" + +namespace mozilla { + +/** + * Opaque is a replacement for integral T in cases where only comparisons + * must be supported, and it's desirable to prevent accidental dependency on + * exact values. + */ +template +class Opaque final +{ + static_assert(mozilla::IsIntegral::value, + "mozilla::Opaque only supports integral types"); + + T mValue; + +public: + Opaque() {} + explicit Opaque(T aValue) : mValue(aValue) {} + + bool operator==(const Opaque& aOther) const { + return mValue == aOther.mValue; + } + + bool operator!=(const Opaque& aOther) const { + return !(*this == aOther); + } +}; + +} // namespace mozilla + +#endif /* mozilla_Opaque_h */ diff --git a/src/third_party/mozjs-38/mongodb.patch b/src/third_party/mozjs-38/mongodb.patch index 15967bd70d9..bfaae03dc3c 100644 --- a/src/third_party/mozjs-38/mongodb.patch +++ b/src/third_party/mozjs-38/mongodb.patch @@ -24,3 +24,27 @@ index e05665f..042a220 100644 const BYTE* ref; const BYTE* const iend = ip + inputSize; +diff --git a/src/third_party/mozjs-38/include/mozilla/Types.h b/src/third_party/mozjs-38/include/mozilla/Types.h +index 2697ec1..e7e18ab 100644 +--- a/src/third_party/mozjs-38/include/mozilla/Types.h ++++ b/src/third_party/mozjs-38/include/mozilla/Types.h +@@ -37,10 +37,7 @@ + * These macros are designed for use by library interfaces -- not for normal + * methods or data used cross-file. + */ +-// MongoDB Modification - SERVER-20311 +-// Disable all visibility hints since we are statically linking on Windows. +-// This prevents mongod.exe and other binaries from exporting these functions. +-#if 0 //defined(WIN32) ++#if defined(WIN32) + # define MOZ_EXPORT __declspec(dllexport) + #else /* Unix */ + # ifdef HAVE_VISIBILITY_ATTRIBUTE +@@ -52,6 +49,7 @@ + # endif + #endif + ++ + /* + * Whereas implementers use MOZ_EXPORT to declare and define library symbols, + * users use MOZ_IMPORT_API and MOZ_IMPORT_DATA to access them. Most often the diff --git a/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h b/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h index 999c91763ed..bee7d546412 100644 --- a/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h @@ -67,10 +67,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h b/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h index dd3535ced05..c075812b991 100644 --- a/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h +++ b/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h b/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h index bebdfeab17b..68e2f9d5f58 100644 --- a/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h @@ -70,10 +70,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h b/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h index 0b97dc2d48f..4d140eecf2e 100644 --- a/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h +++ b/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h b/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h index a9a75e9958d..297ba39d1f5 100644 --- a/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h @@ -59,10 +59,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so.1.0" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h b/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h index abe437abb7a..d2e2bcb0c91 100644 --- a/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h +++ b/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h b/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h index e46fd83c737..e027c288af3 100644 --- a/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h @@ -32,10 +32,10 @@ #define MALLOC_H #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dll" #define MOZ_MEMORY 1 diff --git a/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h b/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h index a78d3daf9fe..0020a59223a 100644 --- a/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h +++ b/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h index a64b834f7c0..2a108fe363a 100644 --- a/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h @@ -63,10 +63,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h index 544278fee5e..290b4a1fd45 100644 --- a/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h index f5069d6d644..6451dc43eff 100644 --- a/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h @@ -71,10 +71,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h index dd3535ced05..c075812b991 100644 --- a/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h index 890078476f8..1180a068dc2 100644 --- a/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h @@ -62,10 +62,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so.1.0" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h index e6cf8c4a69b..59b145407e1 100644 --- a/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h index c66542b7bf7..ae99c33ecca 100644 --- a/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h @@ -58,10 +58,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dylib" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h index 544278fee5e..290b4a1fd45 100644 --- a/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h index 58af7c43c83..d0357db4012 100644 --- a/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h @@ -62,10 +62,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h index 2008beb3d54..bba3ba9ab55 100644 --- a/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h index 0e856775ed6..3daabba2fd7 100644 --- a/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h @@ -33,10 +33,10 @@ #define MALLOC_H #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dll" #define MOZ_MEMORY 1 diff --git a/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h index df5d20e3fe7..db0de4128f1 100644 --- a/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index bef47daabf9..4611f2aa98c 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -16,11 +16,12 @@ *.obj *.pdb *.pyc +.dirstamp .sconf_temp .sconsign.dblite -.dirstamp /Makefile.am /aclocal.m4 +/autom4te.cache/ /build_posix/aclocal/libtool.m4 /build_posix/aclocal/ltoptions.m4 /build_posix/aclocal/ltsugar.m4 @@ -32,7 +33,7 @@ /config.status /configure /configure.ac -/libtool/ +/libtool /stamp-h1 /wiredtiger.h /wiredtiger.pc @@ -63,9 +64,9 @@ WT_TEST/ # Python /lang/python/_wiredtiger.so +/lang/python/wiredtiger.py /lang/python/wiredtiger/__init__.py /lang/python/wiredtiger/_wiredtiger.pyd -/lang/python/wiredtiger.py /lang/python/wiredtiger_wrap.c _wiredtiger.pyd @@ -89,23 +90,28 @@ _wiredtiger.pyd **/examples/c/ex_pack **/examples/c/ex_process **/examples/c/ex_schema -**/examples/c/ex_scope **/examples/c/ex_stat **/examples/c/ex_sync **/examples/c/ex_thread **/test/bloom/t **/test/checkpoint/t -**/test/csuite/test_wt1965_col_efficiency -**/test/csuite/test_wt2246_col_append -**/test/csuite/test_wt2323_join_visibility -**/test/csuite/test_wt2403_lsm_workload -**/test/csuite/test_wt2447_join_main_table -**/test/csuite/test_wt2535_insert_race -**/test/csuite/test_wt2592_join_schema -**/test/csuite/test_wt2695_checksum -**/test/csuite/test_wt2719_reconfig -**/test/csuite/test_wt2834_join_bloom_fix -**/test/csuite/test_wt2853_perf +**/test_scope +**/test_wt1965_col_efficiency +**/test_wt2246_col_append +**/test_wt2323_join_visibility +**/test_wt2403_lsm_workload +**/test_wt2447_join_main_table +**/test_wt2535_insert_race +**/test_wt2592_join_schema +**/test_wt2695_checksum +**/test_wt2719_reconfig +**/test_wt2834_join_bloom_fix +**/test_wt2853_perf +**/test_wt2909_checkpoint_integrity +**/test_wt2999_join_extractor +**/test_wt3120_filesys +**/test_wt3135_search_near_collator +**/test_wt3184_dup_index_collator **/test/cursor_order/cursor_order **/test/fops/t **/test/format/s_dumpcmp @@ -120,4 +126,5 @@ _wiredtiger.pyd **/test/recovery/random-abort **/test/recovery/truncated-log **/test/salvage/t +**/test/syscall/test_wt2336_base **/test/thread/t diff --git a/src/third_party/wiredtiger/NEWS b/src/third_party/wiredtiger/NEWS index af8b15488cc..268949b119f 100644 --- a/src/third_party/wiredtiger/NEWS +++ b/src/third_party/wiredtiger/NEWS @@ -1,3 +1,418 @@ +WiredTiger release 2.9.1, 2016-12-22 +------------------------------------ + +New features and API changes; refer to the API documentation for full details: + +* SERVER-26545 Remove fixed-size limitation on WiredTiger hazard pointers. See the upgrading documentation for details +* WT-283 Add a new WT_SESSION::alter method that can be used to reconfigure table metadata +* WT-2670 Change the default file system access pattern advice for data files from random to no advice. Add access_pattern_hint configuration option for WT_SESSION::create API that can be used to advise the file system of expected access semantics. See the upgrading documentation for details. +* WT-3034 Add support for including updates when reading from named snapshots + +Significant changes and bug fixes: + +* WT-2960 Reduce likelihood of using the lookaside file, especially when inserting multi-megabyte values +* WT-3056 Allow projected table and join cursors to use primary keys +* WT-3070 Fix a bug in search_near on indexes + +Other noteworthy changes since the previous release: + +* WT-2336 Add a test validating schema operations via file system call monitoring +* WT-2402 Pad structures to avoid cache line sharing +* WT-2771 Add a statistic to track per-btree dirty cache usage +* WT-2833 Add projections to wt dump utility +* WT-2969 Possible snapshot corruption during compaction +* WT-3014 Add GCC/clang support for ELF symbol visibility +* WT-3021 Fixes for java log example, raw mode in java, and raw mode in log cursors +* WT-3025 Fix error path in log_force_sync +* WT-3028 Don't check for blocked eviction with in-memory workloads +* WT-3030 Fix a race between scans and splits reading the index hint +* WT-3037 Clean up some log slot comments +* WT-3048 WiredTiger maximum size warning uses the wrong format +* WT-3051 Remove external __wt_hex symbol +* WT-3052 Improve search if index hint is wrong +* WT-3053 Make Python use internal memory allocation again +* WT-3054 Make a PackOutputStream constructor that is compatible with the previous interface. +* WT-3055 When an AsyncOp is created, cache the whether the cursor is "raw" +* WT-3057 WiredTiger hazard pointers should use the WT_REF, not the WT_PAGE +* WT-3061 Syscall testing should support pwrite64 on Linux +* WT-3064 Minor tree cleanups: .gitignore, NEWS misspelling +* WT-3066 Minor code cleanups +* WT-3068 Copy artifacts of test runs in wtperf_run script +* WT-3068 Have Jenkins include specific files for copy rather than exclude +* WT-3069 Fix LevelDB APIs build failures +* WT-3071 Fixed sign-conversion compiler errors in Java and Python SWIG code +* WT-3075 Document and enforce that WiredTiger now depends on Python 2.7 +* WT-3078 Test reconfiguration hang in the statlog server +* WT-3080 Python test suite: add elapsed time for tests +* WT-3082 Python test suite: shorten default run to avoid timeouts +* WT-3084 Fix Coverity resource leak complaint +* WT-3091 Add stats to test_perf001 test, so we can investigate what happened when it failed + +WiredTiger release 2.9.0, 2016-09-06 +------------------------------------ + +New features and API changes; refer to the API documentation for full details: + +* WT-2360 Add nesting and disjunction functionality to cursor joins +* WT-2552 Add a public API to allow custom filesystem implementations +* WT-2738 Remove the ability to change the default checkpoint name. See the upgrading documentation for details. +* WT-2711 Change the statistics log configuration options. Restrict where statistics log files can be created to remove security vulnerability. See upgrading documentation for details. +* WT-2764 Enhance the checkpoint implementation, which led to a change in default eviction settings. See the upgrading documentation for details. +* WT-2880 Add Zstandard compression support +* See the upgrading documentation for information about renamed and removed statistics. + +Significant changes and bug fixes: + +* WT-2026 Fix a bug where in-memory pages were allowed to grow too large +* WT-2343 Ensure we cannot rename/drop when a backup cursor is open +* WT-2520 Ensure WT_SESSION::verify does not alter tables +* WT-2576 Ensure variable-length column-store cannot out-of-order return +* WT-2664 Change eviction so any eviction thread can find candidates +* WT-2702 Resolve an issue where under high thread load, WiredTiger exceeds cache size +* WT-2731 Resolve an issue where raw compression can create pages that are larger than expected +* WT-2737 Allow reconciliation to scrub dirty pages rather than evicting them +* WT-2757 Fix a bug in column stores where it was not possible to retrieve the new record number after an append +* WT-2816 Improve eviction performance + +Issues fixed in MongoDB: + +* SERVER-23504 Resolve a resource leak +* SERVER-23588 Stop using _open_osfhandle on Windows +* SERVER-23659 Improve log error message at startup +* SERVER-23661 Fix an issue where $sample takes disproportionately long time on newly created collection +* SERVER-24306 Fix stall in log_flush switching to new files +* SERVER-24580 Add more eviction stats to track efficiency +* SERVER-24580 Enhance eviction when application threads are contributing +* SERVER-24971 Don't cache buffers after application eviction +* SERVER-25843 Remove a redundant test +* SERVER-25845 Resolve an explicit null dereferenced +* SERVER-25846 Resolve a dereference after null with a check +* SERVER-26753 Don't spin on a read-lock in a tight loop + +Other noteworthy changes since the previous release: + +* WT-2103 Add incremental backup testing to test/format +* WT-2330 Ensure In-memory configurations do not create on-disk collection files +* WT-2450 Resolve an issue where salvage releases pages, then explicitly evicts them. Which can race with the eviction server +* WT-2453 Add multiple eviction queues +* WT-2504 Remove READONLY conditional for base config +* WT-2505 Resolve clang analyzer warnings +* WT-2508 Ensure test programs should remove test directories on the "clean" target +* WT-2512 Change how wtperf throttle values per thread are calculated +* WT-2513 Fix a type conversion warning +* WT-2517 Resolve incorrect uses of setvbuf on Windows +* WT-2518 Optimization of LSM checkpoint handle acquisition +* WT-2522 Resolve incorrect format code in message +* WT-2525 Miscellaneous cleanups of In-memory configurations +* WT-2526 Prevent mixing and matching readonly file-handles and read/write data-handles +* WT-2528 Fix a style error in WiredTiger build +* WT-2529 Be less aggressive asserting in readonly connections +* WT-2531 Resolve a case where in-memory tables are wasting space in truncation +* WT-2532 Change the WT_STREAM_APPEND and WT_STREAM_LINE_BUFFER flags as they overlap +* WT-2533 Don't let in-memory tables return a zero size +* WT-2534 Use atomic add when allocating transaction IDs +* WT-2535 Extend test/format to test for transactions reading their writes +* WT-2537 Check for an old and new uninitialized LSN for recovery +* WT-2539 Make streams a separate handle type from files +* WT-2540 Separate stream and file handle methods +* WT-2542 Resolve fixed-length column store reconciliation overwriting original values +* WT-2544 Fix eviction stats when clear is used +* WT-2546 Change evict server to be smarter when it decides whether or not to evict pages +* WT-2547 Add eviction testing configurations for 1 thread +* WT-2548 Cap the amount of data handed to raw compression +* WT-2550 Add support for raw mode cursors in java +* WT-2553 Add some In-memory documentation and stress testing +* WT-2556 Fix a typo in the Java example code +* WT-2557 Ensure that we truncate logs before closing backup cursor +* WT-2558 Reorganize WT_PAGE structure +* WT-2559 Resolve a potential segfault in Windows logging code +* WT-2560 Add a rwlock to protect transaction state, don't spin on this lock +* WT-2562 Add sleep loop to adapt for slow machines like PPC +* WT-2565 Further limit the amount of data passed to raw compression +* WT-2566 Add explicit memory barriers to Lock/unlock operations +* WT-2567 Add a check to see if logging is enabled for truncate rather than assuming it is +* WT-2568 Add a backward compatible constructor for PackInputStream +* WT-2569 Ensure __win_handle_read is followed by a call to GetLastError() +* WT-2570 Minor lint and cleanups to code +* WT-2571 Join code clean up +* WT-2572 Ensure test/format doesn't select an incompatible set of options for in-memory options configured +* WT-2573 Resolve a potential case where a double free can occur +* WT-2574 Ensure test/format frees all allocated configure memory +* WT-2576 Ensure variable-length column-store cannot out-of-order return +* WT-2577 Resolve a potential double-free following an in-memory split +* WT-2579 Change so diagnostic only code can use file streams directly +* WT-2580 Fix potential SWIG naming conflict in Java +* WT-2581 Resolve a case where we fail to track saved updates during a page split +* WT-2582 Fix a case where eviction can return WT_RESTART when running in-memory +* WT-2584 Change to not use periods at the end of error messages +* WT-2586 Fix examples/ex_config +* WT-2589 Ensure we clear the cursor statistics for LAS when 'clear' is set +* WT-2592 Fix joins for the non-recno, non-raw case +* WT-2593 Free the dirlist at the end of the function +* WT-2595 Fix compiler warning in packing test +* WT-2598 Change so that in-memory has faster lookup on file names +* WT-2599 Split out the checksum code from the support directory +* WT-2600 Clean up test program includes +* WT-2602 In LSM, use the chunk size to set maximum page size in memory +* WT-2605 Add C tests used in testing joins +* WT-2609 Resolve incorrect "skips API_END call" error +* WT-2610 Reduce hazard pointer array size +* WT-2611 Modify wtperf to handle escaped quotes +* WT-2612 Ensure the dist/s_prototypes script does not create a debugging file +* WT-2613 Add WT_UNUSED to a variable to fix Windows compilation +* WT-2615 Enabling checkpoints in test/format leads to reduced concurrency +* WT-2616 Fix a deadlock with in-memory size lookups +* WT-2617 Don't use u_int in the example code. It's not Windows native +* WT-2621 Fix missing parentheses on call to __wt_errno +* WT-2622 Clear old position for all random lookups +* WT-2624 Fix example program build with MSVC 2013 +* WT-2626 Fix MSVC 2015 snprintf redefinition +* WT-2627 Resolve Coverity complaints +* WT-2628 Ensure reconciliation doesn't return without unlocking the page lock +* WT-2629 Ensure stacks are not executable in assembly source +* WT-2630 Rename pluggable filesystem methods to avoid reserved names +* WT-2631 Ensure nullptr is not passed for parameters marked with attribute non-null +* WT-2632 Tolerate EBUSY when a checkpoint cursor is open +* WT-2637 Update file-extension documentation to cover not-supported cases +* WT-2638 Add extension to the Windows file system implementation +* WT-2644 Fix 'wt load -r' (rename) with LSM +* WT-2645 Move the complexity of dump from the dump utility to the metadata cursor +* WT-2646 Add checkpoint_wait configuration option to drop +* WT-2648 Change cache-line alignment for new ports +* WT-2651 Fix a Coverity found resource leak +* WT-2652 Remove unnecessary wt_ftruncate call +* WT-2653 Add display of device configuration to custom file-system example +* WT-2656 Fix GCC 4.7 compiler warnings +* WT-2658 Ensure we only include PPC-specific files in PPC builds +* WT-2659 Assorted lint and cleanup to csuite tests +* WT-2660 Fix a hang between eviction and connection close +* WT-2662 For spell check, strip out double quote literals, they confuse aspell +* WT-2664 Add ability for eviction workers to populate eviction queues +* WT-2665 Limit allocator fragmentation from the WiredTiger cache +* WT-2667 Add fops to Evergreen testing for Windows and Linux +* WT-2668 Create join statistics that are useful and are easy to understand +* WT-2671 Dump more information about the file layout in verify debug mode +* WT-2672 Handle system calls that don't set errno +* WT-2673 Stop automatically increasing memory page max +* WT-2674 Simplify metadata file check +* WT-2676 Don't use key size in column store in-memory splits +* WT-2677 Fix JSON output so only printable ASCII is produced +* WT-2678 Fix cases where metadata implies that an empty value is true +* WT-2682 Add option to configure WiredTiger with strict compiler flags +* WT-2683 Allow in memory storage engine to report zero disk usage +* WT-2685 Fix clear walk hazard pointer failure +* WT-2686 Report an error any time we fail to scan the log +* WT-2687 Provide ability for test suite to verify the exit status of the wt utility +* WT-2688 Improve build error messages when SWIG is unavailable +* WT-2689 Fix heap-use-after-free on cursor error path +* WT-2691 Use wrappers for ctype functions to avoid sign extension errors +* WT-2692 Fix race in file system example +* WT-2693 Ensure open_cursor error paths have consistent error handling +* WT-2695 Integrate s390x accelerated crc32c support +* WT-2696 Wait if we find an unbuffered flag without the size set yet +* WT-2698 Change flag into atomically manipulated field to avoid deadlock +* WT-2702 Block operations when the cache is 100% full +* WT-2704 Fix bug in diagnostic code tracking duration of stuck cache +* WT-2706 Fix lost log writes when switching files +* WT-2707 Add stricter checks in dist/s_label, and add some WT_ERR calls +* WT-2708 Fix a split child-update race with reconciliation/eviction +* WT-2709 Resolve a connection reconfigure segfault in __wt_conn_cache_pool_destroy +* WT-2710 WT_FILE_HANDLE_INMEM no longer needs an off field +* WT-2713 Document WT_PANIC so pluggable filesystems can panic +* WT-2715 Resolve random abort test failure with partial write +* WT-2719 Allow make check without verbose configured +* WT-2719 Fix memory leak in reconfig test +* WT-2720 Change the return codes in run.py to error on failures +* WT-2722 Escape regular expression meta-character so egrep works on all systems +* WT-2724 Pass in session address, not pointer to it +* WT-2728 Only verify the log file header during recovery +* WT-2729 Focus eviction on the largest trees in cache +* WT-2730 Btree can incorrectly match key slots on new pages +* WT-2731 Finer adjustment for page size with raw compression +* WT-2732 Coverity analysis defect 99665, Remove redundant test +* WT-2734 Improve documentation of eviction configuration settings +* WT-2739 pluggable file systems documentation cleanups +* WT-2743 Fixup new I/O thread count statistics +* WT-2744 Detect and ignore partial line +* WT-2746 Add a new statistic tracking I/O for checkpoints +* WT-2751 column-store statistics incorrectly calculates the number of entries +* WT-2752 Fix errors in wtperf config +* WT-2755 Fix for cases where tools treats size_t as 4B type +* WT-2756 Upgrade the autoconf archive package to check for swig 3.0 +* WT-2757 Column tables behave differently when column names are provided +* WT-2759 Releasing the hot-backup lock doesn't require the schema lock +* WT-2760 Fix a bug in backup related to directory sync. Change the filesystem API to make durable the default +* WT-2762 Handle 'Y' as an options from wtperf monitor file +* WT-2763 Change test ID's in test_intpack to fix OS X test errors +* WT-2764 Fix a bug calculating the dirty percentage of cache +* WT-2765 Ensure indices are shown in the dump output +* WT-2766 Don't sweep LAS cache when aren't making progress in eviction +* WT-2767 In test/suite/run.py, add -s N option to run an individual scenario +* WT-2769 Update documentation to reflect correct limits of memory_page_max +* WT-2770 Add statistics tracking schema operations +* WT-2772 Adjust log.wtperf config. Remove unneeded config entries +* WT-2773 Ensure that search_near in an index finds exact matches +* WT-2774 Minor cleanups/improvements +* WT-2778 Enhance output formatting when running Python test suite +* WT-2779 Fix large pages getting generated with raw compression +* WT-2781 Don't take checkpoint lock if checkpoint_wait=0 for bulk cursor open +* WT-2782 Minor text changes for file system functions +* WT-2783 Clean up wtperf configuration object management +* WT-2785 Scrub dirty pages rather than evicting them single-page reconciliation +* WT-2787 Include src/include/wiredtiger_ext.h is problematic +* WT-2788 Do not touch memory already freed during a close API call +* WT-2791 Have evergreen upload artifacts for each build. Rename current artifacts to "Binaries" +* WT-2793 Enhance statistics related to overflow values +* WT-2795 Update documentation about read-only mode +* WT-2796 Fix a memory leak when using the lookaside table +* WT-2798 Fix data consistency bug with table creates during a checkpoint +* WT-2800 Fix incorrect error message +* WT-2801 Prevent eviction of metadata updates by a running checkpoint +* WT-2802 Copy values during commit before releasing snapshot +* WT-2803 Add VERBOSE=1 to all make check jobs +* WT-2804 Don't read values in a tree without a snapshot +* WT-2805 Avoid infinite recursion on error stream failure +* WT-2806 Fix an off-by-one allocation in wtperf +* WT-2807 Change the memory allocator for wtperf performance tests +* WT-2811 The checkpoint session should not ignore it's own transaction ID +* WT-2812 Verify cache_size before dividing to avoid division by 0 +* WT-2813 Configure eviction dirty settings explicitly for LSM in test/format +* WT-2814 Add new single op truncate mode to wtperf +* WT-2816 General improvements to WiredTiger eviction performance +* WT-2817 Add wtperf conf to stress test checkpoints with updates +* WT-2818 Change the page visibility check when queuing pages for eviction +* WT-2820 Add gcc warn_unused_result attribute +* WT-2822 Simplify error handling by using a panic mutex in functions that cannot fail +* WT-2823 Support file handles without a truncate method +* WT-2824 Fix double concatenation the config strings +* WT-2826 Fix a clang38 false positive on uninitialized variable +* WT-2827 Set a reasonable minimum for log_size +* WT-2828 Change wtperf long tests to better match MongoDB workloads +* WT-2829 Fix missing define for BerkeleyDB +* WT-2831 Skip checkpointing if there have been no modifications +* WT-2832 Python test uses hard-coded temporary directory +* WT-2834 Fix sign-change warning with auto conversion from uint to int +* WT-2834 Shared bloom filters allow only a partial shortcut +* WT-2835 Stop WT_CONNECTION.leak-memory skipping memory map and cache cleanup +* WT-2838 Don't free session handles on close if leak memory is configured +* WT-2839 Remove cases ignoring return value of function +* WT-2840 Fix garbage values found with clang analysis +* WT-2841 Initialize verbose checkpoint timer at start of checkpoint +* WT-2842 Add explicit include in wtperf to resolve build warning +* WT-2843 If there is no truncate available, manually zero the log file +* WT-2846 Ensure that all resources are released when destroying a thread group +* WT-2846 Fixes for new thread group code +* WT-2847 Merge fair locks into read/write locks +* WT-2850 Fix clang 4.1 attribute warnings +* WT-2853 Don't force eviction if multiple cursors are pinning the page +* WT-2857 POSIX ftruncate calls should be use HAVE_FTRUNCATE define +* WT-2858 Rename wtperf's CONFIG structure +* WT-2859 Restructure statistics gathering macros +* WT-2863 Support UTF-8 paths on Windows +* WT-2864 Update reconfigure test to detect hangs +* WT-2865 Fix a bug where the eviction server could panic after a WT_NOTFOUND +* WT-2866 Don't set eviction stuck unless the cache is full +* WT-2867 Review and fix barrier usage in __lsm_tree_close +* WT-2868 Add sample_interval to checkpoint-stress.wtperf +* WT-2869 Fix a performance regression on secondaries +* WT-2870 Rename wtperf jobs for consistency +* WT-2871 Make verbose formats and argument types match +* WT-2872 Ensure tests with tiny caches don't get stuck due to the dirty trigger +* WT-2873 Refactor CRC32 code +* WT-2874 Change test_compact01 to avoid eviction +* WT-2875 Add environment variable to disable long csuite tests +* WT-2876 Add an oplog-like ability to wtperf utility +* WT-2878 Fix an issue where verbose changes affected performance +* WT-2881 Add -Wpedantic to clang compiler warning flags +* WT-2883 Remove potentially recursive call for verbose handleops +* WT-2885 Lint __wt_checkpoint_signal function +* WT-2886 Ignore dirty eviction settings when in_memory is true +* WT-2888 Switch functions to return void where possible +* WT-2892 Fix case where hot backup can race with block truncate +* WT-2894 Add wtperf stress workload that tries to induce negative scaling +* WT-2895 Reduce the runtime of make check testing with disable long +* WT-2896 Fix a resource leak +* WT-2897 Fix an issue where checkpoints can become corrupted on failure +* WT-2900 Add ARM8 build support to WiredTiger and fix ARM CRC assembler tags +* WT-2901 Make checkpoint scrubbing configurable +* WT-2902 Track per-tree the efficiency of eviction walks +* WT-2903 Change the eviction_scrub_target default to 15% +* WT-2904 Fix a bug where the reported checkpoint size could be many times data size +* WT-2904 Revert overly strong assertion +* WT-2905 Remove dead code +* WT-2907 Fixed java concurrent close test to have both insert threads and scan threads +* WT-2908 Add a dry-run option to python suite +* WT-2910 When running in-memory, only evict dirty pages +* WT-2911 Add support for gcc6 +* WT-2912 Make --enable-strict run on the zseries test box +* WT-2913 Re-sort eviction queues if we find no new candidates +* WT-2914 In test/csuite program, added explicit call to wiredtiger_open to satisfy some linkers +* WT-2916 Fix and simplify s_whitespace +* WT-2917 Split wtperf's configuration into per-database and per-run parts +* WT-2918 The dist scripts create C files s_whitespace complains about +* WT-2919 Don't mask error returns from style checking scripts +* WT-2920 New eviction statistics +* WT-2921 Reset the count when the last hazard pointer is cleared +* WT-2923 Fix heap-use-after-free on address in compaction +* WT-2924 Ensure we are doing eviction when threads are waiting for it +* WT-2925 Change where the WT_THREAD_PANIC_FAIL flag lives +* WT-2926 WT_CONNECTION.reconfigure can attempt unlock of not-locked lock +* WT-2928 Don't give up when the urgent queue is half empty +* WT-2928 Eviction failing to switch queues can lead to starvation +* WT-2931 Enforce limits on dirty data for in-memory +* WT-2932 Add a configuration option allowing tables to ignore cache limits +* WT-2933 Fix a race between named snapshots and checkpoints +* WT-2937 Only do an eviction walk after a page has been taken from the current queue +* WT-2938 Change assembly file extensions from .S to .sx +* WT-2941 Improve test/format to use faster key-generation functions +* WT-2942 Verbose messages should not have newlines +* WT-2945 Occasional hang running reconfigure fuzz test +* WT-2946 Fix dist/s_docs being incompatible with OS X Xcode installation +* WT-2947 Replace test suite populate functions with *DataSet classes +* WT-2948 Simplify error handling by making __wt_epoch return never fail +* WT-2949 Add option to skip closing the connection after test runs +* WT-2950 Free all reconciliation memory between calls to eviction +* WT-2953 Only test for checkpoint/LAS collision if LAS was used +* WT-2954 Evict pages that exceed memory_page_max even if the transaction ID is stuck +* WT-2955 Add statistics tracking the amount of time threads spend waiting for high level locks +* WT-2959 Ensure WT_SESSION_IMPL is never used before it's initialized +* WT-2961 Add a version drop-down to the web version of the docs +* WT-2962 Allow configuration of builtin extensions +* WT-2963 Fix a race bounding eviction in small caches +* WT-2964 Alter evict walk to not fill all its slots with internal pages when running in aggressive mode +* WT-2965 Remove sleep and retry loop from __evict_exclusive +* WT-2968 Don't open file handles on backup +* WT-2971 Add details on raw-compression into WT documentation +* WT-2975 Fix a leak of statistics data when reopening handles +* WT-2976 Add a statistic tracking how long app threads spend reading and writing +* WT-2977 Remove stash and check code from LSM test +* WT-2984 Keep sufficient history in the metadata for queries +* WT-2985 Fix a race during checkpoint that can cause a core dump +* WT-2987 Fix a bug where opening a cursor on an incomplete table drops core +* WT-2988 Fix __wt_epoch potentially returning garbage values +* WT-2991 Fix Coverity REVERSE_INULL +* WT-2998 Add error messages to error returns that might be confusing +* WT-2999 Added contributed test case that demonstrated the leak +* WT-3000 Wait for previous writes on first write to new log file +* WT-3001 Fix WT_EXTENSION_API references that are named inconsistently +* WT-3002 Allow applications to exempt threads from eviction +* WT-3003 Don't generate log record and op types +* WT-3003 Fix doxygen comment blocks +* WT-3004 Lint, declare functions that don't return a value as void +* WT-3005 Add top-level .gitignore file +* WT-3008 Move wtperf stress jobs to new stress runner folder +* WT-3009 Remove the eviction_dirty_target from test/format runs with less than 20MB cache +* WT-3011 Fix __wt_curjoin_open() saving the wrong URI in the cursor +* WT-3012 Check a btree is LSM primary before switching +* WT-3012 Don't track the LSM Primary as part of dirty bytes in cache +* WT-3015 Change when we will evict internal pages +* WT-3016 Change how file size is determined during compact tests to allow +* WT-3017 Don't set NULL hazard pointers +* WT-3020 Always make LSM chunks evictable when they are switched out +* WT-3022 Change lsm_tree flags to fields to prevent race conditions +* WT-3023 Don't treat splits as eviction making progress +* WT-3024 Fix a hang on close caused by leaving a transaction ID pinned + WiredTiger release 2.8.0, 2015-03-24 ------------------------------------ diff --git a/src/third_party/wiredtiger/README b/src/third_party/wiredtiger/README index 32549d099d5..f7edae2835d 100644 --- a/src/third_party/wiredtiger/README +++ b/src/third_party/wiredtiger/README @@ -1,6 +1,6 @@ -WiredTiger 2.8.1: (March 24, 2016) +WiredTiger 2.9.2: (December 23, 2016) -This is version 2.8.1 of WiredTiger. +This is version 2.9.2 of WiredTiger. WiredTiger release packages and documentation can be found at: @@ -8,7 +8,7 @@ WiredTiger release packages and documentation can be found at: The documentation for this specific release can be found at: - http://source.wiredtiger.com/2.8.1/index.html + http://source.wiredtiger.com/2.9.2/index.html The WiredTiger source code can be found at: diff --git a/src/third_party/wiredtiger/RELEASE_INFO b/src/third_party/wiredtiger/RELEASE_INFO index d29c29b554c..b7145aa2cb3 100644 --- a/src/third_party/wiredtiger/RELEASE_INFO +++ b/src/third_party/wiredtiger/RELEASE_INFO @@ -1,6 +1,6 @@ WIREDTIGER_VERSION_MAJOR=2 -WIREDTIGER_VERSION_MINOR=8 -WIREDTIGER_VERSION_PATCH=1 +WIREDTIGER_VERSION_MINOR=9 +WIREDTIGER_VERSION_PATCH=2 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 0ccdf59babc..b397f662be7 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -294,6 +294,7 @@ env.Depends(wtdll, [filelistfile, version_file]) Default(wtlib, wtdll) wtbin = env.Program("wt", [ + "src/utilities/util_alter.c", "src/utilities/util_backup.c", "src/utilities/util_cpyright.c", "src/utilities/util_compact.c", @@ -312,6 +313,7 @@ wtbin = env.Program("wt", [ "src/utilities/util_rename.c", "src/utilities/util_salvage.c", "src/utilities/util_stat.c", + "src/utilities/util_truncate.c", "src/utilities/util_upgrade.c", "src/utilities/util_verbose.c", "src/utilities/util_verify.c", @@ -374,7 +376,6 @@ examples = [ "ex_pack", "ex_process", "ex_schema", - "ex_scope", "ex_stat", "ex_thread", ] diff --git a/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc b/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc index 0e666ac1dc0..97b8cd6dbdf 100644 --- a/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc +++ b/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc @@ -19,16 +19,19 @@ // under the License. // // ------------------------------------------------------------------- +#define __STDC_FORMAT_MACROS +#include +#include #include +#include #include + #include #include #include #include #include -#include -#include #ifndef STORAGE_LEVELDB_INCLUDE_PERF_COUNT_H_ #include "perf_count.h" @@ -36,8 +39,6 @@ #include "util/coding.h" -#define __STDC_FORMAT_MACROS -#include #ifdef OS_SOLARIS # include diff --git a/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc b/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc index 95c82289e18..c12aedc7bad 100644 --- a/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc +++ b/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc @@ -27,6 +27,7 @@ #include "leveldb_wt.h" #include +#include #include #include #include @@ -143,7 +144,14 @@ class ReplayIteratorImpl : public ReplayIterator { } private: - void SeekTo(WT_LSN *lsn); + /* + * A log sequence number, representing a position in the transaction log. + */ + typedef struct { + uint32_t file; /* Log file number */ + unsigned long long offset; /* Log file offset */ + } LSN; + void SeekTo(LSN *lsn); // No copying allowed ReplayIteratorImpl(const ReplayIterator&) { } void operator=(const ReplayIterator&) { } @@ -151,7 +159,7 @@ class ReplayIteratorImpl : public ReplayIterator { Status status_; WT_CURSOR *cursor_; WT_ITEM key_, value_; - WT_LSN lsn_; + LSN lsn_; bool valid_; uint64_t txnid; uint32_t fileid, opcount, optype, rectype; @@ -201,7 +209,7 @@ ReplayIteratorImpl::Next() { void ReplayIteratorImpl::SeekToLast() { int ret = 0; - WT_LSN last_lsn; + LSN last_lsn; last_lsn.file = 0; if (cursor_ != NULL) { @@ -235,7 +243,7 @@ ReplayIteratorImpl::SeekToLast() { void ReplayIteratorImpl::SeekTo(const std::string& timestamp) { - WT_LSN target_lsn; + LSN target_lsn; int ret = 0; if (timestamp == "all") { @@ -260,7 +268,7 @@ ReplayIteratorImpl::SeekTo(const std::string& timestamp) { // Set the cursor on the first modification record at or after the // given LSN. void -ReplayIteratorImpl::SeekTo(WT_LSN *target_lsn) { +ReplayIteratorImpl::SeekTo(LSN *target_lsn) { int ret = 0; valid_ = false; diff --git a/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h b/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h index dc185183034..351eb9f3dda 100644 --- a/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h +++ b/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h @@ -43,6 +43,7 @@ #include "basho/perf_count.h" #endif +#include #include "wiredtiger.h" #define WT_URI "table:data" diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 5b14a4cdf68..e4eee66e4cb 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -215,6 +215,7 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) return (EINVAL); } workp = &wtperf->workload[wtperf->workload_cnt++]; + workp->table_index = INT32_MAX; while ((ret = scan->next(scan, &k, &v)) == 0) { if (STRING_MATCH("count", k.str, k.len)) { @@ -233,12 +234,28 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) goto err; continue; } + if (STRING_MATCH("pause", k.str, k.len)) { + if ((workp->pause = v.val) < 0) + goto err; + continue; + } if (STRING_MATCH("read", k.str, k.len) || STRING_MATCH("reads", k.str, k.len)) { if ((workp->read = v.val) < 0) goto err; continue; } + if (STRING_MATCH("read_range", k.str, k.len)) { + if ((workp->read_range = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("table", k.str, k.len)) { + if (v.val <= 0) + goto err; + workp->table_index = (int32_t)v.val - 1; + continue; + } if (STRING_MATCH("throttle", k.str, k.len)) { workp->throttle = (uint64_t)v.val; continue; @@ -421,14 +438,13 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) return (EINVAL); } strp = (char **)valueloc; - newlen = v->len + 1; if (*strp == NULL) begin = newstr = dstrdup(v->str); else { - newlen += strlen(*strp) + 1; - newstr = dcalloc(newlen, sizeof(char)); - snprintf(newstr, newlen, - "%s,%*s", *strp, (int)v->len, v->str); + newlen = strlen(*strp) + v->len + strlen(",") + 1; + newstr = dmalloc(newlen); + testutil_check(__wt_snprintf(newstr, newlen, + "%s,%.*s", *strp, (int)v->len, v->str)); /* Free the old value now we've copied it. */ free(*strp); begin = &newstr[(newlen - 1) - v->len]; @@ -622,17 +638,9 @@ config_opt_str(WTPERF *wtperf, const char *optstr) return (ret); } - /* - * Append the current line to our copy of the config. The config is - * stored in the order it is processed, so added options will be after - * any parsed from the original config. We allocate len + 1 to allow for - * a null byte to be added. - */ - config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); - config_line->string = dstrdup(optstr); - TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); - while (ret == 0) { + size_t pos; + if ((ret = scan->next(scan, &k, &v)) != 0) { /* Any parse error has already been reported. */ if (ret == WT_NOTFOUND) @@ -640,6 +648,46 @@ config_opt_str(WTPERF *wtperf, const char *optstr) break; } ret = config_opt(wtperf, &k, &v); + + /* + * Append the key-value pair to our copy of the config. + * The config is stored in the order it is processed, so added + * options will be after any parsed from the original config. + */ + config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); + /* + * If key or value is a string, consider extra space for the + * quotes. Add 2 to the required space for '=' and the ending + * null character in "key=value". + */ + config_line->string = dcalloc( + k.len + (k.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + + v.len + (v.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + 2, 1); + pos = 0; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, k.str, k.len); + pos += k.len; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '='; + pos++; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, v.str, v.len); + pos += v.len; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '\0'; + TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); } if ((t_ret = scan->close(scan)) != 0) { lprintf(wtperf, ret, 0, "Error in config_scan_end"); @@ -663,7 +711,7 @@ config_opt_name_value(WTPERF *wtperf, const char *name, const char *value) /* name="value" */ len = strlen(name) + strlen(value) + 4; optstr = dmalloc(len); - snprintf(optstr, len, "%s=\"%s\"", name, value); + testutil_check(__wt_snprintf(optstr, len, "%s=\"%s\"", name, value)); ret = config_opt_str(wtperf, optstr); free(optstr); return (ret); @@ -728,16 +776,33 @@ config_sanity(WTPERF *wtperf) opts->value_sz_min = opts->value_sz; } - if (opts->readonly && wtperf->workload != NULL) + if (wtperf->workload != NULL) for (i = 0, workp = wtperf->workload; - i < wtperf->workload_cnt; ++i, ++workp) - if (workp->insert != 0 || workp->update != 0 || - workp->truncate != 0) { + i < wtperf->workload_cnt; ++i, ++workp) { + if (opts->readonly && + (workp->insert != 0 || workp->update != 0 || + workp->truncate != 0)) { fprintf(stderr, "Invalid workload: insert, update or " "truncate specified with readonly\n"); return (EINVAL); } + if (workp->insert != 0 && + workp->table_index != INT32_MAX) { + fprintf(stderr, + "Invalid workload: Cannot insert into " + "specific table only\n"); + return (EINVAL); + } + if (workp->table_index != INT32_MAX && + workp->table_index >= (int32_t)opts->table_count) { + fprintf(stderr, + "Workload table index %" PRId32 + " is larger than table count %" PRId32, + workp->table_index, opts->table_count); + return (EINVAL); + } + } return (0); } @@ -754,8 +819,11 @@ config_consolidate(CONFIG_OPTS *opts) /* * This loop iterates over the config queue and for each entry checks if - * a later queue entry has the same key. If there's a match, the current - * queue entry is removed and we continue. + * a later queue entry has the same key. If there's a match, and key is + * "conn_config" or "table_config", the later queue entry is replaced + * with a concatenated entry of the two queue entries, the current queue + * entry is removed. For any other key, if there is a match, the current + * queue entry is removed. */ conf_line = TAILQ_FIRST(&opts->config_head); while (conf_line != NULL) { @@ -771,6 +839,34 @@ config_consolidate(CONFIG_OPTS *opts) if (strncmp(conf_line->string, test_line->string, (size_t)((string_key - conf_line->string) + 1)) == 0) { + if ((strncmp("conn_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + + 1)) == 0) || + (strncmp("table_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + + 1)) == 0)) { + char *concat_str, *val_pointer; + + /* + * To concatenate the two config + * strings, copy the first string to a + * new one, replace the ending '"' with + * a ',' and then concatenate the second + * string's value after its starting '"' + */ + val_pointer = + strchr(test_line->string, '=') + 2; + concat_str = + dmalloc(strlen(conf_line->string) + + strlen(val_pointer) + 1); + strcpy(concat_str, conf_line->string); + concat_str[strlen(concat_str) - 1] = + ','; + strcat(concat_str, val_pointer); + free(test_line->string); + test_line->string = concat_str; + } + TAILQ_REMOVE(&opts->config_head, conf_line, q); free(conf_line->string); free(conf_line); diff --git a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c index 13fa55e86f5..4387860cfb2 100644 --- a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c +++ b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c @@ -80,8 +80,8 @@ cycle_idle_tables(void *arg) } for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { - snprintf(uri, sizeof(uri), - "%s_cycle%07d", wtperf->uris[0], cycle_count); + testutil_check(__wt_snprintf(uri, sizeof(uri), + "%s_cycle%07d", wtperf->uris[0], cycle_count)); /* Don't busy cycle in this loop. */ __wt_sleep(1, 0); @@ -120,6 +120,7 @@ cycle_idle_tables(void *arg) return (NULL); start = stop; +#if 1 /* * Drop the table. Keep retrying on EBUSY failure - it is an * expected return when checkpoints are happening. @@ -136,6 +137,7 @@ cycle_idle_tables(void *arg) } if (check_timing(wtperf, "drop", start, &stop) != 0) return (NULL); +#endif } return (NULL); diff --git a/src/third_party/wiredtiger/bench/wtperf/misc.c b/src/third_party/wiredtiger/bench/wtperf/misc.c index 24b3323a49a..0874794e01e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/misc.c +++ b/src/third_party/wiredtiger/bench/wtperf/misc.c @@ -46,8 +46,8 @@ setup_log_file(WTPERF *wtperf) len = strlen(wtperf->monitor_dir) + strlen(opts->table_name) + strlen(".stat") + 2; fname = dmalloc(len); - snprintf(fname, len, - "%s/%s.stat", wtperf->monitor_dir, opts->table_name); + testutil_check(__wt_snprintf(fname, len, + "%s/%s.stat", wtperf->monitor_dir, opts->table_name)); if ((wtperf->logf = fopen(fname, "w")) == NULL) { ret = errno; fprintf(stderr, "%s: %s\n", fname, strerror(ret)); diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf index 536127f0dd8..4d2a70f1107 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" sess_config="isolation=snapshot" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf index d6218c44af0..6645df835df 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" # close_conn as false allows this test to close/finish faster, but if running diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf index f9aed094aa1..ab7b17ca683 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf @@ -9,7 +9,7 @@ # # This generates about 80 Gb of uncompressed data. But it should compress # well and be small on disk. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" compact=true compression="snappy" sess_config="isolation=snapshot" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf index 2c9540ff589..e8958d20e2c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" sess_config="isolation=snapshot" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf index bbd3a3ba5ed..5daa276e622 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf @@ -1,6 +1,6 @@ # A stress configuration to create long running checkpoints while doing a lot # of updates. -conn_config="cache_size=16GB,eviction=(threads_max=4),log=(enabled=false)" +conn_config="cache_size=16GB,eviction=(threads_max=8),log=(enabled=false)" table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" # Enough data to fill the cache. 150 million 1k records results in two ~11GB # tables diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf index 24da4dd7902..741101d083f 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=1)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf index 25599fadd8d..972bc371f2d 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4),mmap=false" +conn_config="cache_size=50M,eviction=(threads_max=8),mmap=false" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf index a5a29f66fa0..5a2cad6d78e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf @@ -1,4 +1,4 @@ -conn_config="cache_size=1G,eviction=(threads_max=4),session_max=2000" +conn_config="cache_size=1G,eviction=(threads_max=8),session_max=2000" table_config="type=file" table_count=100 close_conn=false diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf index 740fb88c050..96e3f01b325 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4)" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf index e7d967e5c63..3810e6a8294 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4)" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf index ad885d98eb7..641a85dc889 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,eviction=(threads_max=1),lsm_manager=(worker_thread_max=6)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf index 661b8e21924..470dca695dd 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=8)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf index b872d429046..a0f2a78d013 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=8)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf index 6cf50dfb5a5..4379ba22373 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf @@ -16,7 +16,7 @@ # - Config + "-C "checkpoint=(wait=0)": no checkpoints # - Config + "-C "log=(enabled,prealloc=false,file_max=1M)": no pre-allocation # -conn_config="cache_size=5G,log=(enabled=true),checkpoint=(log_size=500M),eviction=(threads_max=4)" +conn_config="cache_size=5G,log=(enabled=true),checkpoint=(log_size=500M),eviction=(threads_max=8)" table_config="type=file" icount=1000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf new file mode 100644 index 00000000000..6cf1d5d2696 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf @@ -0,0 +1,19 @@ +# Create a set of tables with uneven distribution of data +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=20,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +table_config="type=file" +table_count=2000 +icount=0 +random_range=1000000000 +pareto=10 +range_partition=true +report_interval=5 + +run_ops=1000000 +populate_threads=0 +icount=0 +threads=((count=60,inserts=1)) + +# Warn if a latency over 1 second is seen +max_latency=1000 +sample_interval=5 +sample_rate=1 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf index f9e41184f95..58bd1a76b97 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf @@ -1,5 +1,5 @@ # Simulate the MongoDB oplog apply threads on a secondary. -conn_config="cache_size=10GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +conn_config="cache_size=10GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" table_config="allocation_size=4k,memory_page_max=5MB,prefix_compression=false,split_pct=75,leaf_page_max=32k,internal_page_max=16k,type=file" # Spread the workload out over several tables. table_count=4 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf index d7b27f8fda4..f07e6c80b39 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf @@ -2,7 +2,7 @@ # up by dividing the workload across a lot of threads. This needs to be # tuned to the particular machine so the workload is close to capacity in the # steady state, but not overwhelming. -conn_config="cache_size=20GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +conn_config="cache_size=20GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file" # Divide original icount by database_count. table_count=8 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf index b10b08f6035..bee1f431043 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf @@ -1,7 +1,7 @@ # wtperf options file: multi-database configuration attempting to # trigger slow operations by overloading CPU and disk. # References Jira WT-2131 -conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=2),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" +conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=8),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" table_config="allocation_size=4k,prefix_compression=false,split_pct=75,leaf_page_max=4k,internal_page_max=16k,leaf_item_max=1433,internal_item_max=3100,type=file" # Divide original icount by database_count. database_count=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf index ddd9c055eac..1fdba049779 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf @@ -1,5 +1,5 @@ # Create a set of tables with uneven distribution of data -conn_config="cache_size=1G,eviction=(threads_max=4),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" table_config="type=file" table_count=100 icount=0 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf index 380350c88c8..dfb3306a7a5 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf @@ -1,5 +1,5 @@ # Read from a set of tables with uneven distribution of data -conn_config="cache_size=1G,eviction=(threads_max=4),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" table_config="type=file" table_count=100 icount=0 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh index 9968edc468a..fdf3b14e991 100755 --- a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh +++ b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh @@ -99,6 +99,11 @@ while test "$run" -le "$runmax"; do if test "$?" -ne "0"; then exit 1 fi + + # Copy the artifacts from the run + backup_dir=${home}_$(basename $wttest)_${run}_$(date +"%s") + rsync -r -m --include="*Stat*" --include="CONFIG.wtperf" --include="*monitor" --include="latency*" --include="test.stat" --exclude="*" $home/ $backup_dir + # Load is always using floating point, so handle separately l=`grep "^Load time:" ./WT_TEST/test.stat` if test "$?" -eq "0"; then diff --git a/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf index deb8c70d12f..eb6ca1cfddc 100644 --- a/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf @@ -1,4 +1,4 @@ -conn_config="cache_size=2GB,statistics=[fast,clear],statistics_log=(wait=10),eviction=(threads_max=4,threads_min=4)" +conn_config="cache_size=2GB,statistics=[fast,clear],statistics_log=(wait=10),eviction=(threads_max=8,threads_min=4)" table_config="type=file,leaf_page_max=8k,internal_page_max=8k,memory_page_max=2MB,split_deepen_min_child=250" icount=200000 report_interval=5 @@ -6,5 +6,4 @@ run_time=300 reopen_connection=false populate_threads=2 value_sz=256 -read_range=100 -threads=((count=4,inserts=1,throttle=100000),(count=8,reads=1)) +threads=((count=4,inserts=1,throttle=100000),(count=8,reads=1,read_range=100)) diff --git a/src/third_party/wiredtiger/bench/wtperf/track.c b/src/third_party/wiredtiger/bench/wtperf/track.c index 822bdaa4b4a..86a26120a6a 100644 --- a/src/third_party/wiredtiger/bench/wtperf/track.c +++ b/src/third_party/wiredtiger/bench/wtperf/track.c @@ -288,8 +288,8 @@ latency_print_single(WTPERF *wtperf, TRACK *total, const char *name) uint64_t cumops; char path[1024]; - snprintf(path, sizeof(path), - "%s/latency.%s", wtperf->monitor_dir, name); + testutil_check(__wt_snprintf(path, sizeof(path), + "%s/latency.%s", wtperf->monitor_dir, name)); if ((fp = fopen(path, "w")) == NULL) { lprintf(wtperf, errno, 0, "%s", path); return; diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 8c7f0053388..6d79eebe8b2 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -32,9 +32,6 @@ #define DEFAULT_HOME "WT_TEST" #define DEFAULT_MONITOR_DIR "WT_TEST" -static const char * const debug_cconfig = ""; -static const char * const debug_tconfig = ""; - static void *checkpoint_worker(void *); static int drop_all_tables(WTPERF *); static int execute_populate(WTPERF *); @@ -432,19 +429,17 @@ err: wtperf->error = wtperf->stop = true; * search do them. Ensuring the keys we see are always in order. */ static int -do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) +do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor, int64_t read_range) { - CONFIG_OPTS *opts; - size_t range; uint64_t next_val, prev_val; + int64_t range; char *range_key_buf; char buf[512]; int ret; - opts = wtperf->opts; ret = 0; - if (opts->read_range == 0) + if (read_range == 0) return (0); memset(&buf[0], 0, 512 * sizeof(char)); @@ -454,7 +449,7 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) testutil_check(cursor->get_key(cursor, &range_key_buf)); extract_key(range_key_buf, &next_val); - for (range = 0; range < opts->read_range; ++range) { + for (range = 0; range < read_range; ++range) { prev_val = next_val; ret = cursor->next(cursor); /* We are done if we reach the end. */ @@ -475,12 +470,56 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) return (0); } +/* pre_load_data -- + * Pull everything into cache before starting the workload phase. + */ +static int +pre_load_data(WTPERF *wtperf) +{ + CONFIG_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + char *key; + int ret; + size_t i; + + opts = wtperf->opts; + conn = wtperf->conn; + + if ((ret = conn->open_session( + conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); + goto err; + } + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, + wtperf->uris[i], NULL, NULL, &cursor)) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[i]); + goto err; + } + while (cursor->next(cursor) == 0) + if ((ret = cursor->get_key(cursor, &key)) != 0) + goto err; + if ((ret = cursor->close(cursor)) != 0) + goto err; + } + if ((ret = session->close(session, NULL)) != 0) + goto err; + if (ret != 0) +err: lprintf(wtperf, ret, 0, "Pre-workload traverse error"); + return (ret); +} + static void * worker(void *arg) { struct timespec start, stop; CONFIG_OPTS *opts; TRACK *trk; + WORKLOAD *workload; WTPERF *wtperf; WTPERF_THREAD *thread; WT_CONNECTION *conn; @@ -495,13 +534,14 @@ worker(void *arg) char buf[512]; thread = (WTPERF_THREAD *)arg; + workload = thread->workload; wtperf = thread->wtperf; opts = wtperf->opts; conn = wtperf->conn; cursors = NULL; - log_table_cursor = NULL; /* -Wconditional-initialized */ + cursor = log_table_cursor = NULL; /* -Wconditional-initialized */ ops = 0; - ops_per_txn = thread->workload->ops_per_txn; + ops_per_txn = workload->ops_per_txn; session = NULL; trk = NULL; @@ -510,9 +550,9 @@ worker(void *arg) lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); goto err; } - cursors = dcalloc(opts->table_count, sizeof(WT_CURSOR *)); for (i = 0; i < opts->table_count_idle; i++) { - snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i); + testutil_check(__wt_snprintf( + buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); if ((ret = session->open_cursor( session, buf, NULL, NULL, &tmp_cursor)) != 0) { lprintf(wtperf, ret, 0, @@ -525,14 +565,34 @@ worker(void *arg) goto err; } } - for (i = 0; i < opts->table_count; i++) { + if (workload->table_index != INT32_MAX) { if ((ret = session->open_cursor(session, - wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { + wtperf->uris[workload->table_index], + NULL, NULL, &cursor)) != 0) { lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: %s", - wtperf->uris[i]); + wtperf->uris[workload->table_index]); + goto err; + } + if ((ret = session->open_cursor(session, + wtperf->uris[workload->table_index], + NULL, "next_random=true", &thread->rand_cursor)) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: random %s", + wtperf->uris[workload->table_index]); goto err; } + } else { + cursors = dcalloc(opts->table_count, sizeof(WT_CURSOR *)); + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, + wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[i]); + goto err; + } + } } if (opts->log_like_table && (ret = session->open_cursor(session, wtperf->log_table_uri, NULL, NULL, &log_table_cursor)) != 0) { @@ -543,19 +603,19 @@ worker(void *arg) } /* Setup the timer for throttling. */ - if (thread->workload->throttle != 0) + if (workload->throttle != 0) setup_throttle(thread); /* Setup for truncate */ - if (thread->workload->truncate != 0) + if (workload->truncate != 0) if ((ret = setup_truncate(wtperf, thread, session)) != 0) goto err; key_buf = thread->key_buf; value_buf = thread->value_buf; - op = thread->workload->ops; - op_end = op + sizeof(thread->workload->ops); + op = workload->ops; + op_end = op + sizeof(workload->ops); if ((ops_per_txn != 0 || opts->log_like_table) && (ret = session->begin_transaction(session, NULL)) != 0) { @@ -564,6 +624,8 @@ worker(void *arg) } while (!wtperf->stop) { + if (workload->pause != 0) + (void)sleep((unsigned int)workload->pause); /* * Generate the next key and setup operation specific * statistics tracking objects. @@ -603,10 +665,12 @@ worker(void *arg) generate_key(opts, key_buf, next_val); - /* - * Spread the data out around the multiple databases. - */ - cursor = cursors[map_key_to_table(wtperf->opts, next_val)]; + if (workload->table_index == INT32_MAX) + /* + * Spread the data out around the multiple databases. + */ + cursor = cursors[ + map_key_to_table(wtperf->opts, next_val)]; /* * Skip the first time we do an operation, when trk->ops @@ -642,7 +706,8 @@ worker(void *arg) * for several operations, confirming that the * next key is in the correct order. */ - ret = do_range_reads(wtperf, cursor); + ret = do_range_reads(wtperf, + cursor, workload->read_range); } if (ret == 0 || ret == WT_NOTFOUND) @@ -689,7 +754,7 @@ worker(void *arg) */ strncpy(value_buf, value, opts->value_sz_max - 1); - if (thread->workload->update_delta != 0) + if (workload->update_delta != 0) update_value_delta(thread); if (value_buf[0] == 'a') value_buf[0] = 'b'; @@ -806,7 +871,7 @@ op_err: if (ret == WT_ROLLBACK && ops_per_txn != 0) { /* Schedule the next operation */ if (++op == op_end) - op = thread->workload->ops; + op = workload->ops; /* * Decrement throttle ops and check if we should sleep @@ -843,7 +908,7 @@ run_mix_schedule_op(WORKLOAD *workp, int op, int64_t op_cnt) uint8_t *p, *end; /* Jump around the array to roughly spread out the operations. */ - jump = 100 / op_cnt; + jump = (int)(100 / op_cnt); /* * Find a read operation and replace it with another operation. This @@ -884,17 +949,6 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) opts = wtperf->opts; - /* Confirm reads, inserts, truncates and updates cannot all be zero. */ - if (workp->insert == 0 && workp->read == 0 && - workp->truncate == 0 && workp->update == 0) { - lprintf(wtperf, EINVAL, 0, "no operations scheduled"); - return (EINVAL); - } - - /* - * Handle truncate first - it's a special case that can't be used in - * a mixed workload. - */ if (workp->truncate != 0) { if (workp->insert != 0 || workp->read != 0 || workp->update != 0) { @@ -906,6 +960,12 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) return (0); } + /* Confirm reads, inserts and updates cannot all be zero. */ + if (workp->insert == 0 && workp->read == 0 && workp->update == 0) { + lprintf(wtperf, EINVAL, 0, "no operations scheduled"); + return (EINVAL); + } + /* * Check for a simple case where the thread is only doing insert or * update operations (because the default operation for a @@ -1238,7 +1298,8 @@ monitor(void *arg) /* Open the logging file. */ len = strlen(wtperf->monitor_dir) + 100; path = dmalloc(len); - snprintf(path, len, "%s/monitor", wtperf->monitor_dir); + testutil_check(__wt_snprintf( + path, len, "%s/monitor", wtperf->monitor_dir)); if ((fp = fopen(path, "w")) == NULL) { lprintf(wtperf, errno, 0, "%s", path); goto err; @@ -1593,6 +1654,9 @@ close_reopen(WTPERF *wtperf) opts = wtperf->opts; + if (opts->in_memory) + return (0); + if (!opts->readonly && !opts->reopen_connection) return (0); /* @@ -1875,19 +1939,19 @@ create_uris(WTPERF *wtperf) /* If there is only one table, just use the base name. */ wtperf->uris[i] = dmalloc(len); if (opts->table_count == 1) - snprintf(wtperf->uris[i], - len, "table:%s", opts->table_name); + testutil_check(__wt_snprintf(wtperf->uris[i], + len, "table:%s", opts->table_name)); else - snprintf(wtperf->uris[i], - len, "table:%s%05d", opts->table_name, i); + testutil_check(__wt_snprintf(wtperf->uris[i], + len, "table:%s%05d", opts->table_name, i)); } /* Create the log-like-table URI. */ len = strlen("table:") + strlen(opts->table_name) + strlen("_log_table") + 1; wtperf->log_table_uri = dmalloc(len); - snprintf( - wtperf->log_table_uri, len, "table:%s_log_table", opts->table_name); + testutil_check(__wt_snprintf(wtperf->log_table_uri, + len, "table:%s_log_table", opts->table_name)); } static int @@ -1909,7 +1973,8 @@ create_tables(WTPERF *wtperf) } for (i = 0; i < opts->table_count_idle; i++) { - snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i); + testutil_check(__wt_snprintf( + buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); if ((ret = session->create( session, buf, opts->table_config)) != 0) { lprintf(wtperf, ret, 0, @@ -1938,8 +2003,9 @@ create_tables(WTPERF *wtperf) return (ret); } if (opts->index) { - snprintf(buf, 512, "index:%s:val_idx", - wtperf->uris[i] + strlen("table:")); + testutil_check(__wt_snprintf(buf, 512, + "index:%s:val_idx", + wtperf->uris[i] + strlen("table:"))); if ((ret = session->create( session, buf, "columns=(val)")) != 0) { lprintf(wtperf, ret, 0, @@ -2124,15 +2190,15 @@ start_all_runs(WTPERF *wtperf) */ len = strlen(wtperf->home) + 5; next_wtperf->home = dmalloc(len); - snprintf( - next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i); + testutil_check(__wt_snprintf( + next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i)); if (opts->create != 0) recreate_dir(next_wtperf->home); len = strlen(wtperf->monitor_dir) + 5; next_wtperf->monitor_dir = dmalloc(len); - snprintf(next_wtperf->monitor_dir, - len, "%s/D%02d", wtperf->monitor_dir, (int)i); + testutil_check(__wt_snprintf(next_wtperf->monitor_dir, + len, "%s/D%02d", wtperf->monitor_dir, (int)i)); if (opts->create != 0 && strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) recreate_dir(next_wtperf->monitor_dir); @@ -2244,6 +2310,8 @@ start_run(WTPERF *wtperf) opts->checkpoint_threads, checkpoint_worker) != 0) goto err; } + if (opts->pre_load_data && (ret = pre_load_data(wtperf)) != 0) + goto err; /* Execute the workload. */ if ((ret = execute_workload(wtperf)) != 0) goto err; @@ -2361,11 +2429,11 @@ main(int argc, char *argv[]) { CONFIG_OPTS *opts; WTPERF *wtperf, _wtperf; - size_t req_len, sreq_len; + size_t pos, req_len, sreq_len; bool monitor_set; int ch, ret; const char *cmdflags = "C:h:m:O:o:T:"; - const char *config_opts; + const char *append_comma, *config_opts; char *cc_buf, *path, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig; /* The first WTPERF structure (from which all others are derived). */ @@ -2479,9 +2547,9 @@ main(int argc, char *argv[]) */ req_len = strlen(",async=(enabled=true,threads=)") + 4; wtperf->async_config = dmalloc(req_len); - snprintf(wtperf->async_config, req_len, + testutil_check(__wt_snprintf(wtperf->async_config, req_len, ",async=(enabled=true,threads=%" PRIu32 ")", - opts->async_threads); + opts->async_threads)); } if ((ret = config_compress(wtperf)) != 0) goto err; @@ -2502,53 +2570,96 @@ main(int argc, char *argv[]) __wt_stream_set_line_buffer(stdout); /* Concatenate non-default configuration strings. */ - if (opts->verbose > 1 || user_cconfig != NULL || - opts->session_count_idle > 0 || wtperf->compress_ext != NULL || - wtperf->async_config != NULL) { - req_len = strlen(debug_cconfig) + 20; - if (user_cconfig != NULL) - req_len += strlen(user_cconfig); - if (wtperf->async_config != NULL) - req_len += strlen(wtperf->async_config); - if (wtperf->compress_ext != NULL) - req_len += strlen(wtperf->compress_ext); + if (user_cconfig != NULL || opts->session_count_idle > 0 || + wtperf->compress_ext != NULL || wtperf->async_config != NULL || + opts->in_memory) { + req_len = 20; + req_len += wtperf->async_config != NULL ? + strlen(wtperf->async_config) : 0; + req_len += wtperf->compress_ext != NULL ? + strlen(wtperf->compress_ext) : 0; if (opts->session_count_idle > 0) { - sreq_len = strlen(",session_max=") + 6; + sreq_len = strlen("session_max=") + 6; req_len += sreq_len; sess_cfg = dmalloc(sreq_len); - snprintf(sess_cfg, sreq_len, - ",session_max=%" PRIu32, + testutil_check(__wt_snprintf(sess_cfg, sreq_len, + "session_max=%" PRIu32, opts->session_count_idle + - wtperf->workers_cnt + opts->populate_threads + 10); + wtperf->workers_cnt + opts->populate_threads + 10)); } + req_len += opts->in_memory ? strlen("in_memory=true") : 0; + req_len += user_cconfig != NULL ? strlen(user_cconfig) : 0; cc_buf = dmalloc(req_len); - snprintf(cc_buf, req_len, "%s,%s,%s,%s,%s", - wtperf->async_config ? wtperf->async_config : "", - wtperf->compress_ext ? wtperf->compress_ext : "", - opts->verbose > 1 ? debug_cconfig : "", - sess_cfg != NULL ? sess_cfg : "", - user_cconfig != NULL ? user_cconfig : ""); - if (strlen(cc_buf) && (ret = + + pos = 0; + append_comma = ""; + if (wtperf->async_config != NULL && + strlen(wtperf->async_config) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->async_config)); + append_comma = ","; + } + if (wtperf->compress_ext != NULL && + strlen(wtperf->compress_ext) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->compress_ext)); + append_comma = ","; + } + if (opts->in_memory) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, "in_memory=true")); + append_comma = ","; + } + if (sess_cfg != NULL && strlen(sess_cfg) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, sess_cfg)); + append_comma = ","; + } + if (user_cconfig != NULL && strlen(user_cconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, user_cconfig)); + } + + if (strlen(cc_buf) != 0 && (ret = config_opt_name_value(wtperf, "conn_config", cc_buf)) != 0) goto err; } - if (opts->verbose > 1 || opts->index || + if (opts->index || user_tconfig != NULL || wtperf->compress_table != NULL) { - req_len = strlen(debug_tconfig) + 20; - if (user_tconfig != NULL) - req_len += strlen(user_tconfig); - if (wtperf->compress_table != NULL) - req_len += strlen(wtperf->compress_table); - if (opts->index) - req_len += strlen(INDEX_COL_NAMES); + req_len = 20; + req_len += wtperf->compress_table != NULL ? + strlen(wtperf->compress_table) : 0; + req_len += opts->index ? strlen(INDEX_COL_NAMES) : 0; + req_len += user_tconfig != NULL ? strlen(user_tconfig) : 0; tc_buf = dmalloc(req_len); - snprintf(tc_buf, req_len, "%s,%s,%s,%s", - opts->index ? INDEX_COL_NAMES : "", - wtperf->compress_table != NULL ? - wtperf->compress_table : "", - opts->verbose > 1 ? debug_tconfig : "", - user_tconfig ? user_tconfig : ""); - if (strlen(tc_buf) && (ret = + + pos = 0; + append_comma = ""; + if (wtperf->compress_table != NULL && + strlen(wtperf->compress_table) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->compress_table)); + append_comma = ","; + } + if (opts->index) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, INDEX_COL_NAMES)); + append_comma = ","; + } + if (user_tconfig != NULL && strlen(user_tconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, user_tconfig)); + } + + if (strlen(tc_buf) != 0 && (ret = config_opt_name_value(wtperf, "table_config", tc_buf)) != 0) goto err; } @@ -2556,8 +2667,9 @@ main(int argc, char *argv[]) req_len = strlen(opts->table_config) + strlen(LOG_PARTIAL_CONFIG) + 1; wtperf->partial_config = dmalloc(req_len); - snprintf(wtperf->partial_config, req_len, "%s%s", - opts->table_config, LOG_PARTIAL_CONFIG); + testutil_check(__wt_snprintf( + wtperf->partial_config, req_len, "%s%s", + opts->table_config, LOG_PARTIAL_CONFIG)); } /* * Set the config for reopen. If readonly add in that string. @@ -2570,11 +2682,12 @@ main(int argc, char *argv[]) req_len = strlen(opts->conn_config) + 1; wtperf->reopen_config = dmalloc(req_len); if (opts->readonly) - snprintf(wtperf->reopen_config, req_len, "%s%s", - opts->conn_config, READONLY_CONFIG); + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s%s", + opts->conn_config, READONLY_CONFIG)); else - snprintf(wtperf->reopen_config, - req_len, "%s", opts->conn_config); + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s", opts->conn_config)); /* Sanity-check the configuration. */ if ((ret = config_sanity(wtperf)) != 0) @@ -2587,7 +2700,8 @@ main(int argc, char *argv[]) /* Write a copy of the config. */ req_len = strlen(wtperf->home) + strlen("/CONFIG.wtperf") + 1; path = dmalloc(req_len); - snprintf(path, req_len, "%s/CONFIG.wtperf", wtperf->home); + testutil_check(__wt_snprintf( + path, req_len, "%s/CONFIG.wtperf", wtperf->home)); config_opt_log(opts, path); free(path); @@ -2712,7 +2826,8 @@ recreate_dir(const char *name) len = strlen(name) * 2 + 100; buf = dmalloc(len); - (void)snprintf(buf, len, "rm -rf %s && mkdir %s", name, name); + testutil_check(__wt_snprintf( + buf, len, "rm -rf %s && mkdir %s", name, name)); testutil_checkfmt(system(buf), "system: %s", buf); free(buf); } @@ -2779,13 +2894,42 @@ static uint64_t wtperf_rand(WTPERF_THREAD *thread) { CONFIG_OPTS *opts; + WT_CURSOR *rnd_cursor; WTPERF *wtperf; double S1, S2, U; uint64_t rval; + int ret; + char *key_buf; wtperf = thread->wtperf; opts = wtperf->opts; + /* + * If we have a random cursor set up then use it. + */ + if ((rnd_cursor = thread->rand_cursor) != NULL) { + if ((ret = rnd_cursor->next(rnd_cursor)) != 0) { + lprintf(wtperf, ret, 0, "worker: rand next failed"); + /* 0 is outside the expected range. */ + return (0); + } + if ((ret = rnd_cursor->get_key(rnd_cursor, &key_buf)) != 0) { + lprintf(wtperf, ret, 0, + "worker: rand next key retrieval"); + return (0); + } + /* + * Resetting the cursor is not fatal. We still return the + * value we retrieved above. We do it so that we don't + * leave a cursor positioned. + */ + if ((ret = rnd_cursor->reset(rnd_cursor)) != 0) + lprintf(wtperf, ret, 0, + "worker: rand cursor reset failed"); + extract_key(key_buf, &rval); + return (rval); + } + /* * Use WiredTiger's random number routine: it's lock-free and fairly * good. diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 81d74e134f6..3efb8ab700e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -66,6 +66,9 @@ typedef struct { uint64_t throttle; /* Maximum operations/second */ /* Number of operations per transaction. Zero for autocommit */ int64_t ops_per_txn; + int64_t pause; /* Time between scans */ + int64_t read_range; /* Range of reads */ + int32_t table_index; /* Table to focus ops on */ int64_t truncate; /* Truncate ratio */ uint64_t truncate_pct; /* Truncate Percent */ uint64_t truncate_count; /* Truncate Count */ @@ -225,6 +228,7 @@ typedef struct { struct __wtperf_thread { /* Per-thread structure */ WTPERF *wtperf; /* Enclosing configuration */ + WT_CURSOR *rand_cursor; /* Random key cursor */ WT_RAND_STATE rnd; /* Random number generation state */ diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index 680eb53a90e..90f70457407 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -110,6 +110,8 @@ DEF_OPT_AS_UINT32(database_count, 1, DEF_OPT_AS_BOOL(drop_tables, 0, "Whether to drop all tables at the end of the run, and report time taken" " to do the drop.") +DEF_OPT_AS_BOOL(in_memory, 0, + "Whether to create the database in-memory.") DEF_OPT_AS_UINT32(icount, 5000, "number of records to initially populate. If multiple tables are " "configured the count is spread evenly across all tables.") @@ -145,12 +147,13 @@ DEF_OPT_AS_UINT32(populate_ops_per_txn, 0, "phase, zero for auto-commit") DEF_OPT_AS_UINT32(populate_threads, 1, "number of populate threads, 1 for bulk load") +DEF_OPT_AS_BOOL(pre_load_data, 0, + "Scan all data prior to starting the workload phase to warm the cache") DEF_OPT_AS_UINT32(random_range, 0, "if non zero choose a value from within this range as the key for " "insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_BOOL(range_partition, 0, "partition data by range (vs hash)") -DEF_OPT_AS_UINT32(read_range, 0, "scan a range of keys after each search") DEF_OPT_AS_BOOL(readonly, 0, "reopen the connection between populate and workload phases in readonly " "mode. Requires reopen_connection turned on (default). Requires that " @@ -192,9 +195,10 @@ DEF_OPT_AS_STRING(threads, "", "workload configuration: each 'count' " "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " "which would create 2 threads doing nothing but reads and 8 threads " "each doing 50% inserts and 25% reads and updates. Allowed configuration " - "values are 'count', 'throttle', 'update_delta', 'reads', 'inserts', " - "'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are " - "also behavior modifiers, supported modifiers are 'ops_per_txn'") + "values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', " + "'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. " + "There are also behavior modifiers, supported modifiers are " + "'ops_per_txn'") DEF_OPT_AS_CONFIG_STRING(transaction_config, "", "WT_SESSION.begin_transaction configuration string, applied during the " "populate phase when populate_ops_per_txn is nonzero") diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base index 5b945aca5e0..e5228fac885 100644 --- a/src/third_party/wiredtiger/build_posix/Make.base +++ b/src/third_party/wiredtiger/build_posix/Make.base @@ -17,6 +17,7 @@ endif bin_PROGRAMS = wt wt_SOURCES =\ + src/utilities/util_alter.c \ src/utilities/util_backup.c \ src/utilities/util_cpyright.c \ src/utilities/util_compact.c \ @@ -35,6 +36,7 @@ wt_SOURCES =\ src/utilities/util_rename.c \ src/utilities/util_salvage.c \ src/utilities/util_stat.c \ + src/utilities/util_truncate.c \ src/utilities/util_upgrade.c \ src/utilities/util_verbose.c \ src/utilities/util_verify.c \ diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index 55941837249..4ecec37ca6c 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -17,6 +17,7 @@ ext/encryptors/nop ext/encryptors/rotn ext/extractors/csv ext/test/kvs_bdb HAVE_BERKELEY_DB +ext/test/fail_fs . api/leveldb LEVELDB examples/c @@ -40,6 +41,7 @@ test/packing test/readonly test/recovery test/salvage +test/syscall test/thread # Benchmark programs. diff --git a/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 b/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 new file mode 100644 index 00000000000..ca3639715e7 --- /dev/null +++ b/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 @@ -0,0 +1,74 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_COMPILE_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 4 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 7043430a6d6..bc4b31dfee3 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -57,7 +57,7 @@ AH_TEMPLATE( HAVE_CRC32_HARDWARE, [Define to 1 to configure CRC32 hardware support.]) AC_MSG_CHECKING(if --enable-crc32-hardware option specified) AC_ARG_ENABLE(crc32-hardware, - AC_HELP_STRING([--enable-crc32-hardware], + AS_HELP_STRING([--enable-crc32-hardware], [Enable CRC32 hardware support.]), r=$enableval, r=yes) case "$r" in no) wt_cv_enable_crc32_hardware=no;; diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index c107dd017d7..659867fa69e 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -31,6 +31,7 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wstrict-prototypes" w="$w -Wswitch-enum" w="$w -Wundef" + w="$w -Wuninitialized" w="$w -Wunreachable-code" w="$w -Wunsafe-loop-optimizations" w="$w -Wunused" @@ -66,6 +67,10 @@ AC_DEFUN([AM_CLANG_WARNINGS], [ # w="$w -Wno-error=cast-qual" w="$w -Wno-cast-qual" + # On Centos 7.3.1611, system header files aren't compatible with + # -Wdisabled-macro-expansion. + w="$w -Wno-disabled-macro-expansion" + case "$1" in *Apple*clang*version*4.1*) # Apple clang has its own numbering system, and older OS X diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 index 997f571782c..c677ce41192 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 @@ -1,14 +1,14 @@ dnl build by dist/s_version VERSION_MAJOR=2 -VERSION_MINOR=8 -VERSION_PATCH=1 -VERSION_STRING='"WiredTiger 2.8.1: (March 24, 2016)"' +VERSION_MINOR=9 +VERSION_PATCH=2 +VERSION_STRING='"WiredTiger 2.9.2: (December 23, 2016)"' AC_SUBST(VERSION_MAJOR) AC_SUBST(VERSION_MINOR) AC_SUBST(VERSION_PATCH) AC_SUBST(VERSION_STRING) -VERSION_NOPATCH=2.8 +VERSION_NOPATCH=2.9 AC_SUBST(VERSION_NOPATCH) diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version.m4 index 0e199cb9546..29782a22f82 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version.m4 @@ -1,2 +1,2 @@ dnl WiredTiger product version for AC_INIT. Maintained by dist/s_version -2.8.1 +2.9.2 diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index ad00b19a3bb..0fef587b4b8 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -22,33 +22,12 @@ AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) AM_PROG_AS(as gas) -# Configure options. -AM_OPTIONS - define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl + LT_PREREQ(2.2.6) LT_INIT([pic-only]) AC_SUBST([LIBTOOL_DEPS]) - -# If enable-strict is configured, turn on as much error checking as we can for -# this compiler. Intended for developers, and only works for gcc/clang, but it -# fills a need. -if test "$wt_cv_enable_strict" = "yes"; then - wt_cv_cc_version="`$CC --version | sed -eq`" - case "$wt_cv_cc_version" in - *clang*) - AM_CLANG_WARNINGS($wt_cv_cc_version);; - *cc*|*CC*) # cc, CC, gcc, GCC - AM_GCC_WARNINGS($wt_cv_cc_version);; - *) - AC_MSG_ERROR( - [--enable-strict does not support "$wt_cv_cc_version".]);; - esac - - AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" -fi - AM_CONDITIONAL([POSIX_HOST], [true]) AM_CONDITIONAL([WINDOWS_HOST], [false]) @@ -69,6 +48,7 @@ AS_CASE([$host_cpu], [aarch64*], [wt_cv_arm64="yes"], [wt_cv_arm64="no"]) AM_CONDITIONAL([ARM64_HOST], [test "$wt_cv_arm64" = "yes"]) +AS_CASE([$host_os], [*solaris*], [wt_cv_solaris="yes"], [wt_cv_solaris="no"]) # This is a workaround as part of WT-2459. Currently, clang (v3.7) does not # support compiling the ASM code we have to perform the CRC checks on PowerPC. @@ -82,9 +62,17 @@ if test "$wt_cv_powerpc" = "yes" -a "$CC" != "$CCAS"; then fi AC_SUBST(AM_LIBTOOLFLAGS) +# WiredTiger uses anonymous unions to pad structures. It's part of C11, but +# some compilers require -std=c11 to support them. Turn on that flag for any +# compiler that supports it, except for Solaris, where gcc -std=c11 makes +# some none-C11 prototypes unavailable. +if test "$wt_cv_solaris" = "no"; then + AX_CHECK_COMPILE_FLAG([-std=c11], [AM_CFLAGS="$AM_CFLAGS -std=c11"]) +fi + if test "$GCC" = "yes"; then # The Solaris gcc compiler gets the additional -pthreads flag. - if test "`uname -s`" = "SunOS"; then + if test "$wt_cv_solaris" = "yes"; then AM_CFLAGS="$AM_CFLAGS -pthreads" fi @@ -95,11 +83,35 @@ if test "$GCC" = "yes"; then fi else # The Solaris native compiler gets the additional -mt flag. - if test "`uname -s`" = "SunOS"; then + if test "$wt_cv_solaris" = "yes"; then AM_CFLAGS="$AM_CFLAGS -mt" fi fi +# Linux requires _GNU_SOURCE to be defined +AS_CASE([$host_os], [linux*], [AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE"]) + +# Configure options. +AM_OPTIONS + +# If enable-strict is configured, turn on as much error checking as we can for +# this compiler. Intended for developers, and only works for gcc/clang, but it +# fills a need. +if test "$wt_cv_enable_strict" = "yes"; then + wt_cv_cc_version="`$CC --version | sed -eq`" + case "$wt_cv_cc_version" in + *clang*) + AM_CLANG_WARNINGS($wt_cv_cc_version);; + *cc*|*CC*) # cc, CC, gcc, GCC + AM_GCC_WARNINGS($wt_cv_cc_version);; + *) + AC_MSG_ERROR( + [--enable-strict does not support "$wt_cv_cc_version".]);; + esac + + AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" +fi + # Java and Python APIs if test "$wt_cv_enable_java" = "yes" -o "$wt_cv_enable_python" = "yes"; then # Only a warning, we need to build release packages without SWIG. @@ -159,11 +171,6 @@ if test "$ac_cv_sizeof_void_p" != "8" ; then fi AC_MSG_RESULT(yes) -# Linux requires _GNU_SOURCE to be defined -case "$host_os" in -linux*) AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE" ;; -esac - # Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. BUFFER_ALIGNMENT=0 if test "$ax_cv_func_posix_memalign_works" = "yes" ; then diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 1069a7037ce..d83a632321e 100644 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -198,7 +198,8 @@ def get_default(c): return 'false' elif c.default == 'true': return 'true' - elif t == 'string' and c.default == 'none': + elif t == 'string' and c.default == 'none' and \ + not c.flags.get('choices', []): return '' elif t == 'category': return '(%s)' % (','.join('%s=%s' % (subc.name, get_default(subc)) diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 22d06c380ae..22600dd5e29 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -118,8 +118,23 @@ lsm_config = [ ]), ] +file_runtime_config = [ + Config('access_pattern_hint', 'none', r''' + It is recommended that workloads that consist primarily of + updates and/or point queries specify \c random. Workloads that + do many cursor scans through large ranges of data specify + \c sequential and other workloads specify \c none. The + option leads to an advisory call to an appropriate operating + system API where available''', + choices=['none', 'random', 'sequential']), + Config('cache_resident', 'false', r''' + do not ever evict the object's pages from cache. Not compatible with + LSM tables; see @ref tuning_cache_resident for more information''', + type='boolean'), +] + # Per-file configuration -file_config = format_meta + [ +file_config = format_meta + file_runtime_config + [ Config('block_allocation', 'best', r''' configure block allocation. Permitted values are \c "first" or \c "best"; the \c "first" configuration uses a first-available @@ -138,10 +153,6 @@ file_config = format_meta + [ WT_CONNECTION::add_compressor. If WiredTiger has builtin support for \c "lz4", \c "snappy", \c "zlib" or \c "zstd" compression, these names are also available. See @ref compression for more information'''), - Config('cache_resident', 'false', r''' - do not ever evict the object's pages from cache. Not compatible with - LSM tables; see @ref tuning_cache_resident for more information''', - type='boolean'), Config('checksum', 'uncompressed', r''' configure block checksums; permitted values are on (checksum all blocks), off (checksum no blocks) and @@ -190,9 +201,8 @@ file_config = format_meta + [ WiredTiger to consume memory over the configured cache limit''', type='boolean'), Config('internal_key_truncate', 'true', r''' - configure internal key truncation, discarding unnecessary - trailing bytes on internal keys (ignored for custom - collators)''', + configure internal key truncation, discarding unnecessary trailing + bytes on internal keys (ignored for custom collators)''', type='boolean'), Config('internal_page_max', '4KB', r''' the maximum page size for internal nodes, in bytes; the size @@ -285,12 +295,12 @@ file_config = format_meta + [ Config('split_deepen_per_child', '0', r''' entries allocated per child when deepening the tree''', type='int', undoc=True), - Config('split_pct', '75', r''' + Config('split_pct', '90', r''' the Btree page split size as a percentage of the maximum Btree page size, that is, when a Btree page is split, it will be split into smaller pages, where each page is the specified percentage of the maximum Btree page size''', - min='25', max='100'), + min='50', max='100'), ] # File metadata, including both configurable and non-configurable (internal) @@ -396,7 +406,7 @@ connection_runtime_config = [ Config('eviction', '', r''' eviction configuration options''', type='category', subconfig=[ - Config('threads_max', '1', r''' + Config('threads_max', '8', r''' maximum number of threads WiredTiger will start to help evict pages from cache. The number of threads started will vary depending on the current eviction load. Each eviction worker @@ -514,6 +524,7 @@ connection_runtime_config = [ 'checkpoint', 'compact', 'evict', + 'evict_stuck', 'evictserver', 'fileops', 'handleops', @@ -527,6 +538,7 @@ connection_runtime_config = [ 'rebalance', 'reconcile', 'recovery', + 'recovery_progress', 'salvage', 'shared_cache', 'split', @@ -637,6 +649,12 @@ wiredtiger_open_statistics_log_configuration = [ ] session_config = [ + Config('ignore_cache_size', 'false', r''' + when set, operations performed by this session ignore the cache size + and are not blocked when the cache is full. Note that use of this + option for operations that create cache pressure can starve ordinary + sessions that obey the cache size.''', + type='boolean'), Config('isolation', 'read-committed', r''' the default isolation level for operations in this session''', choices=['read-uncommitted', 'read-committed', 'snapshot']), @@ -652,6 +670,11 @@ wiredtiger_open_common =\ should be used (4KB on Linux systems when direct I/O is configured, zero elsewhere)''', min='-1', max='1MB'), + Config('builtin_extension_config', '', r''' + A structure where the keys are the names of builtin extensions and the + values are passed to WT_CONNECTION::load_extension as the \c config + parameter (for example, + builtin_extension_config={zlib={compression_level=3}})'''), Config('checkpoint_sync', 'true', r''' flush files to stable storage when closing or writing checkpoints''', @@ -695,7 +718,7 @@ wiredtiger_open_common =\ ]), Config('extensions', '', r''' list of shared library extensions to load (using dlopen). - Any values specified to an library extension are passed to + Any values specified to a library extension are passed to WT_CONNECTION::load_extension as the \c config parameter (for example, extensions=(/path/ext.so={entry=my_entry}))''', @@ -709,7 +732,7 @@ wiredtiger_open_common =\ Config('hazard_max', '1000', r''' maximum number of simultaneous hazard pointers per session handle''', - min='15'), + min=15, undoc=True), Config('mmap', 'true', r''' Use memory mapping to access files when possible''', type='boolean'), @@ -816,6 +839,8 @@ methods = { 'WT_CURSOR.reconfigure' : Method(cursor_runtime_config), +'WT_SESSION.alter' : Method(file_runtime_config), + 'WT_SESSION.close' : Method([]), 'WT_SESSION.compact' : Method([ @@ -837,9 +862,10 @@ methods = { 'WT_SESSION.drop' : Method([ Config('checkpoint_wait', 'true', r''' - wait for the checkpoint lock, if \c checkpoint_wait=false, perform - the drop operation without taking a lock, returning EBUSY if the - operation conflicts with a running checkpoint''', + wait for concurrent checkpoints to complete before attempting the drop + operation. If \c checkpoint_wait=false, attempt the drop operation + without waiting, returning EBUSY if the operation conflicts with a + running checkpoint''', type='boolean', undoc=True), Config('force', 'false', r''' return success if the object does not exist''', @@ -1109,6 +1135,10 @@ methods = { Config('to', '', r''' drop all snapshots up to and including the specified name'''), ]), + Config('include_updates', 'false', r''' + make updates from the current transaction visible to users of the + named snapshot. Transactions started with such a named snapshot are + restricted to being read-only''', type='boolean'), Config('name', '', r'''specify a name for the snapshot'''), ]), diff --git a/src/third_party/wiredtiger/dist/api_err.py b/src/third_party/wiredtiger/dist/api_err.py index 82f961a4ac9..bd379ac8d70 100644 --- a/src/third_party/wiredtiger/dist/api_err.py +++ b/src/third_party/wiredtiger/dist/api_err.py @@ -82,7 +82,7 @@ for line in open('../src/include/wiredtiger.in', 'r'): ''.join('\n * ' + l for l in textwrap.wrap( textwrap.dedent(err.long_desc).strip(), 77)) + '\n' if err.long_desc else '')) - tfile.write('#define\t%s\t%d\n' % (err.name, err.value)) + tfile.write('#define\t%s\t(%d)\n' % (err.name, err.value)) if 'undoc' in err.flags: tfile.write('/*! @endcond */\n') tfile.write('/*\n') diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index fe9a17b7799..5a3348b940a 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -30,6 +30,7 @@ src/btree/bt_io.c src/btree/bt_misc.c src/btree/bt_ovfl.c src/btree/bt_page.c +src/btree/bt_random.c src/btree/bt_read.c src/btree/bt_rebalance.c src/btree/bt_ret.c @@ -132,6 +133,7 @@ src/os_posix/os_path.c POSIX_HOST src/os_posix/os_priv.c POSIX_HOST src/os_posix/os_setvbuf.c POSIX_HOST src/os_posix/os_sleep.c POSIX_HOST +src/os_posix/os_snprintf.c POSIX_HOST src/os_posix/os_thread.c POSIX_HOST src/os_posix/os_time.c POSIX_HOST src/os_posix/os_yield.c POSIX_HOST @@ -151,7 +153,6 @@ src/os_win/os_snprintf.c WINDOWS_HOST src/os_win/os_thread.c WINDOWS_HOST src/os_win/os_time.c WINDOWS_HOST src/os_win/os_utf8.c WINDOWS_HOST -src/os_win/os_vsnprintf.c WINDOWS_HOST src/os_win/os_winerr.c WINDOWS_HOST src/os_win/os_yield.c WINDOWS_HOST src/packing/pack_api.c @@ -159,6 +160,7 @@ src/packing/pack_impl.c src/packing/pack_stream.c src/reconcile/rec_track.c src/reconcile/rec_write.c +src/schema/schema_alter.c src/schema/schema_create.c src/schema/schema_drop.c src/schema/schema_list.c diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index e200f95fba6..64b5d789e72 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -64,6 +64,7 @@ flags = { 'VERB_COMPACT', 'VERB_EVICT', 'VERB_EVICTSERVER', + 'VERB_EVICT_STUCK', 'VERB_FILEOPS', 'VERB_HANDLEOPS', 'VERB_LOG', @@ -76,6 +77,7 @@ flags = { 'VERB_REBALANCE', 'VERB_RECONCILE', 'VERB_RECOVERY', + 'VERB_RECOVERY_PROGRESS', 'VERB_SALVAGE', 'VERB_SHARED_CACHE', 'VERB_SPLIT', @@ -94,19 +96,19 @@ flags = { 'CONN_CACHE_POOL', 'CONN_CKPT_SYNC', 'CONN_CLOSING', + 'CONN_CLOSING_NO_MORE_OPENS', 'CONN_EVICTION_RUN', 'CONN_IN_MEMORY', 'CONN_LAS_OPEN', 'CONN_LEAK_MEMORY', - 'CONN_LOG_SERVER_RUN', 'CONN_LSM_MERGE', 'CONN_PANIC', 'CONN_READONLY', 'CONN_RECOVERING', 'CONN_SERVER_ASYNC', 'CONN_SERVER_CHECKPOINT', + 'CONN_SERVER_LOG', 'CONN_SERVER_LSM', - 'CONN_SERVER_RUN', 'CONN_SERVER_STATISTICS', 'CONN_SERVER_SWEEP', 'CONN_WAS_BACKUP', @@ -114,14 +116,15 @@ flags = { 'session' : [ 'SESSION_CAN_WAIT', 'SESSION_INTERNAL', - 'SESSION_LOCK_NO_WAIT', 'SESSION_LOCKED_CHECKPOINT', - 'SESSION_LOCKED_HANDLE_LIST', + 'SESSION_LOCKED_HANDLE_LIST_READ', + 'SESSION_LOCKED_HANDLE_LIST_WRITE', 'SESSION_LOCKED_METADATA', 'SESSION_LOCKED_PASS', 'SESSION_LOCKED_SCHEMA', 'SESSION_LOCKED_SLOT', - 'SESSION_LOCKED_TABLE', + 'SESSION_LOCKED_TABLE_READ', + 'SESSION_LOCKED_TABLE_WRITE', 'SESSION_LOCKED_TURTLE', 'SESSION_LOGGING_INMEM', 'SESSION_LOOKASIDE_CURSOR', diff --git a/src/third_party/wiredtiger/dist/package/wiredtiger.spec b/src/third_party/wiredtiger/dist/package/wiredtiger.spec index 365c330d86a..aacdf327c98 100644 --- a/src/third_party/wiredtiger/dist/package/wiredtiger.spec +++ b/src/third_party/wiredtiger/dist/package/wiredtiger.spec @@ -1,5 +1,5 @@ Name: wiredtiger -Version: 2.8.1 +Version: 2.9.2 Release: 1%{?dist} Summary: WiredTiger data storage engine diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index 31abab28910..be33657e640 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -57,7 +57,7 @@ errchk() # Some tests shouldn't return an error, we exclude them here. case "$1" in *s_export|*s_tags) - break;; + ;; *) errfound=1;; esac @@ -84,22 +84,23 @@ run "sh ./s_copyright" run "sh ./s_style" COMMANDS=" -2>&1 ./s_tags > ${t_pfx}tags 2>&1 ./s_define > ${t_pfx}s_define -2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c -2>&1 ./s_funcs > ${t_pfx}s_funcs +2>&1 ./s_docs > ${t_pfx}s_docs 2>&1 ./s_export > ${t_pfx}s_export +2>&1 ./s_funcs > ${t_pfx}s_funcs 2>&1 ./s_getopt > ${t_pfx}s_getopt 2>&1 ./s_label > ${t_pfx}s_label 2>&1 ./s_lang > ${t_pfx}s_lang 2>&1 ./s_longlines > ${t_pfx}s_longlines +2>&1 ./s_python > ${t_pfx}s_python 2>&1 ./s_stat > ${t_pfx}_stat 2>&1 ./s_string > ${t_pfx}s_string -2>&1 python style.py > ${t_pfx}py_style -2>&1 ./s_python > ${t_pfx}s_python +2>&1 ./s_tags > ${t_pfx}tags +2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c +2>&1 ./s_void > ${t_pfx}s_void 2>&1 ./s_whitespace > ${t_pfx}s_whitespace 2>&1 ./s_win > ${t_pfx}s_win -2>&1 ./s_docs > ${t_pfx}s_docs" +2>&1 python style.py > ${t_pfx}py_style" # Parallelize if possible. xp="" @@ -110,14 +111,13 @@ fi echo "$COMMANDS" | xargs $xp -I{} /bin/sh -c {} for f in `find . -name ${t_pfx}\*`; do - if ! `test -s $f`; then - continue + if `test -s $f`; then + LOCAL_NAME=`basename $f` + # Find original command and trim redirect garbage + FAILED_CMD=`echo "$COMMANDS" | grep $LOCAL_NAME | \ + sed -e 's/ >.*//' -e 's/.* //'` + errchk "$FAILED_CMD" $f fi - LOCAL_NAME=`basename $f` - # Find original command and trim redirect garbage - FAILED_CMD=`echo "$COMMANDS" | grep $LOCAL_NAME | \ - sed -e 's/ >.*//' -e 's/.* //'` - errchk "$FAILED_CMD" $f done echo 'dist/s_all run finished' diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 7b11d665de5..8911d888077 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -1,5 +1,4 @@ # List of WiredTiger #defines that are "unused", but it's OK. -ALIGN_CHECK API_CALL API_CALL_NOCONF API_SESSION_INIT @@ -8,17 +7,18 @@ JOINABLE_CURSOR_CALL_CHECK LF_MASK LLONG_MAX LLONG_MIN -SIZE_CHECK TXN_API_CALL TXN_API_CALL_NOCONF TXN_API_END WIN32_LEAN_AND_MEAN +WT_ALIGN_CHECK WT_ATOMIC_CAS WT_ATOMIC_FUNC WT_BLOCK_DESC_SIZE WT_BLOCK_HEADER_SIZE WT_CACHE_LINE_ALIGNMENT -WT_COMPILER_TYPE_ALIGN +WT_CACHE_LINE_PAD_BEGIN +WT_CACHE_LINE_PAD_END WT_CONN_CHECK_PANIC WT_DEADLOCK WT_DEBUG_BYTE @@ -35,10 +35,14 @@ WT_LOG_SLOT_MAXBITS WT_LOG_SLOT_UNBUFFERED_ISSET WT_PACKED_STRUCT_BEGIN WT_PACKED_STRUCT_END +WT_PADDING_CHECK WT_READ_BARRIER WT_REF_SIZE WT_SESSION_LOCKED_CHECKPOINT +WT_SESSION_LOCKED_TABLE_READ +WT_SESSION_LOCKED_TABLE_WRITE WT_SESSION_LOCKED_TURTLE +WT_SIZE_CHECK WT_STATS_FIELD_TO_OFFSET WT_STATS_SLOT_ID WT_STAT_CONN_DECRV @@ -54,7 +58,7 @@ WT_STAT_INCRV_BASE WT_STAT_WRITE WT_TIMEDIFF_US WT_TRET_ERROR_OK -WT_WITH_LOCK +WT_WITH_LOCK_NOWAIT WT_WITH_LOCK_WAIT __F __WIREDTIGER_EXT_H_ diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index f4332257193..6ebffb947ec 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -96,7 +96,8 @@ spellchk() type aspell > /dev/null 2>&1 || return (cd ../src/docs && - cat *.dox | aspell --lang=en --personal=./spell.ok list) | + cat *.dox | + aspell --encoding=iso-8859-1 --lang=en --personal=./spell.ok list) | sort -u > $t test -s $t && { echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index 01835390997..b73767cad13 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -13,7 +13,6 @@ __wt_bloom_get __wt_bulk_insert_fix __wt_bulk_insert_row __wt_bulk_insert_var -__wt_cache_dump __wt_config_getone __wt_cursor_get_raw_value __wt_debug_addr diff --git a/src/third_party/wiredtiger/dist/s_prototypes b/src/third_party/wiredtiger/dist/s_prototypes index 4c07b9aa160..9675cd5a843 100755 --- a/src/third_party/wiredtiger/dist/s_prototypes +++ b/src/third_party/wiredtiger/dist/s_prototypes @@ -39,7 +39,7 @@ proto() -e H \ -e x \ -e '}' \ - -e '# Add the gcc warn_unused_result attribute to any external' \ + -e '# Add the warn_unused_result attribute to any external' \ -e '# functions that return an int.' \ -e '/^extern int /s/$/ WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result))/' \ -e 's/$/;/' \ diff --git a/src/third_party/wiredtiger/dist/s_python b/src/third_party/wiredtiger/dist/s_python index b8aa5848637..bfe3ba57783 100755 --- a/src/third_party/wiredtiger/dist/s_python +++ b/src/third_party/wiredtiger/dist/s_python @@ -17,10 +17,10 @@ test -s $t && { echo '[tab] characters appear in Python scripts:' cat $t } -# Check Python coding standards: check for trailing semi-colons. +# Check Python coding standards: check for uncommented trailing semi-colons. # Don't check too widely, there are third-party tools that fail this test as # well as scripts in this directory that output C code, and so fail the test. -egrep ';$' `find lang test -name '*.py'`> $t +egrep '^[^#]*;$' `find lang test -name '*.py'`> $t test -s $t && { echo 'trailing semi-colons in selected Python code:' cat $t diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 5d5937e1833..6aeeca6faa6 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -25,9 +25,6 @@ cat << UNUSED_STAT_FIELDS lock_checkpoint_count lock_checkpoint_wait_application lock_checkpoint_wait_internal -lock_handle_list_count -lock_handle_list_wait_application -lock_handle_list_wait_internal lock_metadata_count lock_metadata_wait_application lock_metadata_wait_internal diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d19c05e802b..1f7f7d9fd3a 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -21,6 +21,7 @@ Alakuijala Alexandrescu's Alloc Async +AsyncOp Athanassoulis Athlon BBBBB @@ -38,6 +39,7 @@ BTREE BUF BZ Barack +BerkeleyDB Bitfield Bitwise Brueckner @@ -61,10 +63,12 @@ CPUs CRC CSV CStream +CURFILE CURSORs CURSTD CallsCustDate Castagnoli +CentOS Checkpointing Checksum Checksums @@ -88,6 +92,7 @@ DNE DOI DONTNEED DUPLICATEV +DataSet DbCursor DbEnv Decrement @@ -126,6 +131,7 @@ FNV FORALL FOREACH FS +FTRUNCATE FULLFSYNC Facebook FindClose @@ -156,6 +162,7 @@ HHHHLL HHHLL Hendrik HyperLevelDB +ID's IEC IEEE IKEY @@ -167,6 +174,7 @@ INITIALIZER INMEM INSN INTL +INULL ISA ITEMs ITER @@ -217,8 +225,10 @@ MEMALIGN MERCHANTABILITY METADATA MONGODB +MOVEFILE MRXB MRXBOPC +MSDN MSVC MULTI MULTIBLOCK @@ -232,8 +242,7 @@ Metadata Mewhort Mitzenmacher MongoDB -MoveFile -MoveFileW +MoveFileExW Multi MultiByteToWideChar Multithreaded @@ -247,6 +256,7 @@ NONINFRINGEMENT NOTFOUND NOTREACHED NOVALUE +NOWAIT NUL NUM NetBSD @@ -269,6 +279,8 @@ PRIu PRNG PTHREAD PTR +PackInputStream +PackOutputStream Pandis Phong PlatformSDK @@ -300,6 +312,7 @@ Readonly Rebalance RedHat Redistributions +Refactor Resize RocksDB SIMD @@ -331,6 +344,7 @@ Split's Stoica StoreLoad StoreStore +Syscall TAILQ TCMalloc TESTUTIL @@ -397,14 +411,18 @@ WiredTigerPreplog WiredTigerTmplog WiredTigerTxn WithSeeds +Wpedantic WriteFile Wuninitialized Wunused XP +Xcode +Xcode/ Yann ZSTD Zlib Zlib's +Zstandard Zstd Zstd's abcdef @@ -428,6 +446,7 @@ argc args argv asm +aspell assertfmt async asyncopp @@ -547,6 +566,7 @@ create's createCStream crypto cryptobad +csuite csv ctime ctype @@ -598,6 +618,7 @@ dequeue dequeued der dereference +dereferenced desc designator dest @@ -610,6 +631,7 @@ difftime dir dirlist disjunction +disjunctions dlclose dlh dll @@ -633,6 +655,7 @@ dumpcmp dumpfile dup eg +egrep emp encodings encryptor @@ -723,6 +746,7 @@ gettime gettimeofday getv github +gitignore gobare goesc gostring @@ -749,6 +773,7 @@ idx ifdef ifdef's iiSii +iiiS iiii iiu ikey @@ -834,6 +859,7 @@ libdatasource libs libtool libwiredtiger +linkers llll llu loadtext @@ -935,6 +961,7 @@ nowait nset nsnap nul +nullptr nuls numSymbols numbare @@ -948,6 +975,7 @@ oo opcode opendir openfile +oplog optimizations optype ori @@ -969,6 +997,7 @@ pclose pcpu perf pfx +pluggable popen poptable popthreads @@ -1113,12 +1142,15 @@ subgetraw subgets subinit sublicense +subtest subtree sunique superset sw +sx sy sys +syscall sz t's tV @@ -1156,6 +1188,7 @@ txt typedef uB uS +ui uint uintmax unbare @@ -1191,6 +1224,7 @@ upg uri uri's uris +usec usecs usedp userbad @@ -1221,10 +1255,14 @@ vunpack vw vxr waitpid +waker +wakeup +wakeups walk's warmup wb wcursor +whitespace wiredTiger wiredtiger workFactor @@ -1238,6 +1276,7 @@ ws wtbench wti wtperf +wtperf's wts wtstats xF @@ -1250,6 +1289,7 @@ zf zfree zlib zlib's +zseries zstd zstd's zu diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 8e755224ee2..69cf1f667fa 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -93,10 +93,21 @@ else cat $t fi - # Alignment directive before "struct". - egrep 'WT_COMPILER_TYPE_ALIGN.*struct' $f > $t - test -s $t && { - echo "$f: compiler alignment direction must precede \"struct\"" + if ! expr "$f" : 'examples/c/*' > /dev/null && + ! expr "$f" : 'ext/*' > /dev/null && + ! expr "$f" : 'src/os_posix/os_snprintf.c' > /dev/null && + egrep '[^a-z_]snprintf\(|[^a-z_]vsnprintf\(' $f > $t; then + echo "$f: snprintf call, use WiredTiger library replacements" + cat $t + fi + + # If we don't have matching pack-begin and pack-end calls, we don't get + # an error, we just get a Windows performance regression. Using awk and + # not wc to ensure there's no whitespace in the assignment. + egrep WT_PACKED_STRUCT $f > $t + cnt=`awk 'BEGIN { line = 0 } { ++line } END { print line }' < $t` + test `expr "$cnt" % 2` -ne 0 && { + echo "$f: mismatched WT_PACKED_STRUCT_BEGIN/END lines" cat $t } diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void old mode 100644 new mode 100755 index b9c3f5e0ef7..249f043d029 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -5,12 +5,14 @@ trap 'rm -f $t' 0 1 2 3 13 15 cd .. -# Turn a C file into a line per function that returns an int. +# Parse a C file, discarding functions that don't return an int, and formatting +# the remaining functions as a single line. file_parse() { sed -n \ -e '/^int$/b loop' \ -e '/^static int$/b loop' \ + -e '/^static inline int$/b loop' \ -e 'd' \ -e ': loop' \ -e 'H' \ @@ -34,6 +36,7 @@ func_ok() -e '/int __compact_uri_analyze$/d' \ -e '/int __config_parser_close$/d' \ -e '/int __curlog_reset$/d' \ + -e '/int __cursor_fix_implicit$/d' \ -e '/int __handle_close_default$/d' \ -e '/int __handle_progress_default$/d' \ -e '/int __im_file_close$/d' \ @@ -42,7 +45,7 @@ func_ok() -e '/int __im_file_sync$/d' \ -e '/int __im_fs_directory_list_free$/d' \ -e '/int __im_fs_exist$/d' \ - -e '/int __posix_file_close$/d' \ + -e '/int __page_write_gen_wrapped_check$/d' \ -e '/int __posix_terminate$/d' \ -e '/int __rec_destroy_session$/d' \ -e '/int __win_terminate$/d' \ @@ -59,11 +62,14 @@ func_ok() -e '/int __wt_lsm_manager_pop_entry$/d' \ -e '/int __wt_once$/d' \ -e '/int __wt_posix_directory_list_free$/d' \ + -e '/int __wt_spin_init$/d' \ + -e '/int __wt_spin_trylock$/d' \ -e '/int __wt_stat_connection_desc$/d' \ -e '/int __wt_stat_dsrc_desc$/d' \ -e '/int __wt_stat_join_desc$/d' \ -e '/int __wt_win_directory_list_free$/d' \ -e '/int bdb_compare_reverse$/d' \ + -e '/int copyout_val$/d' \ -e '/int csv_error$/d' \ -e '/int csv_terminate$/d' \ -e '/int demo_file_close$/d' \ @@ -72,10 +78,20 @@ func_ok() -e '/int demo_file_sync$/d' \ -e '/int demo_fs_directory_list_free$/d' \ -e '/int demo_fs_exist$/d' \ + -e '/int fail_file_lock$/d' \ + -e '/int fail_file_sync$/d' \ + -e '/int fail_fs_directory_list_free$/d' \ + -e '/int fail_fs_exist$/d' \ + -e '/int fail_fs_simulate_fail$/d' \ + -e '/int fail_fs_terminate$/d' \ -e '/int handle_message$/d' \ -e '/int handle_progress$/d' \ -e '/int helium_cursor_reset$/d' \ -e '/int helium_session_verify$/d' \ + -e '/int index_compare_primary$/d' \ + -e '/int index_compare_S$/d' \ + -e '/int index_compare_u$/d' \ + -e '/int index_extractor_u$/d' \ -e '/int log_print_err$/d' \ -e '/int lz4_error$/d' \ -e '/int lz4_pre_size$/d' \ @@ -108,25 +124,29 @@ func_ok() -e '/int zstd_terminate$/d' } -# Complain about functions which return an "int" but which don't return except -# at the end of the function. This script is a kluge and isn't run by default. for f in `find bench ext src test -name '*.[ci]'`; do if expr "$f" : '.*/windows_shim.c' > /dev/null; then continue fi + # Complain about functions which return an "int" but which don't return + # except at the end of the function. + # # Turn each function into a single line, then discard the function's # final "return" call, then discard any function that still has some # form of return assignment or call. file_parse $f | sed -e 's/return ([^)]*); }$//' \ - -e '/[A-Z]*_API_CALL[A-Z_]*(/d' \ + -e '/[_A-Z]*_API_CALL[_A-Z]*(/d' \ -e '/WT_CURSOR_NEEDKEY(/d' \ -e '/WT_CURSOR_NEEDVALUE(/d' \ -e '/WT_ERR[A-Z_]*(/d' \ -e '/WT_ILLEGAL_VALUE[A-Z_]*(/d' \ -e '/WT_PANIC[A-Z_]*(/d' \ -e '/WT_RET[A-Z_]*(/d' \ + -e '/WT_SIZE_CHECK_PACK(/d' \ + -e '/WT_SIZE_CHECK_UNPACK(/d' \ + -e '/WT_SYSCALL(/d' \ -e '/WT_TRET(/d' \ -e '/[^a-z_]ret = /d' \ -e '/[^a-z_]return (/d' \ @@ -134,8 +154,34 @@ for f in `find bench ext src test -name '*.[ci]'`; do -e 's/^ *//' | func_ok > $t test -s $t && { - echo "========== $f" - cat $t + echo "==============================================" + echo "$f:" + cat $t | sed 's/^/ /' + echo "Function could return void instead of int." + echo "Add false positives to the list in dist/s_void." + echo "==============================================" + } + + # Complain about functions which declare a "ret" value but never use it. + file_parse $f | + grep 'WT_DECL_RET' | + sed -e '/ret =/d' \ + -e '/[_A-Z]*_API_CALL[_A-Z]*(/d' \ + -e '/WT_CURSOR_NEEDKEY/d' \ + -e '/WT_CURSOR_NEEDVALUE/d' \ + -e '/WT_ERR/d' \ + -e '/WT_ILLEGAL_VALUE_ERR/d' \ + -e '/WT_SYSCALL.*ret/d' \ + -e '/WT_TRET/d' \ + -e 's/^\([^(]*\).*/\1/' \ + -e 's/^ *//' > $t + test -s $t && { + echo "==============================================" + echo "$f:" + cat $t | sed 's/^/ /' + echo "Function declares ret without using it." + echo "Add false positives to the list in dist/s_void." + echo "==============================================" } done diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index bcf5201bd90..ac79ffd029a 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -193,6 +193,7 @@ connection_stats = [ CacheStat('cache_bytes_other', 'bytes not belonging to page images in the cache', 'no_clear,no_scale,size'), CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), + CacheStat('cache_eviction_active_workers', 'eviction worker thread active', 'no_clear'), CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), CacheStat('cache_eviction_app_dirty', 'modified pages evicted by application threads'), @@ -205,6 +206,7 @@ connection_stats = [ CacheStat('cache_eviction_force', 'pages evicted because they exceeded the in-memory maximum'), CacheStat('cache_eviction_force_delete', 'pages evicted because they had chains of deleted items'), CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum'), + CacheStat('cache_eviction_force_retune', 'force re-tuning of eviction workers once in a while'), CacheStat('cache_eviction_get_ref', 'eviction calls to get a page'), CacheStat('cache_eviction_get_ref_empty', 'eviction calls to get a page found queue empty'), CacheStat('cache_eviction_get_ref_empty2', 'eviction calls to get a page found queue empty after locking'), @@ -222,12 +224,15 @@ connection_stats = [ CacheStat('cache_eviction_slow', 'eviction server unable to reach eviction goal'), CacheStat('cache_eviction_split_internal', 'internal pages split during eviction'), CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'), + CacheStat('cache_eviction_stable_state_workers', 'eviction worker thread stable number', 'no_clear'), CacheStat('cache_eviction_state', 'eviction state', 'no_clear,no_scale'), CacheStat('cache_eviction_walk', 'pages walked for eviction'), CacheStat('cache_eviction_walks_abandoned', 'eviction walks abandoned'), CacheStat('cache_eviction_walks_active', 'files with active eviction walks', 'no_clear,no_scale'), CacheStat('cache_eviction_walks_started', 'files with new eviction walks started'), CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'), + CacheStat('cache_eviction_worker_created', 'eviction worker thread created'), + CacheStat('cache_eviction_worker_removed', 'eviction worker thread removed'), CacheStat('cache_hazard_checks', 'hazard pointer check calls'), CacheStat('cache_hazard_max', 'hazard pointer maximum array length', 'max_aggregate,no_scale'), CacheStat('cache_hazard_walks', 'hazard pointer check entries walked'), @@ -284,9 +289,7 @@ connection_stats = [ LockStat('lock_checkpoint_count', 'checkpoint lock acquisitions'), LockStat('lock_checkpoint_wait_application', 'checkpoint lock application thread wait time (usecs)'), LockStat('lock_checkpoint_wait_internal', 'checkpoint lock internal thread wait time (usecs)'), - LockStat('lock_handle_list_count', 'handle-list lock acquisitions'), - LockStat('lock_handle_list_wait_application', 'handle-list lock application thread wait time (usecs)'), - LockStat('lock_handle_list_wait_internal', 'handle-list lock internal thread wait time (usecs)'), + LockStat('lock_handle_list_wait_eviction', 'handle-list lock eviction thread wait time (usecs)'), LockStat('lock_metadata_count', 'metadata lock acquisitions'), LockStat('lock_metadata_wait_application', 'metadata lock application thread wait time (usecs)'), LockStat('lock_metadata_wait_internal', 'metadata lock internal thread wait time (usecs)'), @@ -321,10 +324,12 @@ connection_stats = [ LogStat('log_scan_records', 'records processed by log scan'), LogStat('log_scan_rereads', 'log scan records requiring two reads'), LogStat('log_scans', 'log scan operations'), + LogStat('log_slot_active_closed', 'consolidated slot join active slot closed'), LogStat('log_slot_closes', 'consolidated slot closures'), LogStat('log_slot_coalesced', 'written slots coalesced'), LogStat('log_slot_consolidated', 'logging bytes consolidated', 'size'), LogStat('log_slot_joins', 'consolidated slot joins'), + LogStat('log_slot_no_free_slots', 'consolidated slot transitions unable to find free slot'), LogStat('log_slot_races', 'consolidated slot join races'), LogStat('log_slot_switch_busy', 'busy returns attempting to switch slots'), LogStat('log_slot_transitions', 'consolidated slot join transitions'), @@ -367,6 +372,9 @@ connection_stats = [ ########################################## SessionStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), SessionStat('session_open', 'open session count', 'no_clear,no_scale'), + SessionStat('session_table_alter_fail', 'table alter failed calls', 'no_clear,no_scale'), + SessionStat('session_table_alter_skip', 'table alter unchanged and skipped', 'no_clear,no_scale'), + SessionStat('session_table_alter_success', 'table alter successful calls', 'no_clear,no_scale'), SessionStat('session_table_compact_fail', 'table compact failed calls', 'no_clear,no_scale'), SessionStat('session_table_compact_success', 'table compact successful calls', 'no_clear,no_scale'), SessionStat('session_table_create_fail', 'table create failed calls', 'no_clear,no_scale'), @@ -474,6 +482,7 @@ dsrc_stats = [ ########################################## # Cache and eviction statistics ########################################## + CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'), CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'), CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index d5305eec5c8..20936661e06 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -20,7 +20,6 @@ noinst_PROGRAMS = \ ex_pack \ ex_process \ ex_schema \ - ex_scope \ ex_stat \ ex_sync \ ex_thread diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index ea646604a76..82620673fe1 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -557,6 +557,12 @@ session_ops(WT_SESSION *session) /*! [Create a column-store table] */ ret = session->create(session, "table:mytable", "key_format=r,value_format=S"); + + /*! [Alter a table] */ + ret = session->alter(session, + "table:mytable", "access_pattern_hint=random"); + /*! [Alter a table] */ + /*! [Create a column-store table] */ ret = session->drop(session, "table:mytable", NULL); @@ -842,8 +848,8 @@ my_compare(WT_COLLATOR *collator, WT_SESSION *session, p1 = (const char *)value1->data; p2 = (const char *)value2->data; - while (*p1 != '\0' && *p1 == *p2) - p1++, p2++; + for (; *p1 != '\0' && *p1 == *p2; ++p1, ++p2) + ; *cmp = (int)*p2 - (int)*p1; return (0); diff --git a/src/third_party/wiredtiger/examples/c/ex_async.c b/src/third_party/wiredtiger/examples/c/ex_async.c index f7531a5c3d8..5cfafca0418 100644 --- a/src/third_party/wiredtiger/examples/c/ex_async.c +++ b/src/third_party/wiredtiger/examples/c/ex_async.c @@ -170,12 +170,12 @@ main(void) * an asynchronous insert. */ /*! [async set the operation's string key] */ - snprintf(k[i], sizeof(k), "key%d", i); + (void)snprintf(k[i], sizeof(k), "key%d", i); op->set_key(op, k[i]); /*! [async set the operation's string key] */ /*! [async set the operation's string value] */ - snprintf(v[i], sizeof(v), "value%d", i); + (void)snprintf(v[i], sizeof(v), "value%d", i); op->set_value(op, v[i]); /*! [async set the operation's string value] */ @@ -218,7 +218,7 @@ main(void) * Set the operation's string key and value, and then do * an asynchronous search. */ - snprintf(k[i], sizeof(k), "key%d", i); + (void)snprintf(k[i], sizeof(k), "key%d", i); op->set_key(op, k[i]); ret = op->search(op); /*! [async search] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_backup.c b/src/third_party/wiredtiger/examples/c/ex_backup.c index 0697cbb3458..83cc9b22ecc 100644 --- a/src/third_party/wiredtiger/examples/c/ex_backup.c +++ b/src/third_party/wiredtiger/examples/c/ex_backup.c @@ -96,7 +96,7 @@ compare_backups(int i) if (i == 0) (void)strncpy(msg, "MAIN", sizeof(msg)); else - snprintf(msg, sizeof(msg), "%d", i); + (void)snprintf(msg, sizeof(msg), "%d", i); printf( "Iteration %s: Tables %s.%d and %s.%d %s\n", msg, full_out, i, incr_out, i, ret == 0 ? "identical" : "differ"); @@ -131,8 +131,8 @@ setup_directories(void) * For incremental backups we need 0-N. The 0 incremental * directory will compare with the original at the end. */ - snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", - home_incr, i, home_incr, i); + (void)snprintf(buf, sizeof(buf), + "rm -rf %s.%d && mkdir %s.%d", home_incr, i, home_incr, i); if ((ret = system(buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", buf, ret); return (ret); @@ -142,8 +142,8 @@ setup_directories(void) /* * For full backups we need 1-N. */ - snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", - home_full, i, home_full, i); + (void)snprintf(buf, sizeof(buf), + "rm -rf %s.%d && mkdir %s.%d", home_full, i, home_full, i); if ((ret = system(buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", buf, ret); return (ret); @@ -164,8 +164,8 @@ add_work(WT_SESSION *session, int iter) * Perform some operations with individual auto-commit transactions. */ for (i = 0; i < MAX_KEYS; i++) { - snprintf(k, sizeof(k), "key.%d.%d", iter, i); - snprintf(v, sizeof(v), "value.%d.%d", iter, i); + (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i); + (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -187,7 +187,7 @@ take_full_backup(WT_SESSION *session, int i) * directories. Otherwise only into the appropriate full directory. */ if (i != 0) { - snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); hdir = h; } else hdir = home_incr; @@ -200,14 +200,15 @@ take_full_backup(WT_SESSION *session, int i) * Take a full backup into each incremental directory. */ for (j = 0; j < MAX_ITERATIONS; j++) { - snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(h, sizeof(h), + "%s.%d", home_incr, j); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); ret = system(buf); } else { - snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename); ret = system(buf); @@ -237,12 +238,12 @@ take_incr_backup(WT_SESSION *session, int i) * Copy into the 0 incremental directory and then each of the * incremental directories for this iteration and later. */ - snprintf(h, sizeof(h), "%s.0", home_incr); + (void)snprintf(h, sizeof(h), "%s.0", home_incr); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); ret = system(buf); for (j = i; j < MAX_ITERATIONS; j++) { - snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); ret = system(buf); @@ -270,7 +271,8 @@ main(void) int i, ret; char cmd_buf[256]; - snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", home, home); + (void)snprintf(cmd_buf, sizeof(cmd_buf), + "rm -rf %s && mkdir %s", home, home); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (EXIT_FAILURE); diff --git a/src/third_party/wiredtiger/examples/c/ex_data_source.c b/src/third_party/wiredtiger/examples/c/ex_data_source.c index 6ed80dfcf19..387248f6ae2 100644 --- a/src/third_party/wiredtiger/examples/c/ex_data_source.c +++ b/src/third_party/wiredtiger/examples/c/ex_data_source.c @@ -46,6 +46,21 @@ my_data_source_init(WT_CONNECTION *connection) } /*! [WT_EXTENSION_API declaration] */ +/*! [WT_DATA_SOURCE alter] */ +static int +my_alter(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, WT_CONFIG_ARG *config) +/*! [WT_DATA_SOURCE alter] */ +{ + /* Unused parameters */ + (void)dsrc; + (void)session; + (void)uri; + (void)config; + + return (0); +} + /*! [WT_DATA_SOURCE create] */ static int my_create(WT_DATA_SOURCE *dsrc, WT_SESSION *session, @@ -604,6 +619,7 @@ main(void) { /*! [WT_DATA_SOURCE register] */ static WT_DATA_SOURCE my_dsrc = { + my_alter, my_create, my_compact, my_drop, diff --git a/src/third_party/wiredtiger/examples/c/ex_encrypt.c b/src/third_party/wiredtiger/examples/c/ex_encrypt.c index 00dc66fc24d..1520bd286cd 100644 --- a/src/third_party/wiredtiger/examples/c/ex_encrypt.c +++ b/src/third_party/wiredtiger/examples/c/ex_encrypt.c @@ -507,12 +507,12 @@ main(void) * we decrypt on read. */ for (i = 0; i < MAX_KEYS; i++) { - snprintf(keybuf, sizeof(keybuf), "key%d", i); + (void)snprintf(keybuf, sizeof(keybuf), "key%d", i); c1->set_key(c1, keybuf); c2->set_key(c2, keybuf); nc->set_key(nc, keybuf); - snprintf(valbuf, sizeof(valbuf), "value%d", i); + (void)snprintf(valbuf, sizeof(valbuf), "value%d", i); c1->set_value(c1, valbuf); c2->set_value(c2, valbuf); nc->set_value(nc, valbuf); diff --git a/src/third_party/wiredtiger/examples/c/ex_file_system.c b/src/third_party/wiredtiger/examples/c/ex_file_system.c index 524a5d03f89..e807ac54d3b 100644 --- a/src/third_party/wiredtiger/examples/c/ex_file_system.c +++ b/src/third_party/wiredtiger/examples/c/ex_file_system.c @@ -29,35 +29,49 @@ * demonstrates how to use the custom file system interface */ -/* - * Include WiredTiger internal functions: we need architecture portable locking - * in this example, and we use the TAILQ_XXX functions to keep the code simple. - * - * Application-writers SHOULD NOT INCLUDE "wt_internal.h", the public WiredTiger - * include files should be used instead: - * - * #include - * #include - */ -#include "wt_internal.h" +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#else +#include "windows_shim.h" +#endif + +#include +#include /* - * This example code uses internal WiredTiger functions for portable locking. - * We use #defines to clarify the meaning and ignore errors to simplify the - * code. - * - * Application writers SHOULD NOT COPY THIS LOCKING CODE, it's special-case code - * to make this example portable across platforms. + * This example code uses pthread functions for portable locking, we ignore + * errors for simplicity. */ -#define ALLOCATE_FILE_SYSTEM_LOCK(demo_fs) \ - (void)__wt_spin_init(NULL, &(demo_fs)->lock, "demo file handle lock") -#define DESTROY_FILE_SYSTEM_LOCK(wt_session, demo_fs) \ - __wt_spin_destroy((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) -#define LOCK_FILE_SYSTEM(wt_session, demo_fs) \ - __wt_spin_lock((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) -#define UNLOCK_FILE_SYSTEM(wt_session, demo_fs) \ - __wt_spin_unlock( \ - (WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) +static void +allocate_file_system_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_init(lockp, NULL) == 0); +} + +static void +destroy_file_system_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_destroy(lockp) == 0); +} + +static void +lock_file_system(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_wrlock(lockp) == 0); +} + +static void +unlock_file_system(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_unlock(lockp) == 0); +} /* * Example file system implementation, using memory buffers to represent files. @@ -72,7 +86,7 @@ typedef struct { * might require finer granularity, for example, a single lock for the * file system handle list and per-handle locks serializing I/O. */ - WT_SPINLOCK lock; /* Lock */ + pthread_rwlock_t lock; /* Lock */ int opened_file_count; int opened_unique_file_count; @@ -194,19 +208,9 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) * the underlying filesystem implementation. See the main function for * the setup of those configuration strings; here we parse configuration * information as passed in by main, through WiredTiger. - * - * Retrieve our configuration information, the "config" value. */ - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { - (void)wtext->err_printf(wtext, NULL, - "WT_EXTENSION_API.config_get: config: %s", - wtext->strerror(wtext, NULL, ret)); - goto err; - } - - /* Open a WiredTiger parser on the "config" value. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) { + if ((ret = wtext->config_parser_open_arg( + wtext, NULL, config, &config_parser)) != 0) { (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_parser_open: config: %s", wtext->strerror(wtext, NULL, ret)); @@ -249,7 +253,7 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) goto err; } - ALLOCATE_FILE_SYSTEM_LOCK(demo_fs); + allocate_file_system_lock(&demo_fs->lock); /* Initialize the in-memory jump table. */ file_system->fs_directory_list = demo_fs_directory_list; @@ -298,7 +302,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, demo_fh = NULL; wtext = demo_fs->wtext; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); ++demo_fs->opened_file_count; /* @@ -318,7 +322,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, *file_handlep = (WT_FILE_HANDLE *)demo_fh; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -376,7 +380,7 @@ err: free(demo_fh->buf); free(demo_fh); } - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -395,6 +399,9 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, uint32_t allocated, count; int ret = 0; char *name, **entries; + void *p; + + (void)session; /* Unused */ demo_fs = (DEMO_FILE_SYSTEM *)file_system; @@ -406,7 +413,7 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, len = strlen(directory); prefix_len = prefix == NULL ? 0 : strlen(prefix); - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); TAILQ_FOREACH(demo_fh, &demo_fs->fileq, q) { name = demo_fh->iface.name; if (strncmp(name, directory, len) != 0 || @@ -418,14 +425,16 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, * matter if the list is a bit longer than necessary. */ if (count >= allocated) { - entries = realloc( - entries, (allocated + 10) * sizeof(char *)); - if (entries == NULL) { + p = realloc( + entries, (allocated + 10) * sizeof(*entries)); + if (p == NULL) { ret = ENOMEM; goto err; } - memset(entries + allocated * sizeof(char *), - 0, 10 * sizeof(char *)); + + entries = p; + memset(entries + allocated * sizeof(*entries), + 0, 10 * sizeof(*entries)); allocated += 10; } entries[count++] = strdup(name); @@ -434,7 +443,7 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, *dirlistp = entries; *countp = count; -err: UNLOCK_FILE_SYSTEM(session, demo_fs); +err: unlock_file_system(&demo_fs->lock); if (ret == 0) return (0); @@ -476,11 +485,13 @@ demo_fs_exist(WT_FILE_SYSTEM *file_system, { DEMO_FILE_SYSTEM *demo_fs; + (void)session; /* Unused */ + demo_fs = (DEMO_FILE_SYSTEM *)file_system; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); *existp = demo_handle_search(file_system, name) != NULL; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -497,15 +508,16 @@ demo_fs_remove(WT_FILE_SYSTEM *file_system, DEMO_FILE_HANDLE *demo_fh; int ret = 0; - (void)flags; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ demo_fs = (DEMO_FILE_SYSTEM *)file_system; ret = ENOENT; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((demo_fh = demo_handle_search(file_system, name)) != NULL) ret = demo_handle_remove(session, demo_fh); - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -523,11 +535,12 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system, char *copy; int ret = 0; - (void)flags; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ demo_fs = (DEMO_FILE_SYSTEM *)file_system; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((demo_fh = demo_handle_search(file_system, from)) == NULL) ret = ENOENT; else if ((copy = strdup(to)) == NULL) @@ -536,7 +549,7 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system, free(demo_fh->iface.name); demo_fh->iface.name = copy; } - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -555,10 +568,10 @@ demo_fs_size(WT_FILE_SYSTEM *file_system, demo_fs = (DEMO_FILE_SYSTEM *)file_system; ret = ENOENT; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((demo_fh = demo_handle_search(file_system, name)) != NULL) ret = demo_file_size((WT_FILE_HANDLE *)demo_fh, session, sizep); - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -588,7 +601,7 @@ demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) printf("\t%d reads, %d writes\n", demo_fs->read_ops, demo_fs->write_ops); - DESTROY_FILE_SYSTEM_LOCK(session, demo_fs); + destroy_file_system_lock(&demo_fs->lock); free(demo_fs); return (ret); @@ -604,13 +617,15 @@ demo_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) DEMO_FILE_HANDLE *demo_fh; DEMO_FILE_SYSTEM *demo_fs; + (void)session; /* Unused */ + demo_fh = (DEMO_FILE_HANDLE *)file_handle; demo_fs = demo_fh->demo_fs; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if (--demo_fh->ref == 0) ++demo_fs->closed_file_count; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -648,7 +663,7 @@ demo_file_read(WT_FILE_HANDLE *file_handle, wtext = demo_fs->wtext; off = (size_t)offset; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); ++demo_fs->read_ops; if (off < demo_fh->size) { if (len > demo_fh->size - off) @@ -656,7 +671,7 @@ demo_file_read(WT_FILE_HANDLE *file_handle, memcpy(buf, (uint8_t *)demo_fh->buf + off, len); } else ret = EIO; /* EOF */ - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); if (ret == 0) return (0); @@ -677,12 +692,14 @@ demo_file_size( DEMO_FILE_HANDLE *demo_fh; DEMO_FILE_SYSTEM *demo_fs; + (void)session; /* Unused */ + demo_fh = (DEMO_FILE_HANDLE *)file_handle; demo_fs = demo_fh->demo_fs; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); *sizep = (wt_off_t)demo_fh->size; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -749,10 +766,10 @@ demo_file_truncate( demo_fh = (DEMO_FILE_HANDLE *)file_handle; demo_fs = demo_fh->demo_fs; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((ret = demo_buffer_resize(session, demo_fh, offset)) == 0) demo_fh->size = (size_t)offset; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -775,7 +792,7 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wtext = demo_fs->wtext; off = (size_t)offset; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); ++demo_fs->write_ops; if ((ret = demo_buffer_resize(session, demo_fh, offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) == 0) { @@ -783,7 +800,7 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, if (off + len > demo_fh->size) demo_fh->size = off + len; } - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); if (ret == 0) return (0); diff --git a/src/third_party/wiredtiger/examples/c/ex_log.c b/src/third_party/wiredtiger/examples/c/ex_log.c index fdbc39412ae..0d8fbf97233 100644 --- a/src/third_party/wiredtiger/examples/c/ex_log.c +++ b/src/third_party/wiredtiger/examples/c/ex_log.c @@ -291,8 +291,8 @@ main(void) char cmd_buf[256], k[16], v[16]; count_min = 0; - snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s %s && mkdir %s %s", - home1, home2, home1, home2); + (void)snprintf(cmd_buf, sizeof(cmd_buf), + "rm -rf %s %s && mkdir %s %s", home1, home2, home1, home2); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (EXIT_FAILURE); @@ -312,8 +312,8 @@ main(void) * Perform some operations with individual auto-commit transactions. */ for (record_count = 0, i = 0; i < MAX_KEYS; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -324,8 +324,8 @@ main(void) * Perform some operations within a single transaction. */ for (i = MAX_KEYS; i < MAX_KEYS+5; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); diff --git a/src/third_party/wiredtiger/examples/c/ex_scope.c b/src/third_party/wiredtiger/examples/c/ex_scope.c deleted file mode 100644 index 795ad85d57b..00000000000 --- a/src/third_party/wiredtiger/examples/c/ex_scope.c +++ /dev/null @@ -1,217 +0,0 @@ -/*- - * Public Domain 2014-2016 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_scope.c - * demonstrates the scope of buffers holding cursor keys and values. - */ -#include -#include -#include -#include - -#include - -#ifdef _WIN32 -/* snprintf is not supported on <= VS2013 */ -#define snprintf _snprintf -#endif - -static const char *home; - -static int -cursor_scope_ops(WT_CURSOR *cursor) -{ - struct { - const char *op; - const char *key; - const char *value; - int (*apply)(WT_CURSOR *); - } *op, ops[] = { - { "insert", "key1", "value1", cursor->insert, }, - { "update", "key1", "value2", cursor->update, }, - { "search", "key1", "value2", cursor->search, }, - { "remove", "key1", "value2", cursor->remove, }, - { NULL, NULL, NULL, NULL } - }; - WT_SESSION *session; - const char *key, *value; - char keybuf[10], valuebuf[10]; - int ret; - - session = cursor->session; - - for (op = ops; op->key != NULL; op++) { - key = value = NULL; - - /*! [cursor scope operation] */ - (void)snprintf(keybuf, sizeof(keybuf), "%s", op->key); - cursor->set_key(cursor, keybuf); - (void)snprintf(valuebuf, sizeof(valuebuf), "%s", op->value); - cursor->set_value(cursor, valuebuf); - - /* - * The application must keep key and value memory valid until - * the next operation that positions the cursor, modifies the - * data, or resets or closes the cursor. - * - * Modifying either the key or value buffers is not permitted. - */ - - /* Apply the operation (insert, update, search or remove). */ - if ((ret = op->apply(cursor)) != 0) { - fprintf(stderr, - "%s: error performing the operation: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - - /* - * The cursor no longer references application memory, so - * application buffers can be safely overwritten. - */ - strcpy(keybuf, "no key"); - strcpy(valuebuf, "no value"); - - /* - * Check that get_key/value behave as expected after the - * operation. - */ - if (op->apply == cursor->insert) { - /* - * WT_CURSOR::insert no longer references application - * memory, but as it does not position the cursor, it - * doesn't reference memory owned by the cursor, either. - */ - printf("ex_scope: " - "expect two WiredTiger error messages:\n"); - if ((ret = cursor->get_key(cursor, &key)) == 0 || - (ret = cursor->get_value(cursor, &value)) == 0) { - fprintf(stderr, - "%s: error in get_key/value: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - continue; - } - if (op->apply == cursor->remove) { - /* - * WT_CURSOR::remove no longer references application - * memory; as it does not position the cursor, it will - * reference key memory owned by the cursor, but has no - * value. - */ - printf("ex_scope: " - "expect one WiredTiger error message:\n"); - if ((ret = cursor->get_key(cursor, &key)) != 0 || - (ret = cursor->get_value(cursor, &value)) == 0) { - fprintf(stderr, - "%s: error in get_key/value: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - } else /* search, update */{ - /* - * WT_CURSOR::search and WT_CURSOR::update no longer - * reference application memory; as they position the - * cursor, they will reference key/value memory owned - * by the cursor. - */ - if ((ret = cursor->get_key(cursor, &key)) != 0 || - (ret = cursor->get_value(cursor, &value)) != 0) { - fprintf(stderr, - "%s: error in get_key/value: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - } - - /* - * Modifying the memory referenced by either key or value is - * not permitted. - * - * Check that the cursor's key and value are what we expect. - */ - if (key == keybuf || - (op->apply != cursor->remove && value == valuebuf)) { - fprintf(stderr, - "%s: cursor points at application memory!\n", - op->op); - return (EINVAL); - } - - if (strcmp(key, op->key) != 0 || - (op->apply != cursor->remove && - strcmp(value, op->value) != 0)) { - fprintf(stderr, - "%s: unexpected key / value!\n", op->op); - return (EINVAL); - } - /*! [cursor scope operation] */ - } - - return (0); -} - -int -main(void) -{ - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - int ret; - - /* - * Create a clean test directory for this run of the test program if the - * environment variable isn't already set (as is done by make check). - */ - if (getenv("WIREDTIGER_HOME") == NULL) { - home = "WT_HOME"; - ret = system("rm -rf WT_HOME && mkdir WT_HOME"); - } else - home = NULL; - - /* Open a connection, create a simple table, open a cursor. */ - if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0 || - (ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { - fprintf(stderr, "Error connecting to %s: %s\n", - home == NULL ? "." : home, wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - - ret = session->create(session, - "table:scope", "key_format=S,value_format=S,columns=(k,v)"); - - ret = session->open_cursor(session, - "table:scope", NULL, NULL, &cursor); - - ret = cursor_scope_ops(cursor); - - /* Close the connection and clean up. */ - ret = conn->close(conn, NULL); - - return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/third_party/wiredtiger/examples/c/ex_sync.c b/src/third_party/wiredtiger/examples/c/ex_sync.c index 2c610b1e570..b2d74b52f7f 100644 --- a/src/third_party/wiredtiger/examples/c/ex_sync.c +++ b/src/third_party/wiredtiger/examples/c/ex_sync.c @@ -59,8 +59,8 @@ main(void) char cmd_buf[256], k[16], v[16]; const char *conf; - snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", - home, home); + (void)snprintf(cmd_buf, sizeof(cmd_buf), + "rm -rf %s && mkdir %s", home, home); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (EXIT_FAILURE); @@ -98,8 +98,8 @@ main(void) ret = session->commit_transaction(session, conf); ret = session->begin_transaction(session, NULL); } - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -113,8 +113,8 @@ main(void) * Perform some operations within a single transaction. */ for (i = MAX_KEYS; i < MAX_KEYS+5; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -129,8 +129,8 @@ main(void) * Demonstrate using log_flush to force the log to disk. */ for (i = 0; i < MAX_KEYS; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", record_count); - snprintf(v, sizeof(v), "value%d", record_count); + (void)snprintf(k, sizeof(k), "key%d", record_count); + (void)snprintf(v, sizeof(v), "value%d", record_count); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -138,8 +138,8 @@ main(void) ret = session->log_flush(session, "sync=on"); for (i = 0; i < MAX_KEYS; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", record_count); - snprintf(v, sizeof(v), "value%d", record_count); + (void)snprintf(k, sizeof(k), "key%d", record_count); + (void)snprintf(v, sizeof(v), "value%d", record_count); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); diff --git a/src/third_party/wiredtiger/examples/c/ex_thread.c b/src/third_party/wiredtiger/examples/c/ex_thread.c index 7c52d3b8189..fa82bd5f113 100644 --- a/src/third_party/wiredtiger/examples/c/ex_thread.c +++ b/src/third_party/wiredtiger/examples/c/ex_thread.c @@ -30,14 +30,13 @@ * table from multiple threads. */ -#ifndef _WIN32 -#include -#endif #include #include #include -#ifdef _WIN32 +#ifndef _WIN32 +#include +#else #include "windows_shim.h" #endif diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java index 03eab6b38b1..233ad1361d8 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java @@ -35,7 +35,7 @@ import java.util.*; class Lsn { int file; - long offset; + int offset; } public class ex_log { @@ -57,7 +57,7 @@ public class ex_log { conn = wiredtiger.open(home2, CONN_CONFIG); Session session = conn.open_session(null); - session.create(uri, "key_format=U,value_format=U"); + session.create(uri, "key_format=S,value_format=S"); return (session); } @@ -72,17 +72,17 @@ public class ex_log { while ((ret = cursor.next()) == 0) { ret = curs_copy.next(); - byte[] key = cursor.getKeyByteArray(); - byte[] value = cursor.getValueByteArray(); - byte[] key_copy = curs_copy.getKeyByteArray(); - byte[] value_copy = curs_copy.getValueByteArray(); - if (!Arrays.equals(key, key_copy) || - !Arrays.equals(value, value_copy)) { + String key = cursor.getKeyString(); + String value = cursor.getValueString(); + String key_copy = curs_copy.getKeyString(); + String value_copy = curs_copy.getValueString(); + if (!key.equals(key_copy) || + !value.equals(value_copy)) { System.err.println( - "Mismatched: key " + new String(key) + - ", key_copy " + new String(key_copy) + - ", value " + new String(value) + - ", value_copy " + new String(value_copy)); + "Mismatched: key " + key + ", key_copy " + key_copy + + ", value " + value + ", value_copy " + value_copy); + ret = cursor.close(); + ret = curs_copy.close(); return (1); } } @@ -121,7 +121,7 @@ public class ex_log { * A simple walk of the log. */ static int - simple_walk_log(Session session) + simple_walk_log(Session session, int count_min) throws WiredTigerException { Cursor cursor; @@ -129,16 +129,18 @@ public class ex_log { byte[] logrec_key, logrec_value; long txnid; int fileid, opcount, optype, rectype; - int ret; + int count, ret; /*! [log cursor open] */ cursor = session.open_cursor("log:", null, null); /*! [log cursor open] */ + count = 0; while ((ret = cursor.next()) == 0) { + count++; /*! [log cursor get_key] */ lsn.file = cursor.getKeyInt(); - lsn.offset = cursor.getKeyLong(); + lsn.offset = cursor.getKeyInt(); opcount = cursor.getKeyInt(); /*! [log cursor get_key] */ /*! [log cursor get_value] */ @@ -156,6 +158,11 @@ public class ex_log { if (ret == wiredtiger.WT_NOTFOUND) ret = 0; ret = cursor.close(); + if (count < count_min) { + System.err.println("Expected minimum " + count_min + + " records, found " + count); + return (1); + } return (ret); } /*! [log cursor walk] */ @@ -185,7 +192,7 @@ public class ex_log { lsnsave = new Lsn(); while ((ret = cursor.next()) == 0) { lsn.file = cursor.getKeyInt(); - lsn.offset = cursor.getKeyLong(); + lsn.offset = cursor.getKeyInt(); opcount = cursor.getKeyInt(); /* @@ -194,8 +201,10 @@ public class ex_log { * that LSN to the end (where the multi-step transaction * was performed). Just choose the record that is MAX_KEYS. */ - if (++i == MAX_KEYS) - lsnsave = lsn; + if (++i == MAX_KEYS) { + lsnsave.file = lsn.file; + lsnsave.offset = lsn.offset; + } txnid = cursor.getValueLong(); rectype = cursor.getValueInt(); optype = cursor.getValueInt(); @@ -217,10 +226,10 @@ public class ex_log { /* * If the operation is a put, replay it here on the backup - * connection. Note, we cheat by looking only for fileid 1 - * in this example. The metadata is fileid 0. + * connection. Note, we cheat by looking at the fileid. + * The metadata is fileid 0, skip its records. */ - if (fileid == 1 && rectype == wiredtiger.WT_LOGREC_COMMIT && + if (fileid != 0 && rectype == wiredtiger.WT_LOGREC_COMMIT && optype == wiredtiger.WT_LOGOP_ROW_PUT) { if (!in_txn) { ret = session2.begin_transaction(null); @@ -238,15 +247,20 @@ public class ex_log { /* * Compare the tables after replay. They should be identical. */ - if (compare_tables(session, session2) != 0) - System.out.println("compare failed"); + if (compare_tables(session, session2) != 0) { + cursor.close(); + session2.close(null); + wt_conn2.close(null); + return (ret); + } ret = session2.close(null); ret = wt_conn2.close(null); ret = cursor.reset(); /*! [log cursor set_key] */ cursor.putKeyInt(lsnsave.file); - cursor.putKeyLong(lsnsave.offset); + cursor.putKeyInt(lsnsave.offset); + cursor.putKeyInt(0); /*! [log cursor set_key] */ /*! [log cursor search] */ ret = cursor.search(); @@ -256,9 +270,9 @@ public class ex_log { * Walk all records starting with this key. */ first = true; - while (ret == 0) { /*TODO: not quite right*/ + while (ret == 0) { lsn.file = cursor.getKeyInt(); - lsn.offset = cursor.getKeyLong(); + lsn.offset = cursor.getKeyInt(); opcount = cursor.getKeyInt(); if (first) { first = false; @@ -293,8 +307,9 @@ public class ex_log { Connection wt_conn; Cursor cursor; Session session; - int i, record_count, ret; + int count_min, i, record_count, ret; + count_min = 0; try { String command = "/bin/rm -rf " + home1 + " " + home2; Process proc = Runtime.getRuntime().exec(command); @@ -317,6 +332,7 @@ public class ex_log { session = wt_conn.open_session(null); ret = session.create(uri, "key_format=S,value_format=S"); + count_min++; cursor = session.open_cursor(uri, null, null); /* @@ -328,6 +344,7 @@ public class ex_log { cursor.putKeyString(k); cursor.putValueString(v); ret = cursor.insert(); + count_min++; } ret = session.begin_transaction(null); /* @@ -341,10 +358,12 @@ public class ex_log { ret = cursor.insert(); } ret = session.commit_transaction(null); + count_min++; ret = cursor.close(); /*! [log cursor printf] */ ret = session.log_printf("Wrote " + record_count + " records"); + count_min++; /*! [log cursor printf] */ session.close(null); @@ -360,7 +379,7 @@ public class ex_log { } session = wt_conn.open_session(null); - ret = simple_walk_log(session); + ret = simple_walk_log(session, count_min); ret = walk_log(session); ret = session.close(null); ret = wt_conn.close(null); diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index 3665ec48b9a..09a793646e7 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -483,8 +483,7 @@ static int zlib_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *zlib_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret, zlib_level; @@ -497,49 +496,27 @@ zlib_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - /* - * Between 0-9: level: see zlib manual. - */ - zlib_level = (int)v.val; - if (zlib_level < 0 || zlib_level > 9) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: " - "unsupported compression level %d", - zlib_level); - return (EINVAL); - } - *zlib_levelp = zlib_level; - continue; + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) { + /* + * Between 0-9: level: see zlib manual. + */ + zlib_level = (int)v.val; + if (zlib_level < 0 || zlib_level > 9) { + (void)wt_api->err_printf(wt_api, NULL, + "zlib_init_config: " + "unsupported compression level %d", + zlib_level); + return (EINVAL); } - if (ret != WT_NOTFOUND) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { + *zlib_levelp = zlib_level; + } else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zlib configure: %s", + "zlib_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index a459b01d60a..ea8ec97602f 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -232,8 +232,7 @@ static int zstd_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *compression_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret; @@ -246,38 +245,16 @@ zstd_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - *compression_levelp = (int)v.val; - continue; - } - if (ret != WT_NOTFOUND) { + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) + *compression_levelp = (int)v.val; + else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zstd configure: %s", + "zstd_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/src/third_party/wiredtiger/ext/datasources/helium/helium.c b/src/third_party/wiredtiger/ext/datasources/helium/helium.c index 473c569f0cc..c584141b00d 100644 --- a/src/third_party/wiredtiger/ext/datasources/helium/helium.c +++ b/src/third_party/wiredtiger/ext/datasources/helium/helium.c @@ -3329,6 +3329,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) * compile-time should the structure change underneath us. */ static const WT_DATA_SOURCE wtds = { + NULL, /* No session.alter */ helium_session_create, /* session.create */ NULL, /* No session.compaction */ helium_session_drop, /* session.drop */ @@ -3380,15 +3381,9 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) goto err; ds->lockinit = 1; - /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - EMSG_ERR(wt_api, NULL, ret, - "WT_EXTENSION_API.config_get: config: %s", - wt_api->strerror(wt_api, NULL, ret)); - /* Step through the list of Helium sources, opening each one. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) + if ((ret = wt_api->config_parser_open_arg( + wt_api, NULL, config, &config_parser)) != 0) EMSG_ERR(wt_api, NULL, ret, "WT_EXTENSION_API.config_parser_open: config: %s", wt_api->strerror(wt_api, NULL, ret)); diff --git a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c index 559c8e6e33a..0b905a0540d 100644 --- a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c @@ -76,7 +76,7 @@ typedef struct { u_char *shift_forw; /* Encrypt shift data from secretkey */ u_char *shift_back; /* Decrypt shift data from secretkey */ size_t shift_len; /* Length of shift* byte arrays */ - int force_error; /* Force a decrypt error for testing */ + bool force_error; /* Force a decrypt error for testing */ } ROTN_ENCRYPTOR; /*! [WT_ENCRYPTOR initialization structure] */ @@ -429,43 +429,19 @@ rotn_terminate(WT_ENCRYPTOR *encryptor, WT_SESSION *session) static int rotn_configure(ROTN_ENCRYPTOR *rotn_encryptor, WT_CONFIG_ARG *config) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; /* Extension API */ - int ret, t_ret; + int ret; wt_api = rotn_encryptor->wt_api; /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_get")); - - /* Step through the list of configuration options. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_parser_open")); - - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { - if (strncmp("rotn_force_error", k.str, k.len) == 0 && - strlen("rotn_force_error") == k.len) { - rotn_encryptor->force_error = v.val == 0 ? 0 : 1; - continue; - } else { - if ((ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, - NULL, ret, "WT_CONFIG_PARSER.close")); - return (rotn_error(rotn_encryptor, NULL, EINVAL, - "unknown config key")); - } - } - if ((t_ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, t_ret, - "WT_CONFIG_PARSER.close")); - if (ret != WT_NOTFOUND) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_CONFIG_PARSER.next")); + if ((ret = wt_api->config_get( + wt_api, NULL, config, "rotn_force_error", &v)) == 0) + rotn_encryptor->force_error = v.val != 0; + else if (ret != WT_NOTFOUND) + return (rotn_error(rotn_encryptor, NULL, EINVAL, + "error parsing config")); return (0); } diff --git a/src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am b/src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am new file mode 100644 index 00000000000..f31f5395cd1 --- /dev/null +++ b/src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am @@ -0,0 +1,9 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include + +noinst_LTLIBRARIES = libwiredtiger_fail_fs.la +libwiredtiger_fail_fs_la_SOURCES = fail_fs.c + +# libtool hack: noinst_LTLIBRARIES turns off building shared libraries as well +# as installation, it will only build static libraries. As far as I can tell, +# the "approved" libtool way to turn them back on is by adding -rpath. +libwiredtiger_fail_fs_la_LDFLAGS = -avoid-version -module -rpath /nowhere diff --git a/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c new file mode 100644 index 00000000000..d0d8a14c8c2 --- /dev/null +++ b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c @@ -0,0 +1,847 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "queue.h" + +#define FAIL_FS_GIGABYTE (1024 * 1024 * 1024) + +#define FAIL_FS_ENV_ENABLE "WT_FAIL_FS_ENABLE" +#define FAIL_FS_ENV_WRITE_ALLOW "WT_FAIL_FS_WRITE_ALLOW" +#define FAIL_FS_ENV_READ_ALLOW "WT_FAIL_FS_READ_ALLOW" + +/* + * A "fail file system", that is, a file system extension that fails when we + * want it to. This is only used in test frameworks, this fact allows us to + * simplify some error paths. This code is not portable to Windows, as it has + * direct knowledge of file descriptors, environment variables and stack + * traces. + * + * When the filesystem extension is configured, parameters can set how many + * reads or writes can be allowed before failure. If this is not fine-grained + * enough, an 'environment' configuration parameter can be specified. If that + * is used, then on every file system read or write, environment variables are + * checked that control when reading or writing should fail. + */ +typedef struct { + WT_FILE_SYSTEM iface; + /* + * WiredTiger performs schema and I/O operations in parallel, all file + * system and file handle access must be thread-safe. This extension + * uses a single, global file system lock. + */ + pthread_rwlock_t lock; /* Lock */ + bool fail_enabled; + bool use_environment; + bool verbose; + int64_t read_ops; + int64_t write_ops; + int64_t allow_reads; + int64_t allow_writes; + /* Queue of file handles */ + TAILQ_HEAD(fail_file_handle_qh, fail_file_handle) fileq; + WT_EXTENSION_API *wtext; /* Extension functions */ +} FAIL_FILE_SYSTEM; + +typedef struct fail_file_handle { + WT_FILE_HANDLE iface; + + /* + * Track the system file descriptor for each file. + */ + FAIL_FILE_SYSTEM *fail_fs; /* Enclosing file system */ + TAILQ_ENTRY(fail_file_handle) q; /* Queue of handles */ + int fd; /* System file descriptor */ +} FAIL_FILE_HANDLE; + +static int fail_file_close(WT_FILE_HANDLE *, WT_SESSION *); +static void fail_file_handle_remove(WT_SESSION *, FAIL_FILE_HANDLE *); +static int fail_file_lock(WT_FILE_HANDLE *, WT_SESSION *, bool); +static int fail_file_read( + WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); +static int fail_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); +static int fail_file_sync(WT_FILE_HANDLE *, WT_SESSION *); +static int fail_file_truncate(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t); +static int fail_file_write( + WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); +static bool fail_fs_arg( + const char *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, int64_t *); +static int fail_fs_directory_list(WT_FILE_SYSTEM *, WT_SESSION *, + const char *, const char *, char ***, uint32_t *); +static int fail_fs_directory_list_free( + WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t); +static void fail_fs_env(const char *, int64_t *); +static int fail_fs_exist(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); +static int fail_fs_open(WT_FILE_SYSTEM *, WT_SESSION *, + const char *, WT_FS_OPEN_FILE_TYPE, uint32_t, WT_FILE_HANDLE **); +static int fail_fs_remove( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, uint32_t); +static int fail_fs_rename( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t); +static int fail_fs_simulate_fail( + FAIL_FILE_HANDLE *, WT_SESSION *, int64_t, const char *); +static int fail_fs_size( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); +static int fail_fs_terminate(WT_FILE_SYSTEM *, WT_SESSION *); + +/* + * We use pthread functions for portable locking. + * Assert on errors for simplicity. + */ +static void +fail_fs_allocate_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_init(lockp, NULL) == 0); +} + +static void +fail_fs_destroy_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_destroy(lockp) == 0); +} + +static void +fail_fs_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_wrlock(lockp) == 0); +} + +static void +fail_fs_unlock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_unlock(lockp) == 0); +} + +/* + * fail_file_close -- + * ANSI C close. + */ +static int +fail_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + int ret; + + (void)session; /* Unused */ + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + fail_fs = fail_fh->fail_fs; + + /* + * We don't actually open an fd when opening directories for flushing, + * so ignore that case here. + */ + if (fail_fh->fd < 0) + return (0); + ret = close(fail_fh->fd); + fail_fh->fd = -1; + fail_fs_lock(&fail_fs->lock); + fail_file_handle_remove(session, fail_fh); + fail_fs_unlock(&fail_fs->lock); + return (ret); +} + +/* + * fail_file_handle_remove -- + * Destroy an in-memory file handle. Should only happen on remove or + * shutdown. The file system lock must be held during this call. + */ +static void +fail_file_handle_remove(WT_SESSION *session, FAIL_FILE_HANDLE *fail_fh) +{ + FAIL_FILE_SYSTEM *fail_fs; + + (void)session; /* Unused */ + fail_fs = fail_fh->fail_fs; + + TAILQ_REMOVE(&fail_fs->fileq, fail_fh, q); + + free(fail_fh->iface.name); + free(fail_fh); +} + +/* + * fail_file_lock -- + * Lock/unlock a file. + */ +static int +fail_file_lock(WT_FILE_HANDLE *file_handle, WT_SESSION *session, bool lock) +{ + /* Locks are always granted. */ + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + (void)lock; /* Unused */ + + return (0); +} + +/* + * fail_file_read -- + * POSIX pread. + */ +static int +fail_file_read(WT_FILE_HANDLE *file_handle, + WT_SESSION *session, wt_off_t offset, size_t len, void *buf) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; + int64_t envint, read_ops; + int ret; + size_t chunk; + ssize_t nr; + uint8_t *addr; + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + fail_fs = fail_fh->fail_fs; + wtext = fail_fs->wtext; + read_ops = 0; + ret = 0; + + fail_fs_lock(&fail_fs->lock); + + if (fail_fs->use_environment) { + fail_fs_env(FAIL_FS_ENV_ENABLE, &envint); + if (envint != 0) { + if (!fail_fs->fail_enabled) { + fail_fs->fail_enabled = true; + fail_fs_env(FAIL_FS_ENV_READ_ALLOW, + &fail_fs->allow_reads); + fail_fs->read_ops = 0; + } + read_ops = ++fail_fs->read_ops; + } else + fail_fs->fail_enabled = false; + } else + read_ops = ++fail_fs->read_ops; + + fail_fs_unlock(&fail_fs->lock); + + if (fail_fs->fail_enabled && fail_fs->allow_reads != 0 && + read_ops % fail_fs->allow_reads == 0) + return (fail_fs_simulate_fail( + fail_fh, session, read_ops, "read")); + + /* Break reads larger than 1GB into 1GB chunks. */ + for (addr = buf; len > 0; addr += nr, len -= (size_t)nr, offset += nr) { + chunk = (len < FAIL_FS_GIGABYTE) ? len : FAIL_FS_GIGABYTE; + if ((nr = pread(fail_fh->fd, addr, chunk, offset)) <= 0) { + (void)wtext->err_printf(wtext, session, + "%s: handle-read: failed to read %" PRIuMAX + " bytes at offset %" PRIuMAX ": %s", + fail_fh->iface.name, + (uintmax_t)len, (uintmax_t)offset, + wtext->strerror(wtext, NULL, errno)); + ret = (nr == 0 ? WT_ERROR : errno); + break; + } + } + return (ret); +} + +/* + * fail_file_size -- + * Get the size of a file in bytes, by file handle. + */ +static int +fail_file_size( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) +{ + FAIL_FILE_HANDLE *fail_fh; + struct stat statbuf; + int ret; + + (void)session; /* Unused */ + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + ret = 0; + + if ((ret = fstat(fail_fh->fd, &statbuf)) != 0) + return (ret); + *sizep = statbuf.st_size; + return (0); +} + +/* + * fail_file_sync -- + * Ensure the content of the file is stable. This is a no-op in our + * file system. + */ +static int +fail_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + + return (0); +} + +/* + * fail_file_truncate -- + * POSIX ftruncate. + */ +static int +fail_file_truncate( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset) +{ + FAIL_FILE_HANDLE *fail_fh; + + (void)session; /* Unused */ + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + return (ftruncate(fail_fh->fd, offset)); +} + +/* + * fail_file_write -- + * POSIX pwrite. + */ +static int +fail_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, + wt_off_t offset, size_t len, const void *buf) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; + int64_t envint, write_ops; + int ret; + size_t chunk; + ssize_t nr; + const uint8_t *addr; + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + fail_fs = fail_fh->fail_fs; + wtext = fail_fs->wtext; + write_ops = 0; + ret = 0; + + fail_fs_lock(&fail_fs->lock); + + if (fail_fs->use_environment) { + fail_fs_env(FAIL_FS_ENV_ENABLE, &envint); + if (envint != 0) { + if (!fail_fs->fail_enabled) { + fail_fs->fail_enabled = true; + fail_fs_env(FAIL_FS_ENV_WRITE_ALLOW, + &fail_fs->allow_writes); + fail_fs->write_ops = 0; + } + write_ops = ++fail_fs->write_ops; + } else + fail_fs->fail_enabled = false; + } else + write_ops = ++fail_fs->write_ops; + + fail_fs_unlock(&fail_fs->lock); + + if (fail_fs->fail_enabled && fail_fs->allow_writes != 0 && + write_ops % fail_fs->allow_writes == 0) + return (fail_fs_simulate_fail( + fail_fh, session, write_ops, "write")); + + /* Break writes larger than 1GB into 1GB chunks. */ + for (addr = buf; len > 0; addr += nr, len -= (size_t)nr, offset += nr) { + chunk = (len < FAIL_FS_GIGABYTE) ? len : FAIL_FS_GIGABYTE; + if ((nr = pwrite(fail_fh->fd, addr, chunk, offset)) <= 0) { + (void)wtext->err_printf(wtext, session, + "%s: handle-write: failed to write %" PRIuMAX + " bytes at offset %" PRIuMAX ": %s", + fail_fh->iface.name, + (uintmax_t)len, (uintmax_t)offset, + wtext->strerror(wtext, NULL, errno)); + ret = (nr == 0 ? WT_ERROR : errno); + break; + } + } + return (ret); +} + +/* + * fail_fs_arg -- + * If the key matches, return the value interpreted as an integer. + */ +static bool +fail_fs_arg(const char *match, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value, + int64_t *argp) +{ + if (strncmp(match, key->str, key->len) == 0 && + match[key->len] == '\0' && + (value->type == WT_CONFIG_ITEM_BOOL || + value->type == WT_CONFIG_ITEM_NUM)) { + *argp = value->val; + return (true); + } + return (false); +} + +/* + * fail_fs_directory_list -- + * Return a list of files in a given sub-directory. + */ +static int +fail_fs_directory_list(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *directory, + const char *prefix, char ***dirlistp, uint32_t *countp) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + size_t len, prefix_len; + uint32_t allocated, count; + int ret; + char *name, **entries; + void *p; + + (void)session; /* Unused */ + + fail_fs = (FAIL_FILE_SYSTEM *)file_system; + ret = 0; + *dirlistp = NULL; + *countp = 0; + + entries = NULL; + allocated = count = 0; + len = strlen(directory); + prefix_len = prefix == NULL ? 0 : strlen(prefix); + + fail_fs_lock(&fail_fs->lock); + TAILQ_FOREACH(fail_fh, &fail_fs->fileq, q) { + name = fail_fh->iface.name; + if (strncmp(name, directory, len) != 0 || + (prefix != NULL && strncmp(name, prefix, prefix_len) != 0)) + continue; + + /* + * Increase the list size in groups of 10, it doesn't + * matter if the list is a bit longer than necessary. + */ + if (count >= allocated) { + p = realloc( + entries, (allocated + 10) * sizeof(*entries)); + if (p == NULL) { + ret = ENOMEM; + goto err; + } + entries = p; + memset(entries + allocated * sizeof(*entries), + 0, 10 * sizeof(*entries)); + allocated += 10; + } + entries[count++] = strdup(name); + } + + *dirlistp = entries; + *countp = count; + +err: fail_fs_unlock(&fail_fs->lock); + if (ret == 0) + return (0); + + if (entries != NULL) { + while (count > 0) + free(entries[--count]); + free(entries); + } + + return (ret); +} + +/* + * fail_fs_directory_list_free -- + * Free memory allocated by fail_fs_directory_list. + */ +static int +fail_fs_directory_list_free(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, char **dirlist, uint32_t count) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + + if (dirlist != NULL) { + while (count > 0) + free(dirlist[--count]); + free(dirlist); + } + return (0); +} + +/* + * fail_fs_env -- + * If the name is in the environment, return its integral value. + */ +static void +fail_fs_env(const char *name, int64_t *valp) +{ + int64_t result; + char *s, *value; + + result = 0; + if ((value = getenv(name)) != NULL) { + s = value; + if (strcmp(value, "true") == 0) + result = 1; + else if (strcmp(value, "false") != 0) { + result = strtoll(value, &s, 10); + if (*s != '\0') + result = 0; + } + } + *valp = result; +} + +/* + * fail_fs_exist -- + * Return if the file exists. + */ +static int +fail_fs_exist(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *name, bool *existp) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + + *existp = (access(name, F_OK) == 0); + return (0); +} + +/* + * fail_fs_open -- + * fopen for the fail file system. + */ +static int +fail_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, + const char *name, WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags, + WT_FILE_HANDLE **file_handlep) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; + WT_FILE_HANDLE *file_handle; + int fd, open_flags, ret; + + (void)session; /* Unused */ + + *file_handlep = NULL; + + fail_fh = NULL; + fail_fs = (FAIL_FILE_SYSTEM *)file_system; + fd = -1; + ret = 0; + + if (fail_fs->verbose) { + wtext = fail_fs->wtext; + (void)wtext->msg_printf(wtext, session, "fail_fs: open: %s", + name); + } + + fail_fs_lock(&fail_fs->lock); + + open_flags = 0; + if ((flags & WT_FS_OPEN_CREATE) != 0) + open_flags |= O_CREAT; + if ((flags & WT_FS_OPEN_EXCLUSIVE) != 0) + open_flags |= O_EXCL; + if ((flags & WT_FS_OPEN_READONLY) != 0) + open_flags |= O_RDONLY; + else + open_flags |= O_RDWR; + + /* + * Opening a file handle on a directory is only to support filesystems + * that require a directory sync for durability. This is a no-op + * for this file system. + */ + if (file_type == WT_FS_OPEN_FILE_TYPE_DIRECTORY) + fd = -1; + else if ((fd = open(name, open_flags, 0666)) < 0) { + ret = errno; + goto err; + } + + /* We create a handle structure for each open. */ + if ((fail_fh = calloc(1, sizeof(FAIL_FILE_HANDLE))) == NULL) { + ret = ENOMEM; + goto err; + } + + /* Initialize private information. */ + fail_fh->fail_fs = fail_fs; + fail_fh->fd = fd; + + /* Initialize public information. */ + file_handle = (WT_FILE_HANDLE *)fail_fh; + if ((file_handle->name = strdup(name)) == NULL) { + ret = ENOMEM; + goto err; + } + + /* Setup the function call table. */ + file_handle->close = fail_file_close; + file_handle->fh_advise = NULL; + file_handle->fh_extend = NULL; + file_handle->fh_extend_nolock = NULL; + file_handle->fh_lock = fail_file_lock; + file_handle->fh_map = NULL; + file_handle->fh_map_discard = NULL; + file_handle->fh_map_preload = NULL; + file_handle->fh_unmap = NULL; + file_handle->fh_read = fail_file_read; + file_handle->fh_size = fail_file_size; + file_handle->fh_sync = fail_file_sync; + file_handle->fh_sync_nowait = NULL; + file_handle->fh_truncate = fail_file_truncate; + file_handle->fh_write = fail_file_write; + + TAILQ_INSERT_HEAD(&fail_fs->fileq, fail_fh, q); + + *file_handlep = file_handle; + + if (0) { +err: if (fd != -1) + (void)close(fd); + free(fail_fh); + } + + fail_fs_unlock(&fail_fs->lock); + return (ret); +} + +/* + * fail_fs_remove -- + * POSIX remove. + */ +static int +fail_fs_remove(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *name, uint32_t flags) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ + + return (unlink(name)); +} + +/* + * fail_fs_rename -- + * POSIX rename. + */ +static int +fail_fs_rename(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *from, const char *to, uint32_t flags) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ + + return (rename(from, to)); +} + +/* + * fail_fs_simulate_fail -- + * Simulate a failure from this file system by reporting it + * and returning a non-zero return code. + */ +static int +fail_fs_simulate_fail(FAIL_FILE_HANDLE *fail_fh, WT_SESSION *session, + int64_t nops, const char *opkind) +{ + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; +#ifdef __FreeBSD__ + size_t btret, i; +#else + int btret, i; +#endif + void *bt[100]; + char **btstr; + + fail_fs = fail_fh->fail_fs; + if (fail_fs->verbose) { + wtext = fail_fs->wtext; + (void)wtext->msg_printf(wtext, session, + "fail_fs: %s: simulated failure after %" PRId64 + " %s operations", fail_fh->iface.name, nops, opkind); +#ifdef __FreeBSD__ + btret = backtrace(bt, sizeof(bt) / sizeof(bt[0])); +#else + btret = backtrace(bt, (int)(sizeof(bt) / sizeof(bt[0]))); +#endif + if ((btstr = backtrace_symbols(bt, btret)) != NULL) { + for (i = 0; i < btret; i++) + (void)wtext->msg_printf(wtext, session, " %s", + btstr[i]); + free(btstr); + } + } + return (EIO); +} + +/* + * fail_fs_size -- + * Get the size of a file in bytes, by file name. + */ +static int +fail_fs_size(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *name, wt_off_t *sizep) +{ + struct stat statbuf; + int ret; + + (void)file_system; /* Unused */ + (void)session; /* Unused */ + + ret = 0; + if ((ret = stat(name, &statbuf)) != 0) + return (ret); + *sizep = statbuf.st_size; + return (0); +} + +/* + * fail_fs_terminate -- + * Discard any resources on termination + */ +static int +fail_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + + fail_fs = (FAIL_FILE_SYSTEM *)file_system; + + while ((fail_fh = TAILQ_FIRST(&fail_fs->fileq)) != NULL) + fail_file_handle_remove(session, fail_fh); + + fail_fs_destroy_lock(&fail_fs->lock); + free(fail_fs); + + return (0); +} + +/* + * wiredtiger_extension_init -- + * WiredTiger fail filesystem extension. + */ +int +wiredtiger_extension_init(WT_CONNECTION *conn, WT_CONFIG_ARG *config) +{ + FAIL_FILE_SYSTEM *fail_fs; + WT_CONFIG_ITEM k, v; + WT_CONFIG_PARSER *config_parser; + WT_EXTENSION_API *wtext; + WT_FILE_SYSTEM *file_system; + int64_t argval; + int ret; + + ret = 0; + wtext = conn->get_extension_api(conn); + if ((fail_fs = calloc(1, sizeof(FAIL_FILE_SYSTEM))) == NULL) { + (void)wtext->err_printf(wtext, NULL, + "fail_file_system extension_init: %s", + wtext->strerror(wtext, NULL, ENOMEM)); + return (ENOMEM); + } + fail_fs->wtext = wtext; + file_system = (WT_FILE_SYSTEM *)fail_fs; + + /* Get any configuration values. */ + if ((ret = wtext->config_parser_open_arg( + wtext, NULL, config, &config_parser)) != 0) { + (void)wtext->err_printf(wtext, NULL, + "WT_EXTENSION_API.config_parser_open: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { + if (fail_fs_arg("environment", &k, &v, &argval)) { + fail_fs->use_environment = (argval != 0); + continue; + } else if (fail_fs_arg("verbose", &k, &v, &argval)) { + fail_fs->verbose = (argval != 0); + continue; + } else if (fail_fs_arg("allow_writes", &k, &v, + &fail_fs->allow_writes)) + continue; + else if (fail_fs_arg("allow_reads", &k, &v, + &fail_fs->allow_reads)) + continue; + + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.next: unexpected configuration " + "information: %.*s=%.*s: %s", + (int)k.len, k.str, (int)v.len, v.str, + wtext->strerror(wtext, NULL, ret)); + goto err; + } + if (ret != WT_NOTFOUND) { + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.next: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + if ((ret = config_parser->close(config_parser)) != 0) { + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.close: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + if (fail_fs->allow_writes != 0 || fail_fs->allow_reads != 0) + fail_fs->fail_enabled = true; + + fail_fs_allocate_lock(&fail_fs->lock); + /* Initialize the in-memory jump table. */ + file_system->fs_directory_list = fail_fs_directory_list; + file_system->fs_directory_list_free = fail_fs_directory_list_free; + file_system->fs_exist = fail_fs_exist; + file_system->fs_open_file = fail_fs_open; + file_system->fs_remove = fail_fs_remove; + file_system->fs_rename = fail_fs_rename; + file_system->fs_size = fail_fs_size; + file_system->terminate = fail_fs_terminate; + if ((ret = conn->set_file_system(conn, file_system, NULL)) != 0) { + (void)wtext->err_printf(wtext, NULL, + "WT_CONNECTION.set_file_system: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + return (0); + +err: free(fail_fs); + return (ret); +} diff --git a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c index 866cd0663ce..0791b077750 100644 --- a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c +++ b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c @@ -1016,9 +1016,10 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { /* * List of the WT_DATA_SOURCE methods -- it's static so it breaks at - * compile-time should the structure changes underneath us. + * compile-time should the structure change underneath us. */ static WT_DATA_SOURCE wtds = { + NULL, /* No session.alter */ kvs_session_create, /* session.create */ NULL, /* No session.compaction */ kvs_session_drop, /* session.drop */ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data new file mode 100644 index 00000000000..abf3fe5cb9c --- /dev/null +++ b/src/third_party/wiredtiger/import.data @@ -0,0 +1,6 @@ +{ + "commit": "b8f590dea0400666ef26e21adf11c5997bb5ef1b", + "github": "wiredtiger/wiredtiger.git", + "vendor": "wiredtiger", + "branch": "mongodb-3.2" +} diff --git a/src/third_party/wiredtiger/lang/java/Makefile.am b/src/third_party/wiredtiger/lang/java/Makefile.am index 7184fe610dc..2ff822a5d08 100644 --- a/src/third_party/wiredtiger/lang/java/Makefile.am +++ b/src/third_party/wiredtiger/lang/java/Makefile.am @@ -49,6 +49,7 @@ JAVA_JUNIT = \ $(JAVATEST)/ConcurrentCloseTest.java \ $(JAVATEST)/CursorTest.java \ $(JAVATEST)/CursorTest02.java \ + $(JAVATEST)/CursorTest03.java \ $(JAVATEST)/ExceptionTest.java \ $(JAVATEST)/PackTest.java \ $(JAVATEST)/PackTest02.java \ diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index 2264cb31ef1..3606bed1d69 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -27,6 +27,7 @@ COPYDOC(__wt_async_op, WT_ASYNC_OP, remove) COPYDOC(__wt_async_op, WT_ASYNC_OP, compact) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_id) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_type) +COPYDOC(__wt_session, WT_SESSION, alter) COPYDOC(__wt_session, WT_SESSION, close) COPYDOC(__wt_session, WT_SESSION, reconfigure) COPYDOC(__wt_session, WT_SESSION, open_cursor) diff --git a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java index 46b3aef0974..b6804a2992f 100644 --- a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java +++ b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java @@ -42,6 +42,7 @@ public class PackOutputStream { protected PackFormatInputStream format; protected ByteArrayOutputStream packed; protected byte[] intBuf; + protected boolean isRaw; /** * Constructor. @@ -50,9 +51,21 @@ public class PackOutputStream { * defines the layout of this packed value. */ public PackOutputStream(String format) { - this.format = new PackFormatInputStream(format, false); - intBuf = new byte[MAX_INT_BYTES]; - packed = new ByteArrayOutputStream(100); + this(format, false); + } + + /** + * Constructor. + * + * \param format A String that contains the WiredTiger format that + * defines the layout of this packed value. + * \param isRaw The stream is opened raw. + */ + public PackOutputStream(String format, boolean isRaw) { + this.format = new PackFormatInputStream(format, isRaw); + this.intBuf = new byte[MAX_INT_BYTES]; + this.packed = new ByteArrayOutputStream(100); + this.isRaw = isRaw; } /** @@ -111,7 +124,9 @@ public class PackOutputStream { throws WiredTigerPackingException { int padBytes = 0; - format.checkType('U', false); + if (!isRaw) { + format.checkType('U', false); + } boolean havesize = format.hasLength(); char type = format.getType(); if (havesize) { diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index d6e0660dc2c..275b708090c 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -127,7 +127,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(javain) uint64_t "$javainput" %typemap(javaout) uint64_t { - return $jnicall; + return ($jnicall); } /* Return byte[] from cursor.get_value */ @@ -137,7 +137,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(javain) WT_ITEM, WT_ITEM * "$javainput" %typemap(javaout) WT_ITEM, WT_ITEM * { - return $jnicall; + return ($jnicall); } %typemap(in) WT_ITEM * (WT_ITEM item) %{ @@ -165,7 +165,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(out) int %{ if ($1 != 0 && $1 != WT_NOTFOUND) { throwWiredTigerException(jenv, $1); - return $null; + return ($null); } $result = $1; %} @@ -174,7 +174,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { if (!val) { SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, #name " is null"); - return $null; + return ($null); } %enddef @@ -319,6 +319,15 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %rename (getValueFormat) __wt_async_op::getValue_format; %rename (getType) __wt_async_op::get_type; +/* + * Special cases: override the out typemap, return checking is done in the + * wrapper. + */ +%typemap(out) int __wt_cursor::compare_wrap, + int __wt_cursor::equals_wrap %{ + $result = $1; +%} + /* SWIG magic to turn Java byte strings into data / size. */ %apply (char *STRING, int LENGTH) { (char *data, int size) }; @@ -529,7 +538,6 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %} %extend __wt_async_op { - %javamethodmodifiers get_key_wrap "protected"; WT_ITEM get_key_wrap(JNIEnv *jenv) { WT_ITEM k; @@ -537,7 +545,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; k.data = NULL; if ((ret = $self->get_key($self, &k)) != 0) throwWiredTigerException(jenv, ret); - return k; + return (k); } %javamethodmodifiers get_value_wrap "protected"; @@ -547,40 +555,40 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; v.data = NULL; if ((ret = $self->get_value($self, &v)) != 0) throwWiredTigerException(jenv, ret); - return v; + return (v); } %javamethodmodifiers insert_wrap "protected"; int insert_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->insert($self); + return ($self->insert($self)); } %javamethodmodifiers remove_wrap "protected"; int remove_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->remove($self); + return ($self->remove($self)); } %javamethodmodifiers search_wrap "protected"; int search_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->search($self); + return ($self->search($self)); } %javamethodmodifiers update_wrap "protected"; int update_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->update($self); + return ($self->update($self)); } %javamethodmodifiers _java_raw "protected"; bool _java_raw(JNIEnv *jenv) { (void)jenv; JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->c.lang_private; - return jcb->cursor_raw; + return (jcb->cursor_raw); } %javamethodmodifiers _java_init "protected"; @@ -597,6 +605,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %typemap(javabody) struct __wt_async_op %{ private long swigCPtr; protected boolean swigCMemOwn; + protected boolean javaRaw; protected String keyFormat; protected String valueFormat; protected PackOutputStream keyPacker; @@ -609,13 +618,14 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; swigCPtr = cPtr; keyFormat = getKey_format(); valueFormat = getValue_format(); - keyPacker = new PackOutputStream(keyFormat); - valuePacker = new PackOutputStream(valueFormat); + javaRaw = _java_raw(); + keyPacker = new PackOutputStream(keyFormat, javaRaw); + valuePacker = new PackOutputStream(valueFormat, javaRaw); wiredtigerJNI.AsyncOp__java_init(swigCPtr, this, this); } protected static long getCPtr($javaclassname obj) { - return (obj == null) ? 0 : obj.swigCPtr; + return ((obj == null) ? 0 : obj.swigCPtr); } %} @@ -1098,7 +1108,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; if (keyUnpacker == null) keyUnpacker = new PackInputStream(keyFormat, get_key_wrap(), - _java_raw()); + javaRaw); return keyUnpacker; } @@ -1112,7 +1122,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; if (valueUnpacker == null) valueUnpacker = new PackInputStream(valueFormat, get_value_wrap(), - _java_raw()); + javaRaw); return valueUnpacker; } @@ -1127,7 +1137,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; k.data = NULL; if ((ret = $self->get_key($self, &k)) != 0) throwWiredTigerException(jenv, ret); - return k; + return (k); } %javamethodmodifiers get_value_wrap "protected"; @@ -1137,20 +1147,20 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; v.data = NULL; if ((ret = $self->get_value($self, &v)) != 0) throwWiredTigerException(jenv, ret); - return v; + return (v); } %javamethodmodifiers insert_wrap "protected"; int insert_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->insert($self); + return ($self->insert($self)); } %javamethodmodifiers remove_wrap "protected"; int remove_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->remove($self); + return ($self->remove($self)); } %javamethodmodifiers reset_wrap "protected"; @@ -1161,7 +1171,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %javamethodmodifiers search_wrap "protected"; int search_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->search($self); + return ($self->search($self)); } %javamethodmodifiers search_near_wrap "protected"; @@ -1181,7 +1191,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int update_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->update($self); + return ($self->update($self)); } %javamethodmodifiers compare_wrap "protected"; @@ -1189,7 +1199,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int cmp, ret = $self->compare($self, other, &cmp); if (ret != 0) throwWiredTigerException(jenv, ret); - return cmp; + return (cmp); } %javamethodmodifiers equals_wrap "protected"; @@ -1197,14 +1207,14 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int cmp, ret = $self->equals($self, other, &cmp); if (ret != 0) throwWiredTigerException(jenv, ret); - return cmp; + return (cmp); } %javamethodmodifiers _java_raw "protected"; bool _java_raw(JNIEnv *jenv) { (void)jenv; JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private; - return jcb->cursor_raw; + return (jcb->cursor_raw); } %javamethodmodifiers _java_init "protected"; @@ -1232,8 +1242,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; swigCPtr = cPtr; keyFormat = getKey_format(); valueFormat = getValue_format(); - keyPacker = new PackOutputStream(keyFormat); - valuePacker = new PackOutputStream(valueFormat); + keyPacker = new PackOutputStream(keyFormat, _java_raw()); + valuePacker = new PackOutputStream(valueFormat, _java_raw()); wiredtigerJNI.Cursor__java_init(swigCPtr, this, this); } @@ -1953,7 +1963,7 @@ WT_CONNECTION *wiredtiger_open_wrap(JNIEnv *jenv, const char *home, const char * err: if (ret != 0) throwWiredTigerException(jenv, ret); - return conn; + return (conn); } } @@ -1983,7 +1993,7 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return asyncop; + return (asyncop); } } @@ -2007,7 +2017,7 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return session; + return (session); } } @@ -2035,14 +2045,14 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return cursor; + return (cursor); } } %extend __wt_async_op { long get_id_wrap(JNIEnv *jenv) { WT_UNUSED(jenv); - return (self->get_id(self)); + return ((long)self->get_id(self)); } } @@ -2053,6 +2063,6 @@ err: if (ret != 0) ret = self->transaction_pinned_range(self, &range); if (ret != 0) throwWiredTigerException(jenv, ret); - return range; + return ((long)range); } } diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am index 03c65a57028..b32d0321194 100644 --- a/src/third_party/wiredtiger/lang/python/Makefile.am +++ b/src/third_party/wiredtiger/lang/python/Makefile.am @@ -17,7 +17,8 @@ install-exec-local: (cd $(PYSRC) && \ $(PYTHON) setup.py build_py -d $(abs_builddir)/build && \ $(PYTHON) setup.py build_ext -f -b $(abs_builddir)/build $(PYDIRS) && \ - $(PYTHON) setup.py install_lib -b $(abs_builddir)/build --skip-build $(PYTHON_INSTALL_ARG)) + $(PYTHON) setup.py install_lib -b $(abs_builddir)/build --skip-build $(PYTHON_INSTALL_ARG) && \ + rm -rf $(abs_builddir)/build) # We build in different places for an install vs running from the tree: # clean up both. Don't rely on "setup.py clean" -- everything that should diff --git a/src/third_party/wiredtiger/lang/python/setup.py b/src/third_party/wiredtiger/lang/python/setup.py index cf5b0c5cab1..9063a891fb9 100644 --- a/src/third_party/wiredtiger/lang/python/setup.py +++ b/src/third_party/wiredtiger/lang/python/setup.py @@ -35,8 +35,9 @@ from distutils.core import setup, Extension if not 'ARCHFLAGS' in os.environ: os.environ['ARCHFLAGS'] = '' -# Suppress warnings building SWIG generated code -extra_cflags = [ '-w', '-I../../src/include'] +# Suppress warnings building SWIG generated code. SWIG boiler plate +# functions have sign conversion warnings, so those warnings must be disabled. +extra_cflags = [ '-w', '-I../../src/include', '-Wno-sign-conversion'] dir = os.path.dirname(__file__) diff --git a/src/third_party/wiredtiger/lang/python/setup_pip.py b/src/third_party/wiredtiger/lang/python/setup_pip.py new file mode 100644 index 00000000000..636eecab80a --- /dev/null +++ b/src/third_party/wiredtiger/lang/python/setup_pip.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# This script builds a Python source distribution that can built be installed +# via pip install. This must be run in a git repository to determine the files +# to package. Also as a prerequisite, SWIG must be run as the generated files +# are part of the package. To create the distribution, in this directory, run +# "python setup_pip.py sdist", this creates a tar.gz file under ./dist . +from __future__ import print_function +import os, os.path, re, shutil, site, sys +from setuptools import setup, Distribution +from distutils.extension import Extension +import distutils.sysconfig +import distutils.ccompiler +from distutils.errors import CompileError, LinkError +import subprocess +from subprocess import call +import setuptools.command.install +import setuptools.command.build_ext + +# msg -- +# Print a message to stderr. +def msg(s): + print(os.path.basename(__file__) + ": " + s, file=sys.stderr) + +# die -- +# For failures, show a message and exit. +def die(s): + msg(s) + sys.exit(1) + +# build_commands -- +# Run a sequence of commands, and die if any fail. +def build_commands(commands, build_dir, build_env): + for command in commands: + callargs = [ 'sh', '-c', command ] + verbose_command = '"' + '" "'.join(callargs) + '"' + print('running: ' + verbose_command) + if call(callargs, cwd=build_dir, env=build_env) != 0: + die('build command failed: ' + verbose_command) + +# check_needed_dependencies -- +# Make a quick check of any needed library dependencies, and +# add to the library path and include path as needed. If a library +# is not found, it is not definitive. +def check_needed_dependencies(builtins, inc_paths, lib_paths): + library_dirs = get_library_dirs() + compiler = distutils.ccompiler.new_compiler() + distutils.sysconfig.customize_compiler(compiler) + compiler.set_library_dirs(library_dirs) + missing = [] + for name, libname, instructions in builtins: + found = compiler.find_library_file(library_dirs, libname) + if found is None: + msg(libname + ": missing") + msg(instructions) + msg("after installing it, set LD_LIBRARY_PATH or DYLD_LIBRARY_PATH") + missing.append(libname) + else: + package_top = os.path.dirname(os.path.dirname(found)) + inc_paths.append(os.path.join(package_top, 'include')) + lib_paths.append(os.path.join(package_top, 'lib')) + + # XXX: we are not accounting for other directories that might be + # discoverable via /sbin/ldconfig. It might be better to write a tiny + # compile using -lsnappy, -lz... + # + #if len(missing) > 0: + # die("install packages for: " + str(missing)) + +# find_executable -- +# Locate an executable in the PATH. +def find_executable(exename, path): + p = subprocess.Popen(['which', exename ], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate('') + out = str(out) # needed for Python3 + if out == '': + if err != '': + err = ': "' + err + '"' + die('"' + exename + '": not found in path' + err) + dirname = os.path.dirname(out) + if not dirname in path: + path.append(dirname) + +# get_build_path -- +# Create a PATH that can be used for installation. Apparently, +# installation commands are run with a restricted PATH, and +# autoreconf/aclocal will not normally be found. +def get_build_path(): + build_paths = [] + find_executable('autoreconf', build_paths) + find_executable('aclocal', build_paths) + build_path = os.environ['PATH'] + ':' + ':'.join(build_paths) + return build_path + +# get_compile_flags -- +# Get system specific compile flags. Return a triple: C preprocessor +# flags, C compilation flags and linker flags. +def get_compile_flags(inc_paths, lib_paths): + # Suppress warnings building SWIG generated code + if sys.platform == 'win32' and cc == 'msvc': + cflags = ['/arch:SSE2', '/EHsc'] + cppflags = [] + ldflags = [] + # Windows untested and incomplete, don't claim that it works. + die('Windows is not supported by this setup script') + else: + cflags = [ '-w', '-Wno-sign-conversion', '-std=c11' ] + cppflags = ['-I' + path for path in inc_paths] + cppflags.append('-DHAVE_CONFIG_H') + ldflags = ['-L' + path for path in lib_paths] + if sys.platform == 'darwin': + cflags.extend([ '-arch', 'x86_64' ]) + return (cppflags, cflags, ldflags) + +# get_sources_curdir -- +# Get a list of sources from the current directory +def get_sources_curdir(): + DEVNULL = open(os.devnull, 'w') + gitproc = subprocess.Popen( + ['git', 'ls-tree', '-r', '--name-only', 'HEAD^{tree}'], + stdin=DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sources = [line.rstrip() for line in gitproc.stdout.readlines()] + err = gitproc.stderr.read() + gitproc.wait() + subret = gitproc.returncode + if subret != 0 or err: + msg("git command to get sources returned " + str(subret) + + ", error=" + str(err)) + die("this command must be run in a git repository") + return sources + +# get_wiredtiger_versions -- +# Read the version information from the RELEASE_INFO file. +def get_wiredtiger_versions(wt_dir): + v = {} + for l in open(os.path.join(wt_dir, 'RELEASE_INFO')): + if re.match(r'WIREDTIGER_VERSION_(?:MAJOR|MINOR|PATCH)=', l): + exec(l, v) + wt_ver = '%d.%d' % (v['WIREDTIGER_VERSION_MAJOR'], + v['WIREDTIGER_VERSION_MINOR']) + wt_full_ver = wt_ver + '.%d' % (v['WIREDTIGER_VERSION_PATCH']) + return (wt_ver, wt_full_ver) + +# get_library_dirs +# Build a plausible set of library directories. +def get_library_dirs(): + dirs = [] + dirs.append("/usr/local/lib") + dirs.append("/usr/local/lib64") + dirs.append("/lib/x86_64-linux-gnu") + dirs.append("/opt/local/lib") + dirs.append("/usr/lib") + dirs.append("/usr/lib64") + for path in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'LIBRARY_PATH']: + if path in os.environ: + dirs.extend(os.environ[path].split(':')) + dirs = list(set(filter(os.path.isdir, dirs))) + return dirs + +# source_filter +# Make any needed changes to the sources list. Any entry that +# needs to be moved is returned in a dictionary. +def source_filter(sources): + result = [] + movers = dict() + py_dir = os.path.join('lang', 'python') + pywt_dir = os.path.join(py_dir, 'wiredtiger') + pywt_prefix = pywt_dir + os.path.sep + for f in sources: + if not re.match(source_regex, f): + continue + src = f + dest = f + # move all lang/python files to the top level. + if dest.startswith(pywt_prefix): + dest = os.path.basename(dest) + if dest == 'pip_init.py': + dest = '__init__.py' + if dest != src: + movers[dest] = src + result.append(dest) + # Add SWIG generated files + result.append('wiredtiger.py') + movers['wiredtiger.py'] = os.path.join(pywt_dir, '__init__.py') + result.append(os.path.join(py_dir, 'wiredtiger_wrap.c')) + return result, movers + +################################################################ +# Do some initial setup and checks. +this_abs_script = os.path.abspath(__file__) +this_dir = os.path.dirname(this_abs_script) +pip_command = None +for arg in sys.argv[1:]: + if arg[0] != '-' and pip_command == None: + pip_command = arg + break + +if this_dir.endswith(os.sep + os.path.join('lang', 'python')): + wt_dir = os.path.dirname(os.path.dirname(this_dir)) + os.chdir(wt_dir) +elif os.path.isfile(os.path.join(this_dir, 'LICENSE')): + wt_dir = this_dir +else: + die('running from an unknown directory') + +python3 = (sys.version_info[0] > 2) +if python3: + die('Python3 is not yet supported') + +# Ensure that Extensions won't be built for 32 bit, +# that won't work with WiredTiger. +if sys.maxsize < 2**32: + die('need to be running on a 64 bit system, and have a 64 bit Python') + +python_rel_dir = os.path.join('lang', 'python') +build_dir = os.path.join(wt_dir, 'build_posix') +makefile = os.path.join(build_dir, 'Makefile') +built_sentinal = os.path.join(build_dir, 'built.txt') +conf_make_dir = 'build_posix' +wt_swig_lib_name = os.path.join(python_rel_dir, '_wiredtiger.so') + +################################################################ +# Put together build options for the WiredTiger extension. +short_description = 'high performance, scalable, production quality, ' + \ + 'NoSQL, Open Source extensible platform for data management' +long_description = 'WiredTiger is a ' + short_description + '.\n\n' + \ + open(os.path.join(wt_dir, 'README')).read() + +wt_ver, wt_full_ver = get_wiredtiger_versions(wt_dir) +build_path = get_build_path() + +# We only need a small set of directories to build a WT library, +# we also include any files at the top level. +source_regex = r'^(?:(?:api|build_posix|ext|lang/python|src|dist)/|[^/]*$)' + +# The builtins that we include in this distribution. +builtins = [ + # [ name, libname, instructions ] + [ 'snappy', 'snappy', + 'Note: a suitable version of snappy can be found at\n' + \ + ' https://github.com/google/snappy/releases/download/' + \ + '1.1.3/snappy-1.1.3.tar.gz\n' + \ + 'It can be installed via: yum install snappy snappy-devel' + \ + 'or via: apt-get install libsnappy-dev' ], + [ 'zlib', 'z', + 'Need to install zlib\n' + \ + 'It can be installed via: apt-get install zlib1g' ] +] +builtin_names = [b[0] for b in builtins] +builtin_libraries = [b[1] for b in builtins] + +# Here's the configure/make operations we perform before the python extension +# is linked. +configure_cmds = [ + './makemake --clean-and-make', + './reconf', + # force building a position independent library; it will be linked + # into a single shared library with the SWIG interface code. + 'CFLAGS="${CFLAGS:-} -fPIC -DPIC" ' + \ + '../configure --enable-python --with-builtins=' + ','.join(builtin_names) +] + +# build all the builtins, at the moment they are all compressors. +make_cmds = [] +for name in builtin_names: + make_cmds.append('(cd ext/compressors/' + name + '/; make)') +make_cmds.append('make libwiredtiger.la') + +inc_paths = [ os.path.join(build_dir, 'src', 'include'), build_dir, '.' ] +lib_paths = [ '.' ] # wiredtiger.so is moved into the top level directory + +check_needed_dependencies(builtins, inc_paths, lib_paths) + +cppflags, cflags, ldflags = get_compile_flags(inc_paths, lib_paths) + +# If we are creating a source distribution, create a staging directory +# with just the right sources. Put the result in the python dist directory. +if pip_command == 'sdist': + sources, movers = source_filter(get_sources_curdir()) + stage_dir = os.path.join(python_rel_dir, 'stage') + shutil.rmtree(stage_dir, True) + os.makedirs(stage_dir) + shutil.copy2(this_abs_script, os.path.join(stage_dir, 'setup.py')) + for f in sources: + d = os.path.join(stage_dir, os.path.dirname(f)) + if not os.path.isdir(d): + os.makedirs(d) + if f in movers: + src = movers[f] + else: + src = f + # Symlinks are not followed in setup, we need to use real files. + shutil.copy2(src, os.path.join(stage_dir, f)) + os.chdir(stage_dir) + sys.argv.append('--dist-dir=' + os.path.join('..', 'dist')) +else: + sources = [ os.path.join(python_rel_dir, 'wiredtiger_wrap.c') ] + +wt_ext = Extension('_wiredtiger', + sources = sources, + extra_compile_args = cflags + cppflags, + extra_link_args = ldflags, + libraries = builtin_libraries, + extra_objects = [ os.path.join(build_dir, '.libs', 'libwiredtiger.a') ], + include_dirs = inc_paths, + library_dirs = lib_paths, +) +extensions = [ wt_ext ] +env = { "CFLAGS" : ' '.join(cflags), + "CPPFLAGS" : ' '.join(cppflags), + "LDFLAGS" : ' '.join(ldflags), + "PATH" : build_path } + +class BinaryDistribution(Distribution): + def is_pure(self): + return False + +class WTInstall(setuptools.command.install.install): + def run(self): + self.run_command("build_ext") + return setuptools.command.install.install.run(self) + +class WTBuildExt(setuptools.command.build_ext.build_ext): + def __init__(self, *args, **kwargs): + setuptools.command.build_ext.build_ext.__init__(self, *args, **kwargs) + + def run(self): + # only run this once + if not os.path.isfile(built_sentinal): + try: + os.remove(makefile) + except OSError: + pass + self.execute( + lambda: build_commands(configure_cmds, conf_make_dir, env), [], + 'wiredtiger configure') + if not os.path.isfile(makefile): + die('configure failed, file does not exist: ' + makefile) + self.execute( + lambda: build_commands(make_cmds, conf_make_dir, env), [], + 'wiredtiger make') + open(built_sentinal, 'a').close() + return setuptools.command.build_ext.build_ext.run(self) + +setup( + name = 'wiredtiger', + version = wt_full_ver, + author = 'The WiredTiger Development Team, part of MongoDB', + author_email = 'info@wiredtiger.com', + description = short_description, + license='GPL2,GPL3,Commercial', + long_description = long_description, + url = 'http://source.wiredtiger.com/', + keywords = 'scalable NoSQL database datastore engine open source', + packages = ['wiredtiger'], + ext_package = 'wiredtiger', + ext_modules = extensions, + include_package_data = True, + distclass = BinaryDistribution, + package_dir = { 'wiredtiger' : '.' }, + cmdclass = { 'install': WTInstall, 'build_ext': WTBuildExt }, + package_data = { + 'wiredtiger' : [ wt_swig_lib_name, '*.py' ] + }, + classifiers=[ + 'Intended Audience :: Developers', + 'Programming Language :: C', + 'Programming Language :: C++', + 'Programming Language :: Python', + 'Programming Language :: Java', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: BSD', + 'Operating System :: POSIX :: Linux', + 'Operating System :: POSIX :: SunOS/Solaris', + ] +) + +if pip_command == 'sdist': + shutil.rmtree(os.path.join(this_dir, 'stage')) diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py b/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py new file mode 100644 index 00000000000..d59c8218976 --- /dev/null +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# pip_init.py +# This is installed as __init__.py, and imports the file created by SWIG. +# This is needed because SWIG's import helper code created by certain SWIG +# versions may be broken, see: https://github.com/swig/swig/issues/769 . +# Importing indirectly seems to avoid these issues. +import os, sys +fname = os.path.basename(__file__) +if fname != '__init__.py' and fname != '__init__.pyc': + print(__file__ + ': this file is not yet installed') + sys.exit(1) + +# After importing the SWIG-generated file, copy all symbols from from it +# to this module so they will appear in the wiredtiger namespace. +me = sys.modules[__name__] +sys.path.append(os.path.dirname(__file__)) # needed for Python3 +import wiredtiger +for name in dir(wiredtiger): + value = getattr(wiredtiger, name) + setattr(me, name, value) diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index 54bcb7cd26c..3319332aa04 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -240,8 +240,7 @@ __async_start(WT_SESSION_IMPL *session) async = conn->async; TAILQ_INIT(&async->formatqh); WT_RET(__wt_spin_init(session, &async->ops_lock, "ops")); - WT_RET(__wt_cond_alloc( - session, "async flush", false, &async->flush_cond)); + WT_RET(__wt_cond_alloc(session, "async flush", &async->flush_cond)); WT_RET(__wt_async_op_init(session)); /* @@ -339,17 +338,15 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) * 2. If async is off, and the user wants it on, start it. * 3. If not a toggle and async is off, we're done. */ - if (conn->async_cfg && !run) { - /* Case 1 */ + if (conn->async_cfg && !run) { /* Case 1 */ WT_TRET(__wt_async_flush(session)); ret = __wt_async_destroy(session); conn->async_cfg = false; return (ret); - } else if (!conn->async_cfg && run) - /* Case 2 */ + } + if (!conn->async_cfg && run) /* Case 2 */ return (__async_start(session)); - else if (!conn->async_cfg) - /* Case 3 */ + if (!conn->async_cfg) /* Case 3 */ return (0); /* @@ -443,7 +440,7 @@ __wt_async_destroy(WT_SESSION_IMPL *session) session, async->worker_tids[i])); async->worker_tids[i] = 0; } - WT_TRET(__wt_cond_destroy(session, &async->flush_cond)); + __wt_cond_destroy(session, &async->flush_cond); /* Close the server threads' sessions. */ for (i = 0; i < conn->async_workers; i++) @@ -541,7 +538,7 @@ retry: async->flush_op.state = WT_ASYNCOP_READY; WT_RET(__wt_async_op_enqueue(session, &async->flush_op)); while (async->flush_state != WT_ASYNC_FLUSH_COMPLETE) - __wt_cond_wait(session, async->flush_cond, 100000); + __wt_cond_wait(session, async->flush_cond, 100000, NULL); /* * Flush is done. Clear the flags. */ diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index 8c074e503d9..6908802dbff 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -254,7 +254,6 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) { WT_ASYNC *async; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; uint64_t cur_head, cur_tail, my_alloc, my_slot; #ifdef HAVE_DIAGNOSTIC WT_ASYNC_OP_IMPL *my_op; @@ -307,7 +306,7 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) WT_ORDERED_READ(cur_head, async->head); } WT_PUBLISH(async->head, my_alloc); - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index b1bc3902f7c..11f59ed14f1 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -107,7 +107,7 @@ __async_flush_wait(WT_SESSION_IMPL *session, WT_ASYNC *async, uint64_t my_gen) { while (async->flush_state == WT_ASYNC_FLUSHING && async->flush_gen == my_gen) - __wt_cond_wait(session, async->flush_cond, 10000); + __wt_cond_wait(session, async->flush_cond, 10000, NULL); } /* diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index dadd17de4a6..a67efca62a3 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -192,6 +192,7 @@ __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; @@ -225,7 +226,7 @@ __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; WT_RET(__wt_vpack_uint(pp, 0, a)); - a = (uint64_t)ci->ckpt_size; + a = ci->ckpt_size; WT_RET(__wt_vpack_uint(pp, 0, a)); return (0); diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 48522768dc9..05e4dcc098e 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -53,7 +53,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_DECL_RET; uint8_t *endp; - WT_UNUSED(addr_size); ci = NULL; /* diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 6d5d48bdb22..eb6647dd03c 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -22,6 +22,12 @@ __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) /* Switch to first-fit allocation. */ __wt_block_configure_first_fit(block, true); + /* Reset the compaction state information. */ + block->compact_pct_tenths = 0; + block->compact_pages_reviewed = 0; + block->compact_pages_skipped = 0; + block->compact_pages_written = 0; + return (0); } @@ -55,7 +61,6 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) { - WT_DECL_RET; WT_EXT *ext; WT_EXTLIST *el; wt_off_t avail_eighty, avail_ninety, eighty, ninety; @@ -71,16 +76,6 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) if (block->size <= WT_MEGABYTE) return (0); - /* - * Reset the compaction state information. This is done here, not in the - * compaction "start" routine, because this function is called first to - * determine if compaction is useful. - */ - block->compact_pct_tenths = 0; - block->compact_pages_reviewed = 0; - block->compact_pages_skipped = 0; - block->compact_pages_written = 0; - __wt_spin_lock(session, &block->live_lock); /* Dump the current state of the file. */ @@ -142,7 +137,7 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) __wt_spin_unlock(session, &block->live_lock); - return (ret); + return (0); } /* @@ -153,7 +148,6 @@ int __wt_block_compact_page_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) { - WT_DECL_RET; WT_EXT *ext; WT_EXTLIST *el; wt_off_t limit, offset; @@ -198,7 +192,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, } #endif - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 26acc8c560f..da7a06d873d 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -634,11 +634,11 @@ __wt_block_off_free( */ if ((ret = __wt_block_off_remove_overlap( session, block, &block->live.alloc, offset, size)) == 0) - ret = __block_merge(session, block, - &block->live.avail, offset, (wt_off_t)size); + ret = __block_merge( + session, block, &block->live.avail, offset, size); else if (ret == WT_NOTFOUND) - ret = __block_merge(session, block, - &block->live.discard, offset, (wt_off_t)size); + ret = __block_merge( + session, block, &block->live.discard, offset, size); return (ret); } @@ -1247,7 +1247,8 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_DECL_RET; WT_EXT *ext; WT_PAGE_HEADER *dsk; - size_t entries, size; + uint32_t entries; + size_t size; uint8_t *p; WT_RET(__block_extlist_dump(session, block, el, "write")); @@ -1377,8 +1378,8 @@ __wt_block_extlist_init(WT_SESSION_IMPL *session, size = (name == NULL ? 0 : strlen(name)) + strlen(".") + (extname == NULL ? 0 : strlen(extname) + 1); WT_RET(__wt_calloc_def(session, size, &el->name)); - (void)snprintf(el->name, size, "%s.%s", - name == NULL ? "" : name, extname == NULL ? "" : extname); + WT_RET(__wt_snprintf(el->name, size, "%s.%s", + name == NULL ? "" : name, extname == NULL ? "" : extname)); el->offset = WT_BLOCK_INVALID_OFFSET; el->track_size = track_size; diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 4d15942709a..07ceb4c8159 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -201,6 +201,12 @@ __wt_block_open(WT_SESSION_IMPL *session, * "direct_io=checkpoint" configures direct I/O for readonly data files. */ flags = 0; + WT_ERR(__wt_config_gets(session, cfg, "access_pattern_hint", &cval)); + if (WT_STRING_MATCH("random", cval.str, cval.len)) + LF_SET(WT_FS_OPEN_ACCESS_RAND); + else if (WT_STRING_MATCH("sequential", cval.str, cval.len)) + LF_SET(WT_FS_OPEN_ACCESS_SEQ); + if (readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_CHECKPOINT)) LF_SET(WT_FS_OPEN_DIRECTIO); if (!readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_DATA)) diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 1eeabd63d92..8d4aec7df75 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -24,8 +24,6 @@ __wt_bm_preload( uint32_t checksum, size; bool mapped; - WT_UNUSED(addr_size); - block = bm->block; WT_STAT_CONN_INCR(session, block_preload); @@ -41,7 +39,7 @@ __wt_bm_preload( (uint8_t *)bm->map + offset, size, bm->mapped_cookie); if (!mapped && handle->fh_advise != NULL) ret = handle->fh_advise(handle, (WT_SESSION *)session, - (wt_off_t)offset, (wt_off_t)size, WT_FILE_HANDLE_WILLNEED); + offset, (wt_off_t)size, WT_FILE_HANDLE_WILLNEED); if (ret != EBUSY && ret != ENOTSUP) return (ret); diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 5ba95bb598e..b06a5062f50 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -168,7 +168,6 @@ __wt_block_salvage_valid(WT_SESSION_IMPL *session, wt_off_t offset; uint32_t size, checksum; - WT_UNUSED(session); WT_UNUSED(addr_size); /* diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 94824ad19f8..154765ed079 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -22,7 +22,7 @@ static int __verify_set_file_size(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *); ((off) / (block)->allocsize - 1) #ifdef HAVE_VERBOSE #define WT_FRAG_TO_OFF(block, frag) \ - (((wt_off_t)(frag + 1)) * (block)->allocsize) + (((wt_off_t)((frag) + 1)) * (block)->allocsize) #endif /* diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index d08aba45920..ea7859d6a38 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -43,10 +43,10 @@ __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len) * more targeted solution at some point. */ if (!conn->hot_backup) { - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (!conn->hot_backup) ret = __wt_ftruncate(session, block->fh, len); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); } /* diff --git a/src/third_party/wiredtiger/src/bloom/bloom.c b/src/third_party/wiredtiger/src/bloom/bloom.c index dde08d8a982..b8d75678835 100644 --- a/src/third_party/wiredtiger/src/bloom/bloom.c +++ b/src/third_party/wiredtiger/src/bloom/bloom.c @@ -37,8 +37,8 @@ __bloom_init(WT_SESSION_IMPL *session, len += strlen(config); WT_ERR(__wt_calloc_def(session, len, &bloom->config)); /* Add the standard config at the end, so it overrides user settings. */ - (void)snprintf(bloom->config, len, - "%s,%s", config == NULL ? "" : config, WT_BLOOM_TABLE_CONFIG); + WT_ERR(__wt_snprintf(bloom->config, len, + "%s,%s", config == NULL ? "" : config, WT_BLOOM_TABLE_CONFIG)); bloom->session = session; @@ -95,6 +95,7 @@ int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_BLOOM *bloom; WT_DECL_RET; @@ -148,6 +149,7 @@ int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_BLOOM *bloom; WT_CURSOR *c; @@ -178,6 +180,7 @@ err: WT_TRET(__wt_bloom_close(bloom)); */ void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { uint64_t h1, h2; uint32_t i; @@ -195,6 +198,7 @@ __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) */ int __wt_bloom_finalize(WT_BLOOM *bloom) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_CURSOR *c; WT_DECL_RET; @@ -304,6 +308,7 @@ err: /* Don't return WT_NOTFOUND from a failed search. */ */ int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_BLOOM_HASH bhash; @@ -363,6 +368,7 @@ __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) */ int __wt_bloom_close(WT_BLOOM *bloom) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; WT_SESSION_IMPL *session; @@ -385,6 +391,7 @@ __wt_bloom_close(WT_BLOOM *bloom) */ int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; WT_SESSION *wt_session; diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index e005674762d..4b40d0aeed0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -60,7 +60,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) */ if (mod->rec_result == WT_PM_REC_REPLACE || mod->rec_result == WT_PM_REC_MULTIBLOCK) - __wt_writelock(session, &page->page_lock); + WT_PAGE_LOCK(session, page); if (mod->rec_result == WT_PM_REC_REPLACE) ret = bm->compact_page_skip(bm, session, @@ -80,7 +80,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) if (mod->rec_result == WT_PM_REC_REPLACE || mod->rec_result == WT_PM_REC_MULTIBLOCK) - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); return (ret); } @@ -90,16 +90,15 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) * Compact a file. */ int -__wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_compact(WT_SESSION_IMPL *session) { WT_BM *bm; WT_BTREE *btree; WT_DECL_RET; WT_REF *ref; + u_int i; bool skip; - WT_UNUSED(cfg); - btree = S2BT(session); bm = btree->bm; ref = NULL; @@ -129,7 +128,13 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) __wt_spin_lock(session, &btree->flush_lock); /* Walk the tree reviewing pages to see if they should be re-written. */ - for (;;) { + for (i = 0;;) { + /* Periodically check if we've run out of time. */ + if (++i > 100) { + WT_ERR(__wt_session_compact_check_timeout(session)); + i = 0; + } + /* * Pages read for compaction aren't "useful"; don't update the * read generation of pages already in memory, and if a page is diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 4d3976f9647..21e575ffca9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -338,7 +338,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { } /* Check for the end of the page. */ - if (cbt->row_iteration_slot >= page->pg_row_entries * 2 + 1) + if (cbt->row_iteration_slot >= page->entries * 2 + 1) return (WT_NOTFOUND); ++cbt->row_iteration_slot; @@ -356,7 +356,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { cbt->ins = NULL; cbt->slot = cbt->row_iteration_slot / 2 - 1; - rip = &page->pg_row_d[cbt->slot]; + rip = &page->pg_row[cbt->slot]; upd = __wt_txn_read(session, WT_ROW_UPDATE(page, rip)); if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) { if (__wt_txn_visible_all(session, upd->txnid)) @@ -579,20 +579,20 @@ __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt) int __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) { + WT_CURSOR *cursor; WT_DECL_RET; WT_PAGE *page; WT_SESSION_IMPL *session; uint32_t flags; bool newpage; + cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cbt->iface.session; WT_STAT_CONN_INCR(session, cursor_next); WT_STAT_DATA_INCR(session, cursor_next); - flags = WT_READ_SKIP_INTL; /* Tree walk flags. */ - if (truncating) - LF_SET(WT_READ_TRUNCATE); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); WT_RET(__cursor_func_init(cbt, false)); @@ -608,6 +608,9 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) * found. Then, move to the next page, until we reach the end of the * file. */ + flags = WT_READ_SKIP_INTL; /* tree walk flags */ + if (truncating) + LF_SET(WT_READ_TRUNCATE); for (newpage = false;; newpage = true) { page = cbt->ref == NULL ? NULL : cbt->ref->page; @@ -676,6 +679,8 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) if (ret == 0) WT_ERR(__wt_cursor_key_order_check(session, cbt, true)); #endif + if (ret == 0) + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); err: if (ret != 0) WT_TRET(__cursor_reset(cbt)); diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 2dd443ffac1..bf4bdad6529 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -458,13 +458,13 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage) if (!F_ISSET_ATOMIC(page, WT_PAGE_BUILD_KEYS)) WT_RET(__wt_row_leaf_keys(session, page)); - if (page->pg_row_entries == 0) + if (page->entries == 0) cbt->ins_head = WT_ROW_INSERT_SMALLEST(page); else cbt->ins_head = - WT_ROW_INSERT_SLOT(page, page->pg_row_entries - 1); + WT_ROW_INSERT_SLOT(page, page->entries - 1); cbt->ins = WT_SKIP_LAST(cbt->ins_head); - cbt->row_iteration_slot = page->pg_row_entries * 2 + 1; + cbt->row_iteration_slot = page->entries * 2 + 1; cbt->rip_saved = NULL; goto new_insert; } @@ -515,7 +515,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { cbt->ins = NULL; cbt->slot = cbt->row_iteration_slot / 2 - 1; - rip = &page->pg_row_d[cbt->slot]; + rip = &page->pg_row[cbt->slot]; upd = __wt_txn_read(session, WT_ROW_UPDATE(page, rip)); if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) { if (__wt_txn_visible_all(session, upd->txnid)) @@ -535,20 +535,20 @@ new_insert: if ((ins = cbt->ins) != NULL) { int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) { + WT_CURSOR *cursor; WT_DECL_RET; WT_PAGE *page; WT_SESSION_IMPL *session; uint32_t flags; bool newpage; + cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cbt->iface.session; WT_STAT_CONN_INCR(session, cursor_prev); WT_STAT_DATA_INCR(session, cursor_prev); - flags = WT_READ_PREV | WT_READ_SKIP_INTL; /* Tree walk flags. */ - if (truncating) - LF_SET(WT_READ_TRUNCATE); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); WT_RET(__cursor_func_init(cbt, false)); @@ -564,6 +564,9 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) * found. Then, move to the previous page, until we reach the start * of the file. */ + flags = WT_READ_PREV | WT_READ_SKIP_INTL; /* tree walk flags */ + if (truncating) + LF_SET(WT_READ_TRUNCATE); for (newpage = false;; newpage = true) { page = cbt->ref == NULL ? NULL : cbt->ref->page; @@ -631,6 +634,8 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) if (ret == 0) WT_ERR(__wt_cursor_key_order_check(session, cbt, false)); #endif + if (ret == 0) + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); err: if (ret != 0) WT_TRET(__cursor_reset(cbt)); diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 0454ea7c20e..f0aa632551b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -8,6 +8,84 @@ #include "wt_internal.h" +/* + * When returning an error, we need to restore the cursor to a valid state, the + * upper-level cursor code is likely to retry. This structure and the associated + * functions are used save and restore the cursor state. + */ +typedef struct { + WT_ITEM key; + WT_ITEM value; + uint64_t recno; + uint32_t flags; +} WT_CURFILE_STATE; + +/* + * __cursor_state_save -- + * Save the cursor's external state. + */ +static inline void +__cursor_state_save(WT_CURSOR *cursor, WT_CURFILE_STATE *state) +{ + WT_ITEM_SET(state->key, cursor->key); + WT_ITEM_SET(state->value, cursor->value); + state->recno = cursor->recno; + state->flags = cursor->flags; +} + +/* + * __cursor_state_restore -- + * Restore the cursor's external state. + */ +static inline void +__cursor_state_restore(WT_CURSOR *cursor, WT_CURFILE_STATE *state) +{ + if (F_ISSET(state, WT_CURSTD_KEY_EXT)) + WT_ITEM_SET(cursor->key, state->key); + if (F_ISSET(state, WT_CURSTD_VALUE_EXT)) + WT_ITEM_SET(cursor->value, state->value); + cursor->recno = state->recno; + F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + F_SET(cursor, F_MASK(state, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT)); + +} + +/* + * __cursor_page_pinned -- + * Return if we have a page pinned and it's not been flagged for forced + * eviction (the forced eviction test is so we periodically release pages + * grown too large). + */ +static inline bool +__cursor_page_pinned(WT_CURSOR_BTREE *cbt) +{ + return (F_ISSET(cbt, WT_CBT_ACTIVE) && + cbt->ref->page->read_gen != WT_READGEN_OLDEST); +} + +/* + * __cursor_copy_int_key -- + * If we're pointing into the tree, save the key into local memory. + */ +static inline int +__cursor_copy_int_key(WT_CURSOR *cursor) +{ + /* + * We're about to discard the cursor's position and the cursor layer + * might retry the operation. We discard pinned pages on error, which + * will invalidate pinned keys. Clear WT_CURSTD_KEY_INT in all cases, + * the underlying page is gone whether we can allocate memory or not. + */ + if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + F_CLR(cursor, WT_CURSTD_KEY_INT); + if (!WT_DATA_IN_ITEM(&cursor->key)) + WT_RET(__wt_buf_set((WT_SESSION_IMPL *)cursor->session, + &cursor->key, cursor->key.data, cursor->key.size)); + F_SET(cursor, WT_CURSTD_KEY_EXT); + } + return (0); +} + /* * __cursor_size_chk -- * Return if an inserted item is too large. @@ -41,7 +119,7 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) if (kv->size > WT_BTREE_MAX_OBJECT_SIZE) WT_RET_MSG(session, EINVAL, "item size of %" WT_SIZET_FMT " exceeds the maximum " - "supported WiredTiger size of %d", + "supported WiredTiger size of %" PRIu32, kv->size, WT_BTREE_MAX_OBJECT_SIZE); /* Check what the block manager can actually write. */ @@ -54,6 +132,34 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) return (0); } +/* + * __cursor_disable_bulk -- + * Disable bulk loads into a tree. + */ +static inline void +__cursor_disable_bulk(WT_SESSION_IMPL *session, WT_BTREE *btree) +{ + /* + * Once a tree (other than the LSM primary) is no longer empty, eviction + * should pay attention to it, and it's no longer possible to bulk-load + * into it. + */ + if (!btree->original) + return; + if (btree->lsm_primary) { + btree->original = 0; /* Make the next test faster. */ + return; + } + + /* + * We use a compare-and-swap here to avoid races among the first inserts + * into a tree. Eviction is disabled when an empty tree is opened, and + * it must only be enabled once. + */ + if (__wt_atomic_cas8(&btree->original, 1, 0)) + __wt_evict_file_exclusive_off(session); +} + /* * __cursor_fix_implicit -- * Return if search went past the end of the tree. @@ -76,11 +182,11 @@ __cursor_fix_implicit(WT_BTREE *btree, WT_CURSOR_BTREE *cbt) } /* - * __cursor_valid -- + * __wt_cursor_valid -- * Return if the cursor references an valid key/value pair. */ -static inline bool -__cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) +bool +__wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) { WT_BTREE *btree; WT_CELL *cell; @@ -163,7 +269,7 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) * column-store pages don't have slots, but map one-to-one to * keys, check for retrieval past the end of the page. */ - if (cbt->recno >= cbt->ref->ref_recno + page->pg_fix_entries) + if (cbt->recno >= cbt->ref->ref_recno + page->entries) return (false); /* @@ -173,9 +279,9 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) break; case BTREE_COL_VAR: /* The search function doesn't check for empty pages. */ - if (page->pg_var_entries == 0) + if (page->entries == 0) return (false); - WT_ASSERT(session, cbt->slot < page->pg_var_entries); + WT_ASSERT(session, cbt->slot < page->entries); /* * Column-store updates are stored as "insert" objects. If @@ -191,16 +297,16 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) * backing store; check the cell for a record already deleted * when read. */ - cip = &page->pg_var_d[cbt->slot]; + cip = &page->pg_var[cbt->slot]; if ((cell = WT_COL_PTR(page, cip)) == NULL || __wt_cell_type(cell) == WT_CELL_DEL) return (false); break; case BTREE_ROW: /* The search function doesn't check for empty pages. */ - if (page->pg_row_entries == 0) + if (page->entries == 0) return (false); - WT_ASSERT(session, cbt->slot < page->pg_row_entries); + WT_ASSERT(session, cbt->slot < page->entries); /* * See above: for row-store, no insert object can have the same @@ -263,7 +369,7 @@ __cursor_col_modify( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool is_remove) { return (__wt_col_modify(session, - cbt, cbt->iface.recno, &cbt->iface.value, NULL, is_remove)); + cbt, cbt->iface.recno, &cbt->iface.value, NULL, is_remove, false)); } /* @@ -275,7 +381,7 @@ __cursor_row_modify( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool is_remove) { return (__wt_row_modify(session, - cbt, &cbt->iface.key, &cbt->iface.value, NULL, is_remove)); + cbt, &cbt->iface.key, &cbt->iface.value, NULL, is_remove, false)); } /* @@ -285,13 +391,17 @@ __cursor_row_modify( int __wt_btcur_reset(WT_CURSOR_BTREE *cbt) { + WT_CURSOR *cursor; WT_SESSION_IMPL *session; + cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cbt->iface.session; WT_STAT_CONN_INCR(session, cursor_reset); WT_STAT_DATA_INCR(session, cursor_reset); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + return (__cursor_reset(cbt)); } @@ -303,6 +413,7 @@ int __wt_btcur_search(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -317,20 +428,28 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); + __cursor_state_save(cursor, &state); + + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key, then re-save the cursor state. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + /* * If we have a page pinned, search it; if we don't have a page pinned, * or the search of the pinned page doesn't find an exact match, search * from the root. */ valid = false; - if (F_ISSET(cbt, WT_CBT_ACTIVE) && - cbt->ref->page->read_gen != WT_READGEN_OLDEST) { - WT_ERR(__wt_txn_cursor_op(session)); + if (__cursor_page_pinned(cbt)) { + __wt_txn_cursor_op(session); WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, cbt->ref, false) : __cursor_col_search(session, cbt, cbt->ref)); - valid = cbt->compare == 0 && __cursor_valid(cbt, &upd); + valid = cbt->compare == 0 && __wt_cursor_valid(cbt, &upd); } if (!valid) { WT_ERR(__cursor_func_init(cbt, true)); @@ -338,7 +457,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, NULL, false) : __cursor_col_search(session, cbt, NULL)); - valid = cbt->compare == 0 && __cursor_valid(cbt, &upd); + valid = cbt->compare == 0 && __wt_cursor_valid(cbt, &upd); } if (valid) @@ -352,6 +471,8 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) cbt->v = 0; cursor->value.data = &cbt->v; cursor->value.size = 1; + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); } else ret = WT_NOTFOUND; @@ -360,8 +481,10 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_ERR(__wt_cursor_key_order_init(session, cbt)); #endif -err: if (ret != 0) +err: if (ret != 0) { WT_TRET(__cursor_reset(cbt)); + __cursor_state_restore(cursor, &state); + } return (ret); } @@ -373,6 +496,7 @@ int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -389,6 +513,15 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_STAT_CONN_INCR(session, cursor_search_near); WT_STAT_DATA_INCR(session, cursor_search_near); + __cursor_state_save(cursor, &state); + + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key, then re-save the cursor state. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + /* * If we have a row-store page pinned, search it; if we don't have a * page pinned, or the search of the pinned page doesn't find an exact @@ -402,10 +535,8 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) * existing record. */ valid = false; - if (btree->type == BTREE_ROW && - F_ISSET(cbt, WT_CBT_ACTIVE) && - cbt->ref->page->read_gen != WT_READGEN_OLDEST) { - WT_ERR(__wt_txn_cursor_op(session)); + if (btree->type == BTREE_ROW && __cursor_page_pinned(cbt)) { + __wt_txn_cursor_op(session); WT_ERR(__cursor_row_search(session, cbt, cbt->ref, true)); @@ -418,16 +549,15 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) * might be legitimately positioned after the last page slot). * Ignore those cases, it makes things too complicated. */ - if (cbt->slot != 0 && - cbt->slot != cbt->ref->page->pg_row_entries - 1) - valid = __cursor_valid(cbt, &upd); + if (cbt->slot != 0 && cbt->slot != cbt->ref->page->entries - 1) + valid = __wt_cursor_valid(cbt, &upd); } if (!valid) { WT_ERR(__cursor_func_init(cbt, true)); WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, NULL, true) : __cursor_col_search(session, cbt, NULL)); - valid = __cursor_valid(cbt, &upd); + valid = __wt_cursor_valid(cbt, &upd); } /* @@ -456,6 +586,8 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) cursor->value.data = &cbt->v; cursor->value.size = 1; exact = 0; + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); } else if ((ret = __wt_btcur_next(cbt, false)) != WT_NOTFOUND) exact = 1; else { @@ -463,22 +595,25 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, NULL, true) : __cursor_col_search(session, cbt, NULL)); - if (__cursor_valid(cbt, &upd)) { + if (__wt_cursor_valid(cbt, &upd)) { exact = cbt->compare; ret = __wt_kv_return(session, cbt, upd); } else if ((ret = __wt_btcur_prev(cbt, false)) != WT_NOTFOUND) exact = -1; } +err: if (ret == 0 && exactp != NULL) + *exactp = exact; + #ifdef HAVE_DIAGNOSTIC if (ret == 0) - WT_ERR(__wt_cursor_key_order_init(session, cbt)); + WT_TRET(__wt_cursor_key_order_init(session, cbt)); #endif -err: if (ret != 0) + if (ret != 0) { WT_TRET(__cursor_reset(cbt)); - if (exactp != NULL && (ret == 0 || ret == WT_NOTFOUND)) - *exactp = exact; + __cursor_state_restore(cursor, &state); + } return (ret); } @@ -490,9 +625,11 @@ int __wt_btcur_insert(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; + bool append_key; btree = cbt->btree; cursor = &cbt->iface; @@ -503,30 +640,86 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCRV(session, cursor_insert_bytes, cursor->key.size + cursor->value.size); + __cursor_state_save(cursor, &state); + if (btree->type == BTREE_ROW) WT_RET(__cursor_size_chk(session, &cursor->key)); WT_RET(__cursor_size_chk(session, &cursor->value)); + /* It's no longer possible to bulk-load into the tree. */ + __cursor_disable_bulk(session, btree); + + /* + * Insert a new record if WT_CURSTD_APPEND configured, (ignoring any + * application set record number). Although append can't be configured + * for a row-store, this code would break if it were, and that's owned + * by the upper cursor layer, be cautious. + */ + append_key = + F_ISSET(cursor, WT_CURSTD_APPEND) && btree->type != BTREE_ROW; + /* - * The tree is no longer empty: eviction should pay attention to it, - * and it's no longer possible to bulk-load into it. + * If inserting with overwrite configured, and positioned to an on-page + * key, the update doesn't require another search. The cursor won't be + * positioned on a page with an external key set, but be sure. Cursors + * configured for append aren't included, regardless of whether or not + * they meet all other criteria. */ - if (btree->bulk_load_ok) { - btree->bulk_load_ok = false; - __wt_btree_evictable(session, true); + if (__cursor_page_pinned(cbt) && + F_ISSET_ALL(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_OVERWRITE) && + !append_key) { + WT_ERR(__wt_txn_autocommit_check(session)); + /* + * The cursor position may not be exact (the cursor's comparison + * value not equal to zero). Correct to an exact match so we can + * update whatever we're pointing at. + */ + cbt->compare = 0; + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(session, cbt, false) : + __cursor_col_modify(session, cbt, false); + if (ret == 0) + goto done; + + /* + * The pinned page goes away if we fail for any reason, make + * sure there's a local copy of any key. (Restart could still + * use the pinned page, but that's an unlikely path.) Re-save + * the cursor state: we may retry but eventually fail. + */ + WT_TRET(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + goto err; } -retry: WT_RET(__cursor_func_init(cbt, true)); + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Re-save the cursor state: we may retry but + * eventually fail. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { + WT_ERR(__cursor_row_search(session, cbt, NULL, true)); + /* + * If not overwriting, fail if the key exists, else insert the + * key/value pair. + */ + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && + cbt->compare == 0 && __wt_cursor_valid(cbt, NULL)) + WT_ERR(WT_DUPLICATE_KEY); + + ret = __cursor_row_modify(session, cbt, false); + } else { /* - * If WT_CURSTD_APPEND is set, insert a new record (ignoring - * the application's record number). The real record number - * is assigned by the serialized append operation. + * Optionally insert a new record (ignoring the application's + * record number). The real record number is allocated by the + * serialized append operation. */ - if (F_ISSET(cursor, WT_CURSTD_APPEND)) + if (append_key) cbt->iface.recno = WT_RECNO_OOB; WT_ERR(__cursor_col_search(session, cbt, NULL)); @@ -538,26 +731,14 @@ retry: WT_RET(__cursor_func_init(cbt, true)); * Fail in that case, the record exists. */ if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && - ((cbt->compare == 0 && __cursor_valid(cbt, NULL)) || + ((cbt->compare == 0 && __wt_cursor_valid(cbt, NULL)) || (cbt->compare != 0 && __cursor_fix_implicit(btree, cbt)))) WT_ERR(WT_DUPLICATE_KEY); WT_ERR(__cursor_col_modify(session, cbt, false)); - if (F_ISSET(cursor, WT_CURSTD_APPEND)) - cbt->iface.recno = cbt->recno; - break; - case BTREE_ROW: - WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - /* - * If not overwriting, fail if the key exists, else insert the - * key/value pair. - */ - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && - cbt->compare == 0 && __cursor_valid(cbt, NULL)) - WT_ERR(WT_DUPLICATE_KEY); - ret = __cursor_row_modify(session, cbt, false); - break; + if (append_key) + cbt->iface.recno = cbt->recno; } err: if (ret == WT_RESTART) { @@ -565,11 +746,17 @@ err: if (ret == WT_RESTART) { WT_STAT_DATA_INCR(session, cursor_restart); goto retry; } - /* Insert doesn't maintain a position across calls, clear resources. */ - if (ret == 0) - WT_TRET(__curfile_leave(cbt)); + +done: /* Insert doesn't maintain a position across calls, clear resources. */ + if (ret == 0) { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (append_key) + F_SET(cursor, WT_CURSTD_KEY_INT); + } + WT_TRET(__cursor_reset(cbt)); if (ret != 0) - WT_TRET(__cursor_reset(cbt)); + __cursor_state_restore(cursor, &state); + return (ret); } @@ -605,16 +792,15 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) } /* - * __wt_btcur_update_check -- + * __wt_btcur_insert_check -- * Check whether an update would conflict. * - * This can be used to replace WT_CURSOR::insert or WT_CURSOR::update, so - * they only check for conflicts without updating the tree. It is used to - * maintain snapshot isolation for transactions that span multiple chunks - * in an LSM tree. + * This can replace WT_CURSOR::insert, so it only checks for conflicts without + * updating the tree. It is used to maintain snapshot isolation for transactions + * that span multiple chunks in an LSM tree. */ int -__wt_btcur_update_check(WT_CURSOR_BTREE *cbt) +__wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; WT_CURSOR *cursor; @@ -625,31 +811,35 @@ __wt_btcur_update_check(WT_CURSOR_BTREE *cbt) btree = cbt->btree; session = (WT_SESSION_IMPL *)cursor->session; -retry: WT_RET(__cursor_func_init(cbt, true)); + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Unlike most of the btree cursor routines, + * we don't have to save/restore the cursor key state, none of the + * work done here changes the key state. + */ + WT_ERR(__cursor_copy_int_key(cursor)); - switch (btree->type) { - case BTREE_ROW: +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - /* - * Just check for conflicts. - */ + /* Just check for conflicts. */ ret = __curfile_update_check(cbt); - break; - case BTREE_COL_FIX: - case BTREE_COL_VAR: + } else WT_ERR(__wt_illegal_value(session, NULL)); - break; - } err: if (ret == WT_RESTART) { WT_STAT_CONN_INCR(session, cursor_restart); WT_STAT_DATA_INCR(session, cursor_restart); goto retry; } - WT_TRET(__curfile_leave(cbt)); - if (ret != 0) - WT_TRET(__cursor_reset(cbt)); + + /* Insert doesn't maintain a position across calls, clear resources. */ + if (ret == 0) + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_TRET(__cursor_reset(cbt)); + return (ret); } @@ -661,9 +851,11 @@ int __wt_btcur_remove(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; + bool positioned; btree = cbt->btree; cursor = &cbt->iface; @@ -673,22 +865,80 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCR(session, cursor_remove); WT_STAT_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size); -retry: WT_RET(__cursor_func_init(cbt, true)); + __cursor_state_save(cursor, &state); - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: + /* + * WT_CURSOR.remove has a unique semantic, the cursor stays positioned + * if it starts positioned, otherwise clear the cursor on completion. + */ + positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); + + /* + * If remove positioned to an on-page key, the remove doesn't require + * another search. We don't care about the "overwrite" configuration + * because regardless of the overwrite setting, any existing record is + * removed, and the record must exist with a positioned cursor. The + * cursor won't be positioned on a page with an external key set, but + * be sure. + */ + if (__cursor_page_pinned(cbt) && F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_ERR(__wt_txn_autocommit_check(session)); + + /* + * The cursor position may not be exact (the cursor's comparison + * value not equal to zero). Correct to an exact match so we can + * remove whatever we're pointing at. + */ + cbt->compare = 0; + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(session, cbt, true) : + __cursor_col_modify(session, cbt, true); + if (ret == 0) + goto done; + + /* + * The pinned page goes away if we fail for any reason, make + * sure there's a local copy of any key. (Restart could still + * use the pinned page, but that's an unlikely path.) Re-save + * the cursor state: we may retry but eventually fail. + */ + WT_TRET(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + goto err; + } + + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Re-save the cursor state: we may retry but + * eventually fail. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { + WT_ERR(__cursor_row_search(session, cbt, NULL, false)); + + /* Check whether an update would conflict. */ + WT_ERR(__curfile_update_check(cbt)); + + if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) + WT_ERR(WT_NOTFOUND); + + ret = __cursor_row_modify(session, cbt, true); + } else { WT_ERR(__cursor_col_search(session, cbt, NULL)); /* * If we find a matching record, check whether an update would * conflict. Do this before checking if the update is visible - * in __cursor_valid, or we can miss conflict. + * in __wt_cursor_valid, or we can miss conflict. */ WT_ERR(__curfile_update_check(cbt)); /* Remove the record if it exists. */ - if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) { + if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) { if (!__cursor_fix_implicit(btree, cbt)) WT_ERR(WT_NOTFOUND); /* @@ -704,19 +954,6 @@ retry: WT_RET(__cursor_func_init(cbt, true)); cbt->recno = cursor->recno; } else ret = __cursor_col_modify(session, cbt, true); - break; - case BTREE_ROW: - /* Remove the record if it exists. */ - WT_ERR(__cursor_row_search(session, cbt, NULL, false)); - - /* Check whether an update would conflict. */ - WT_ERR(__curfile_update_check(cbt)); - - if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) - WT_ERR(WT_NOTFOUND); - - ret = __cursor_row_modify(session, cbt, true); - break; } err: if (ret == WT_RESTART) { @@ -724,15 +961,27 @@ err: if (ret == WT_RESTART) { WT_STAT_DATA_INCR(session, cursor_restart); goto retry; } + /* - * If the cursor is configured to overwrite and the record is not - * found, that is exactly what we want. + * If the cursor is configured to overwrite and the record is not found, + * that is exactly what we want, return success. */ if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) && ret == WT_NOTFOUND) ret = 0; - if (ret != 0) +done: /* + * If the cursor was positioned, it stays positioned, point the cursor + * at an internal copy of the key. Otherwise, there's no position or + * key/value. + */ + if (ret == 0) + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (ret == 0 && positioned) + WT_TRET(__wt_key_return(session, cbt)); + else WT_TRET(__cursor_reset(cbt)); + if (ret != 0) + __cursor_state_restore(cursor, &state); return (ret); } @@ -745,6 +994,7 @@ int __wt_btcur_update(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -757,24 +1007,71 @@ __wt_btcur_update(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCR(session, cursor_update); WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); + __cursor_state_save(cursor, &state); + if (btree->type == BTREE_ROW) WT_RET(__cursor_size_chk(session, &cursor->key)); WT_RET(__cursor_size_chk(session, &cursor->value)); + /* It's no longer possible to bulk-load into the tree. */ + __cursor_disable_bulk(session, btree); + /* - * The tree is no longer empty: eviction should pay attention to it, - * and it's no longer possible to bulk-load into it. + * If update positioned to an on-page key, the update doesn't require + * another search. We don't care about the "overwrite" configuration + * because regardless of the overwrite setting, any existing record is + * updated, and the record must exist with a positioned cursor. The + * cursor won't be positioned on a page with an external key set, but + * be sure. */ - if (btree->bulk_load_ok) { - btree->bulk_load_ok = false; - __wt_btree_evictable(session, true); + if (__cursor_page_pinned(cbt) && F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_ERR(__wt_txn_autocommit_check(session)); + /* + * The cursor position may not be exact (the cursor's comparison + * value not equal to zero). Correct to an exact match so we can + * update whatever we're pointing at. + */ + cbt->compare = 0; + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(session, cbt, false) : + __cursor_col_modify(session, cbt, false); + if (ret == 0) + goto done; + + /* + * The pinned page goes away if we fail for any reason, make + * sure there's a local copy of any key. (Restart could still + * use the pinned page, but that's an unlikely path.) Re-save + * the cursor state: we may retry but eventually fail. + */ + WT_TRET(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + goto err; } -retry: WT_RET(__cursor_func_init(cbt, true)); + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Re-save the cursor state: we may retry but + * eventually fail. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { + WT_ERR(__cursor_row_search(session, cbt, NULL, true)); + /* + * If not overwriting, check for conflicts and fail if the key + * does not exist. + */ + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { + WT_ERR(__curfile_update_check(cbt)); + if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) + WT_ERR(WT_NOTFOUND); + } + ret = __cursor_row_modify(session, cbt, false); + } else { WT_ERR(__cursor_col_search(session, cbt, NULL)); /* @@ -787,25 +1084,12 @@ retry: WT_RET(__cursor_func_init(cbt, true)); */ if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { WT_ERR(__curfile_update_check(cbt)); - if ((cbt->compare != 0 || !__cursor_valid(cbt, NULL)) && + if ((cbt->compare != 0 || + !__wt_cursor_valid(cbt, NULL)) && !__cursor_fix_implicit(btree, cbt)) WT_ERR(WT_NOTFOUND); } ret = __cursor_col_modify(session, cbt, false); - break; - case BTREE_ROW: - WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - /* - * If not overwriting, check for conflicts and fail if the key - * does not exist. - */ - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { - WT_ERR(__curfile_update_check(cbt)); - if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) - WT_ERR(WT_NOTFOUND); - } - ret = __cursor_row_modify(session, cbt, false); - break; } err: if (ret == WT_RESTART) { @@ -822,116 +1106,14 @@ err: if (ret == WT_RESTART) { * To make this work, we add a field to the btree cursor to pass back a * pointer to the modify function's allocated update structure. */ - if (ret == 0) +done: if (ret == 0) WT_TRET(__wt_kv_return(session, cbt, cbt->modify_update)); - if (ret != 0) + if (ret != 0) { WT_TRET(__cursor_reset(cbt)); - return (ret); -} - -/* - * __wt_btcur_next_random -- - * Move to a random record in the tree. There are two algorithms, one - * where we select a record at random from the whole tree on each - * retrieval and one where we first select a record at random from the - * whole tree, and then subsequently sample forward from that location. - * The sampling approach allows us to select reasonably uniform random - * points from unbalanced trees. - */ -int -__wt_btcur_next_random(WT_CURSOR_BTREE *cbt) -{ - WT_BTREE *btree; - WT_DECL_RET; - WT_SESSION_IMPL *session; - WT_UPDATE *upd; - wt_off_t size; - uint64_t skip; - - session = (WT_SESSION_IMPL *)cbt->iface.session; - btree = cbt->btree; - - /* - * Only supports row-store: applications can trivially select a random - * value from a column-store, if there were any reason to do so. - */ - if (btree->type != BTREE_ROW) - WT_RET_MSG(session, ENOTSUP, - "WT_CURSOR.next_random only supported by row-store tables"); - - WT_STAT_CONN_INCR(session, cursor_next); - WT_STAT_DATA_INCR(session, cursor_next); - - /* - * If retrieving random values without sampling, or we don't have a - * page reference, pick a roughly random leaf page in the tree. - */ - if (cbt->ref == NULL || cbt->next_random_sample_size == 0) { - /* - * Skip past the sample size of the leaf pages in the tree - * between each random key return to compensate for unbalanced - * trees. - * - * Use the underlying file size divided by its block allocation - * size as our guess of leaf pages in the file (this can be - * entirely wrong, as it depends on how many pages are in this - * particular checkpoint, how large the leaf and internal pages - * really are, and other factors). Then, divide that value by - * the configured sample size and increment the final result to - * make sure tiny files don't leave us with a skip value of 0. - * - * !!! - * Ideally, the number would be prime to avoid restart issues. - */ - if (cbt->next_random_sample_size != 0) { - WT_ERR(btree->bm->size(btree->bm, session, &size)); - cbt->next_random_leaf_skip = (uint64_t) - ((size / btree->allocsize) / - cbt->next_random_sample_size) + 1; - } - - /* - * Choose a leaf page from the tree. - */ - WT_ERR(__cursor_func_init(cbt, true)); - WT_WITH_PAGE_INDEX( - session, ret = __wt_row_random_descent(session, cbt)); - WT_ERR(ret); - } else { - /* - * Read through the tree, skipping leaf pages. Be cautious about - * the skip count: if the last leaf page skipped was also the - * last leaf page in the tree, it may be set to zero on return - * with the end-of-walk condition. - * - * Pages read for data sampling aren't "useful"; don't update - * the read generation of pages already in memory, and if a page - * is read, set its generation to a low value so it is evicted - * quickly. - */ - for (skip = - cbt->next_random_leaf_skip; cbt->ref == NULL || skip > 0;) - WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip, - WT_READ_NO_GEN | - WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + __cursor_state_restore(cursor, &state); } - /* - * Select a random entry from the leaf page. If it's not valid, move to - * the next entry, if that doesn't work, move to the previous entry. - */ - WT_ERR(__wt_row_random_leaf(session, cbt)); - if (__cursor_valid(cbt, &upd)) - WT_ERR(__wt_kv_return(session, cbt, upd)); - else { - if ((ret = __wt_btcur_next(cbt, false)) == WT_NOTFOUND) - ret = __wt_btcur_prev(cbt, false); - WT_ERR(ret); - } - return (0); - -err: WT_TRET(__cursor_reset(cbt)); return (ret); } @@ -1060,9 +1242,12 @@ __cursor_truncate(WT_SESSION_IMPL *session, WT_DECL_RET; /* - * First, call the standard cursor remove method to do a full search and - * re-position the cursor because we don't have a saved copy of the - * page's write generation information, which we need to remove records. + * First, call the cursor search method to re-position the cursor: we + * may not have a cursor position (if the higher-level truncate code + * switched the cursors to have an "external" cursor key, and because + * we don't save a copy of the page's write generation information, + * which we need to remove records. + * * Once that's done, we can delete records without a full search, unless * we encounter a restart error because the page was modified by some * other thread of control; in that case, repeat the full search to @@ -1075,20 +1260,19 @@ __cursor_truncate(WT_SESSION_IMPL *session, * instantiated the end cursor, so we know that page is pinned in memory * and we can proceed without concern. */ -retry: WT_RET(__wt_btcur_remove(start)); +retry: WT_RET(__wt_btcur_search(start)); + WT_ASSERT(session, + F_MASK((WT_CURSOR *)start, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + + for (;;) { + if ((ret = rmfunc(session, start, 1)) != 0) + break; - /* - * Reset ret each time through so that we don't loop forever in - * the cursor equals case. - */ - for (ret = 0;;) { if (stop != NULL && __cursor_equals(start, stop)) break; if ((ret = __wt_btcur_next(start, true)) != 0) break; - start->compare = 0; /* Exact match */ - if ((ret = rmfunc(session, start, 1)) != 0) - break; + start->compare = 0; /* Exact match */ } if (ret == WT_RESTART) { @@ -1121,29 +1305,32 @@ __cursor_truncate_fix(WT_SESSION_IMPL *session, * record 37, records 1-36 magically appear. Those records can't be * deleted, which means we have to ignore already "deleted" records. * - * First, call the standard cursor remove method to do a full search and - * re-position the cursor because we don't have a saved copy of the - * page's write generation information, which we need to remove records. + * First, call the cursor search method to re-position the cursor: we + * may not have a cursor position (if the higher-level truncate code + * switched the cursors to have an "external" cursor key, and because + * we don't save a copy of the page's write generation information, + * which we need to remove records. + * * Once that's done, we can delete records without a full search, unless * we encounter a restart error because the page was modified by some * other thread of control; in that case, repeat the full search to * refresh the page's modification information. */ -retry: WT_RET(__wt_btcur_remove(start)); - /* - * Reset ret each time through so that we don't loop forever in - * the cursor equals case. - */ - for (ret = 0;;) { +retry: WT_RET(__wt_btcur_search(start)); + WT_ASSERT(session, + F_MASK((WT_CURSOR *)start, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + + for (;;) { + value = (const uint8_t *)start->iface.value.data; + if (*value != 0 && + (ret = rmfunc(session, start, 1)) != 0) + break; + if (stop != NULL && __cursor_equals(start, stop)) break; if ((ret = __wt_btcur_next(start, true)) != 0) break; start->compare = 0; /* Exact match */ - value = (const uint8_t *)start->iface.value.data; - if (*value != 0 && - (ret = rmfunc(session, start, 1)) != 0) - break; } if (ret == WT_RESTART) { @@ -1263,7 +1450,7 @@ __wt_btcur_close(WT_CURSOR_BTREE *cbt, bool lowlevel) * Skip the usual cursor tear-down in that case. */ if (!lowlevel) - ret = __curfile_leave(cbt); + ret = __cursor_reset(cbt); __wt_buf_free(session, &cbt->_row_key); __wt_buf_free(session, &cbt->_tmp); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index bcc7d27a569..fdc33b608ec 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -34,7 +34,7 @@ static const /* Output separator */ static int __debug_cell(WT_DBG *, const WT_PAGE_HEADER *, WT_CELL_UNPACK *); static int __debug_cell_data( - WT_DBG *, WT_PAGE *, int type, const char *, WT_CELL_UNPACK *); + WT_DBG *, WT_PAGE *, int, const char *, WT_CELL_UNPACK *); static int __debug_col_skip(WT_DBG *, WT_INSERT_HEAD *, const char *, bool); static int __debug_config(WT_SESSION_IMPL *, WT_DBG *, const char *); static int __debug_dsk_cell(WT_DBG *, const WT_PAGE_HEADER *); @@ -64,7 +64,7 @@ __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v) const char *cfg[2] = { NULL, NULL }; char buf[256]; - snprintf(buf, sizeof(buf), "verbose=[%s]", v); + WT_RET(__wt_snprintf(buf, sizeof(buf), "verbose=[%s]", v)); cfg[0] = buf; return (__wt_verbose_config(session, cfg)); } @@ -77,7 +77,7 @@ static inline int __debug_hex_byte(WT_DBG *ds, uint8_t v) { return (ds->f( - ds, "#%c%c", __wt_hex[(v & 0xf0) >> 4], __wt_hex[v & 0x0f])); + ds, "#%c%c", __wt_hex((v & 0xf0) >> 4), __wt_hex(v & 0x0f))); } /* @@ -87,6 +87,7 @@ __debug_hex_byte(WT_DBG *ds, uint8_t v) static int __dmsg_event(WT_DBG *ds, const char *fmt, ...) { + WT_DECL_RET; WT_ITEM *msg; WT_SESSION_IMPL *session; size_t len, space; @@ -107,8 +108,9 @@ __dmsg_event(WT_DBG *ds, const char *fmt, ...) p = (char *)msg->mem + msg->size; space = msg->memsize - msg->size; va_start(ap, fmt); - len = (size_t)vsnprintf(p, space, fmt, ap); + ret = __wt_vsnprintf_len_set(p, space, &len, fmt, ap); va_end(ap); + WT_RET(ret); /* Check if there was enough space. */ if (len < space) { @@ -447,13 +449,14 @@ __debug_tree_shape_info(WT_PAGE *page) v = page->memory_footprint; if (v >= WT_GIGABYTE) - snprintf(buf, sizeof(buf), + (void)__wt_snprintf(buf, sizeof(buf), "(%p %" PRIu64 "G)", (void *)page, v / WT_GIGABYTE); else if (v >= WT_MEGABYTE) - snprintf(buf, sizeof(buf), + (void)__wt_snprintf(buf, sizeof(buf), "(%p %" PRIu64 "M)", (void *)page, v / WT_MEGABYTE); else - snprintf(buf, sizeof(buf), "(%p %" PRIu64 ")", (void *)page, v); + (void)__wt_snprintf(buf, sizeof(buf), + "(%p %" PRIu64 ")", (void *)page, v); return (buf); } @@ -652,7 +655,7 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) page = ref->page; mod = page->modify; - WT_RET(ds->f(ds, "%p", (void *)page)); + WT_RET(ds->f(ds, "%p", (void *)ref)); switch (page->type) { case WT_PAGE_COL_INT: @@ -662,29 +665,30 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) break; case WT_PAGE_COL_FIX: WT_RET(ds->f(ds, " recno %" PRIu64, ref->ref_recno)); - entries = page->pg_fix_entries; + entries = page->entries; break; case WT_PAGE_COL_VAR: WT_RET(ds->f(ds, " recno %" PRIu64, ref->ref_recno)); - entries = page->pg_var_entries; + entries = page->entries; break; case WT_PAGE_ROW_INT: WT_INTL_INDEX_GET(session, page, pindex); entries = pindex->entries; break; case WT_PAGE_ROW_LEAF: - entries = page->pg_row_entries; + entries = page->entries; break; WT_ILLEGAL_VALUE(session); } WT_RET(ds->f(ds, ": %s\n", __wt_page_type_string(page->type))); - WT_RET(ds->f(ds, - "\t" "disk %p, entries %" PRIu32, (void *)page->dsk, entries)); + WT_RET(ds->f(ds, "\t" "disk %p", (void *)page->dsk)); + if (page->dsk != NULL) + WT_RET(ds->f( + ds, ", dsk_mem_size %" PRIu32, page->dsk->mem_size)); + WT_RET(ds->f(ds, ", entries %" PRIu32, entries)); WT_RET(ds->f(ds, ", %s", __wt_page_is_modified(page) ? "dirty" : "clean")); - WT_RET(ds->f(ds, ", %s", __wt_rwlock_islocked( - session, &page->page_lock) ? "locked" : "unlocked")); if (F_ISSET_ATOMIC(page, WT_PAGE_BUILD_KEYS)) WT_RET(ds->f(ds, ", keys-built")); @@ -696,8 +700,6 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) WT_RET(ds->f(ds, ", evict-lru")); if (F_ISSET_ATOMIC(page, WT_PAGE_OVERFLOW_KEYS)) WT_RET(ds->f(ds, ", overflow-keys")); - if (F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_BLOCK)) - WT_RET(ds->f(ds, ", split-block")); if (F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_INSERT)) WT_RET(ds->f(ds, ", split-insert")); if (F_ISSET_ATOMIC(page, WT_PAGE_UPDATE_IGNORE)) @@ -837,7 +839,8 @@ __debug_page_col_var(WT_DBG *ds, WT_REF *ref) __wt_cell_unpack(cell, unpack); rle = __wt_cell_rle(unpack); } - snprintf(tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle); + WT_RET(__wt_snprintf( + tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle)); WT_RET( __debug_cell_data(ds, page, WT_PAGE_COL_VAR, tag, unpack)); @@ -1003,37 +1006,37 @@ __debug_ref(WT_DBG *ds, WT_REF *ref) WT_SESSION_IMPL *session; size_t addr_size; const uint8_t *addr; + const char *state; session = ds->session; - WT_RET(ds->f(ds, "\t")); switch (ref->state) { case WT_REF_DISK: - WT_RET(ds->f(ds, "disk")); + state = "disk"; break; case WT_REF_DELETED: - WT_RET(ds->f(ds, "deleted")); + state = "deleted"; break; case WT_REF_LOCKED: - WT_RET(ds->f(ds, "locked %p", (void *)ref->page)); + state = "locked"; break; case WT_REF_MEM: - WT_RET(ds->f(ds, "memory %p", (void *)ref->page)); + state = "memory"; break; case WT_REF_READING: - WT_RET(ds->f(ds, "reading")); + state = "reading"; break; case WT_REF_SPLIT: - WT_RET(ds->f(ds, "split")); + state = "split"; break; default: - WT_RET(ds->f(ds, "INVALID")); + state = "INVALID"; break; } __wt_ref_info(ref, &addr, &addr_size, NULL); - return (ds->f(ds, " %s\n", - __wt_addr_string(session, addr, addr_size, ds->tmp))); + return (ds->f(ds, "\t" "%p %s %s\n", (void *)ref, + state, __wt_addr_string(session, addr, addr_size, ds->tmp))); } /* @@ -1104,9 +1107,9 @@ __debug_cell(WT_DBG *ds, const WT_PAGE_HEADER *dsk, WT_CELL_UNPACK *unpack) case WT_CELL_VALUE_OVFL_RM: type = "ovfl"; addr: WT_RET(__wt_scr_alloc(session, 128, &buf)); - WT_RET(ds->f(ds, ", %s %s", type, + ret = ds->f(ds, ", %s %s", type, __wt_addr_string( - session, unpack->data, unpack->size, buf))); + session, unpack->data, unpack->size, buf)); __wt_scr_free(session, &buf); WT_RET(ret); break; diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 00e41475de9..b55ad291c5e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -318,13 +318,12 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) * hard case is if a page splits: the update structures might be moved * to different pages, and we still have to find them all for an abort. */ - if (page_del != NULL) WT_RET(__wt_calloc_def( - session, page->pg_row_entries + 1, &page_del->update_list)); + session, page->entries + 1, &page_del->update_list)); /* Allocate the per-page update array. */ - WT_ERR(__wt_calloc_def(session, page->pg_row_entries, &upd_array)); + WT_ERR(__wt_calloc_def(session, page->entries, &upd_array)); page->modify->mod_row_update = upd_array; /* @@ -332,7 +331,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) * structures, fill in the per-page update array with references to * deleted items. */ - for (i = 0, size = 0; i < page->pg_row_entries; ++i) { + for (i = 0, size = 0; i < page->entries; ++i) { WT_ERR(__wt_calloc_one(session, &upd)); WT_UPDATE_DELETED_SET(upd); diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index 162bc07a1c2..04c0a5d410d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -16,29 +16,66 @@ static void __free_skip_array( WT_SESSION_IMPL *, WT_INSERT_HEAD **, uint32_t, bool); static void __free_skip_list(WT_SESSION_IMPL *, WT_INSERT *, bool); static void __free_update(WT_SESSION_IMPL *, WT_UPDATE **, uint32_t, bool); +static void __page_out_int(WT_SESSION_IMPL *, WT_PAGE **, bool); /* - * __wt_ref_out -- + * __wt_ref_out_int -- * Discard an in-memory page, freeing all memory associated with it. */ void -__wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) +__wt_ref_out_int(WT_SESSION_IMPL *session, WT_REF *ref, bool rewrite) { /* * A version of the page-out function that allows us to make additional * diagnostic checks. + * + * The WT_REF cannot be the eviction thread's location. */ WT_ASSERT(session, S2BT(session)->evict_ref != ref); - __wt_page_out(session, &ref->page); +#ifdef HAVE_DIAGNOSTIC + { + WT_HAZARD *hp; + int i; + /* + * Make sure no other thread has a hazard pointer on the page we are + * about to discard. This is complicated by the fact that readers + * publish their hazard pointer before re-checking the page state, so + * our check can race with readers without indicating a real problem. + * Wait for up to a second for hazard pointers to be cleared. + */ + for (hp = NULL, i = 0; i < 100; i++) { + if ((hp = __wt_hazard_check(session, ref)) == NULL) + break; + __wt_sleep(0, 10000); + } + if (hp != NULL) + __wt_errx(session, + "discarded page has hazard pointer: (%p: %s, line %d)", + (void *)hp->ref, hp->file, hp->line); + WT_ASSERT(session, hp == NULL); + } +#endif + + __page_out_int(session, &ref->page, rewrite); } /* - * __wt_page_out -- + * __wt_ref_out -- * Discard an in-memory page, freeing all memory associated with it. */ void -__wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) +__wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) +{ + __wt_ref_out_int(session, ref, false); +} + +/* + * __page_out_int -- + * Discard an in-memory page, freeing all memory associated with it. + */ +static void +__page_out_int(WT_SESSION_IMPL *session, WT_PAGE **pagep, bool rewrite) { WT_PAGE *page; WT_PAGE_HEADER *dsk; @@ -61,31 +98,6 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) */ WT_ASSERT(session, !__wt_page_is_modified(page)); WT_ASSERT(session, !F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)); - WT_ASSERT(session, !__wt_rwlock_islocked(session, &page->page_lock)); - -#ifdef HAVE_DIAGNOSTIC - { - WT_HAZARD *hp; - int i; - /* - * Make sure no other thread has a hazard pointer on the page we are - * about to discard. This is complicated by the fact that readers - * publish their hazard pointer before re-checking the page state, so - * our check can race with readers without indicating a real problem. - * Wait for up to a second for hazard pointers to be cleared. - */ - for (hp = NULL, i = 0; i < 100; i++) { - if ((hp = __wt_page_hazard_check(session, page)) == NULL) - break; - __wt_sleep(0, 10000); - } - if (hp != NULL) - __wt_errx(session, - "discarded page has hazard pointer: (%p: %s, line %d)", - (void *)hp->page, hp->file, hp->line); - WT_ASSERT(session, hp == NULL); - } -#endif /* * If a root page split, there may be one or more pages linked from the @@ -101,7 +113,7 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) } /* Update the cache's information. */ - __wt_cache_page_evict(session, page); + __wt_cache_page_evict(session, page, rewrite); dsk = (WT_PAGE_HEADER *)page->dsk; if (F_ISSET_ATOMIC(page, WT_PAGE_DISK_ALLOC)) @@ -145,6 +157,16 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) __wt_overwrite_and_free(session, page); } +/* + * __wt_page_out -- + * Discard an in-memory page, freeing all memory associated with it. + */ +void +__wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) +{ + __page_out_int(session, pagep, false); +} + /* * __free_page_modify -- * Discard the page's associated modification structures. @@ -204,8 +226,7 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) if (mod->mod_col_update != NULL) __free_skip_array(session, mod->mod_col_update, page->type == - WT_PAGE_COL_FIX ? 1 : page->pg_var_entries, - update_ignore); + WT_PAGE_COL_FIX ? 1 : page->entries, update_ignore); break; case WT_PAGE_ROW_LEAF: /* @@ -217,12 +238,12 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) */ if (mod->mod_row_insert != NULL) __free_skip_array(session, mod->mod_row_insert, - page->pg_row_entries + 1, update_ignore); + page->entries + 1, update_ignore); /* Free the update array. */ if (mod->mod_row_update != NULL) __free_update(session, mod->mod_row_update, - page->pg_row_entries, update_ignore); + page->entries, update_ignore); break; } @@ -232,6 +253,7 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_ovfl_discard_free(session, page); __wt_free(session, page->modify->ovfl_track); + __wt_spin_destroy(session, &page->modify->page_lock); __wt_free(session, page->modify); } @@ -330,7 +352,7 @@ static void __free_page_col_var(WT_SESSION_IMPL *session, WT_PAGE *page) { /* Free the RLE lookup array. */ - __wt_free(session, page->pg_var_repeats); + __wt_free(session, page->u.col_var.repeats); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 337a3ea036f..a0da7df0998 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -14,6 +14,44 @@ static int __btree_page_sizes(WT_SESSION_IMPL *); static int __btree_preload(WT_SESSION_IMPL *); static int __btree_tree_open_empty(WT_SESSION_IMPL *, bool); +/* + * __btree_clear -- + * Clear a Btree, either on handle discard or re-open. + */ +static int +__btree_clear(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_DECL_RET; + + btree = S2BT(session); + + /* + * If the tree hasn't gone through an open/close cycle, there's no + * cleanup to be done. + */ + if (!F_ISSET(btree, WT_BTREE_CLOSED)) + return (0); + + /* Close the Huffman tree. */ + __wt_btree_huffman_close(session); + + /* Terminate any associated collator. */ + if (btree->collator_owned && btree->collator->terminate != NULL) + WT_TRET(btree->collator->terminate( + btree->collator, &session->iface)); + + /* Destroy locks. */ + __wt_rwlock_destroy(session, &btree->ovfl_lock); + __wt_spin_destroy(session, &btree->flush_lock); + + /* Free allocated memory. */ + __wt_free(session, btree->key_format); + __wt_free(session, btree->value_format); + + return (ret); +} + /* * __wt_btree_open -- * Open a Btree. @@ -28,12 +66,27 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_DATA_HANDLE *dhandle; WT_DECL_RET; size_t root_addr_size; + uint32_t mask; uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE]; const char *filename; bool creation, forced_salvage, readonly; - dhandle = session->dhandle; btree = S2BT(session); + dhandle = session->dhandle; + + /* + * This may be a re-open of an underlying object and we have to clean + * up. We can't clear the operation flags, however, they're set by the + * connection handle software that called us. + */ + WT_RET(__btree_clear(session)); + + mask = F_MASK(btree, WT_BTREE_SPECIAL_FLAGS); + memset(btree, 0, sizeof(*btree)); + btree->flags = mask; + + /* Set the data handle first, our called functions reasonably use it. */ + btree->dhandle = dhandle; /* Checkpoint files are readonly. */ readonly = dhandle->checkpoint != NULL || @@ -126,6 +179,20 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) } } + /* + * Eviction ignores trees until the handle's open flag is set, configure + * eviction before that happens. + * + * Files that can still be bulk-loaded cannot be evicted. + * Permanently cache-resident files can never be evicted. + * Special operations don't enable eviction. (The underlying commands + * may turn on eviction, but it's their decision.) + */ + if (btree->original || + F_ISSET(btree, WT_BTREE_IN_MEMORY | WT_BTREE_REBALANCE | + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) + WT_ERR(__wt_evict_file_exclusive_on(session)); + if (0) { err: WT_TRET(__wt_btree_close(session)); } @@ -147,7 +214,24 @@ __wt_btree_close(WT_SESSION_IMPL *session) btree = S2BT(session); + /* + * The close process isn't the same as discarding the handle: we might + * re-open the handle, which isn't a big deal, but the backing blocks + * for the handle may not yet have been discarded from the cache, and + * eviction uses WT_BTREE structure elements. Free backing resources + * but leave the rest alone, and we'll discard the structure when we + * discard the data handle. + * + * Handles can be closed multiple times, ignore all but the first. + */ + if (F_ISSET(btree, WT_BTREE_CLOSED)) + return (0); + F_SET(btree, WT_BTREE_CLOSED); + + /* Discard any underlying block manager resources. */ if ((bm = btree->bm) != NULL) { + btree->bm = NULL; + /* Unload the checkpoint, unless it's a special command. */ if (!F_ISSET(btree, WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) @@ -155,33 +239,26 @@ __wt_btree_close(WT_SESSION_IMPL *session) /* Close the underlying block manager reference. */ WT_TRET(bm->close(bm, session)); - - btree->bm = NULL; } - /* Close the Huffman tree. */ - __wt_btree_huffman_close(session); - - /* Destroy locks. */ - __wt_rwlock_destroy(session, &btree->ovfl_lock); - __wt_spin_destroy(session, &btree->flush_lock); - - /* Free allocated memory. */ - __wt_free(session, btree->key_format); - __wt_free(session, btree->value_format); + return (ret); +} - if (btree->collator_owned) { - if (btree->collator->terminate != NULL) - WT_TRET(btree->collator->terminate( - btree->collator, &session->iface)); - btree->collator_owned = 0; - } - btree->collator = NULL; - btree->kencryptor = NULL; +/* + * __wt_btree_discard -- + * Discard a Btree. + */ +int +__wt_btree_discard(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_DECL_RET; - btree->bulk_load_ok = false; + ret = __btree_clear(session); - F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); + btree = S2BT(session); + __wt_overwrite_and_free(session, btree); + session->dhandle->handle = NULL; return (ret); } @@ -267,9 +344,9 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_config_gets(session, cfg, "cache_resident", &cval)); if (cval.val) - F_SET(btree, WT_BTREE_IN_MEMORY | WT_BTREE_NO_EVICTION); + F_SET(btree, WT_BTREE_IN_MEMORY); else - F_CLR(btree, WT_BTREE_IN_MEMORY | WT_BTREE_NO_EVICTION); + F_CLR(btree, WT_BTREE_IN_MEMORY); WT_RET(__wt_config_gets(session, cfg, "ignore_in_memory_cache_size", &cval)); @@ -282,6 +359,14 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } else F_CLR(btree, WT_BTREE_IGNORE_CACHE); + /* + * The metadata isn't blocked by in-memory cache limits because metadata + * "unroll" is performed by updates that are potentially blocked by the + * cache-full checks. + */ + if (WT_IS_METADATA(btree->dhandle)) + F_SET(btree, WT_BTREE_IGNORE_CACHE); + WT_RET(__wt_config_gets(session, cfg, "log.enabled", &cval)); if (cval.val) F_CLR(btree, WT_BTREE_NO_LOGGING); @@ -341,7 +426,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) * always inherit from the connection. */ WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); - if (WT_IS_METADATA(session, btree->dhandle) || cval.len == 0) + if (WT_IS_METADATA(btree->dhandle) || cval.len == 0) btree->kencryptor = conn->kencryptor; else if (WT_STRING_MATCH("none", cval.str, cval.len)) btree->kencryptor = NULL; @@ -359,8 +444,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } /* Initialize locks. */ - WT_RET(__wt_rwlock_alloc( - session, &btree->ovfl_lock, "btree overflow lock")); + WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock)); WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ @@ -432,7 +516,7 @@ __wt_btree_tree_open( * Failure to open metadata means that the database is unavailable. * Try to provide a helpful failure message. */ - if (ret != 0 && WT_IS_METADATA(session, session->dhandle)) { + if (ret != 0 && WT_IS_METADATA(session->dhandle)) { __wt_errx(session, "WiredTiger has failed to open its metadata"); __wt_errx(session, "This may be due to the database" @@ -483,13 +567,10 @@ __btree_tree_open_empty(WT_SESSION_IMPL *session, bool creation) /* * Newly created objects can be used for cursor inserts or for bulk * loads; set a flag that's cleared when a row is inserted into the - * tree. Objects being bulk-loaded cannot be evicted, we set it - * globally, there's no point in searching empty trees for eviction. + * tree. */ - if (creation) { - btree->bulk_load_ok = true; - __wt_btree_evictable(session, false); - } + if (creation) + btree->original = 1; /* * A note about empty trees: the initial tree is a single root page. @@ -581,27 +662,6 @@ __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) return (0); } -/* - * __wt_btree_evictable -- - * Setup or release a cache-resident tree. - */ -void -__wt_btree_evictable(WT_SESSION_IMPL *session, bool on) -{ - WT_BTREE *btree; - - btree = S2BT(session); - - /* Permanently cache-resident files can never be evicted. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) - return; - - if (on) - F_CLR(btree, WT_BTREE_NO_EVICTION); - else - F_SET(btree, WT_BTREE_NO_EVICTION); -} - /* * __btree_preload -- * Pre-load internal pages. @@ -728,9 +788,16 @@ __btree_page_sizes(WT_SESSION_IMPL *session) * Get the split percentage (reconciliation splits pages into smaller * than the maximum page size chunks so we don't split every time a * new entry is added). Determine how large newly split pages will be. + * Set to the minimum, if the read value is less than that. */ WT_RET(__wt_config_gets(session, cfg, "split_pct", &cval)); - btree->split_pct = (int)cval.val; + if (cval.val < WT_BTREE_MIN_SPLIT_PCT) { + btree->split_pct = WT_BTREE_MIN_SPLIT_PCT; + WT_RET(__wt_msg(session, + "Re-setting split_pct for %s to the minimum allowed of " + "%d%%.", session->dhandle->name, WT_BTREE_MIN_SPLIT_PCT)); + } else + btree->split_pct = (int)cval.val; intl_split_size = __wt_split_page_size(btree, btree->maxintlpage); leaf_split_size = __wt_split_page_size(btree, btree->maxleafpage); diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index a8645f79dbe..b5e4d52394a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -183,7 +183,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t dst_len, len, result_len, size, src_len; int compression_failed; /* Extension API, so not a bool. */ uint8_t *dst, *src; - bool data_checksum, encrypted; + bool data_checksum, encrypted, timer; btree = S2BT(session); bm = btree->bm; @@ -216,7 +216,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, &result_len)); WT_ASSERT(session, dsk->mem_size == result_len + WT_BLOCK_COMPRESS_SKIP); - ctmp->size = (uint32_t)result_len + WT_BLOCK_COMPRESS_SKIP; + ctmp->size = result_len + WT_BLOCK_COMPRESS_SKIP; ip = ctmp; } else { WT_ASSERT(session, dsk->mem_size == buf->size); @@ -357,7 +357,8 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, data_checksum = !compressed; break; } - if (!F_ISSET(session, WT_SESSION_INTERNAL)) + timer = !F_ISSET(session, WT_SESSION_INTERNAL); + if (timer) __wt_epoch(session, &start); /* Call the block manager to write the block. */ @@ -367,7 +368,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, bm, session, ip, addr, addr_sizep, data_checksum, checkpoint_io)); /* Update some statistics now that the write is done */ - if (!F_ISSET(session, WT_SESSION_INTERNAL)) { + if (timer) { __wt_epoch(session, &stop); WT_STAT_CONN_INCR(session, cache_write_app_count); WT_STAT_CONN_INCRV(session, cache_write_app_time, diff --git a/src/third_party/wiredtiger/src/btree/bt_misc.c b/src/third_party/wiredtiger/src/btree/bt_misc.c index b6e2cc07f5a..3bec65c2567 100644 --- a/src/third_party/wiredtiger/src/btree/bt_misc.c +++ b/src/third_party/wiredtiger/src/btree/bt_misc.c @@ -14,6 +14,7 @@ */ const char * __wt_page_type_string(u_int type) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { switch (type) { case WT_PAGE_INVALID: diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index 29ea561db3a..ae0da62af57 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -67,11 +67,11 @@ __wt_ovfl_read(WT_SESSION_IMPL *session, * Acquire the overflow lock, and retest the on-page cell's value inside * the lock. */ - __wt_readlock(session, S2BT(session)->ovfl_lock); + __wt_readlock(session, &S2BT(session)->ovfl_lock); ret = __wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM ? __wt_ovfl_txnc_search(page, unpack->data, unpack->size, store) : __ovfl_read(session, unpack->data, unpack->size, store); - __wt_readunlock(session, S2BT(session)->ovfl_lock); + __wt_readunlock(session, &S2BT(session)->ovfl_lock); return (ret); } @@ -249,7 +249,7 @@ __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) * Acquire the overflow lock to avoid racing with a thread reading the * backing overflow blocks. */ - __wt_writelock(session, btree->ovfl_lock); + __wt_writelock(session, &btree->ovfl_lock); switch (unpack->raw) { case WT_CELL_KEY_OVFL: @@ -263,7 +263,7 @@ __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_ILLEGAL_VALUE(session); } - __wt_writeunlock(session, btree->ovfl_lock); + __wt_writeunlock(session, &btree->ovfl_lock); /* Free the backing disk blocks. */ return (bm->free(bm, session, unpack->data, unpack->size)); diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 7bac7079fe8..f20f6398e37 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -67,7 +67,7 @@ __wt_page_alloc(WT_SESSION_IMPL *session, switch (type) { case WT_PAGE_COL_FIX: - page->pg_fix_entries = alloc_entries; + page->entries = alloc_entries; break; case WT_PAGE_COL_INT: case WT_PAGE_ROW_INT: @@ -102,12 +102,12 @@ err: if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) { } break; case WT_PAGE_COL_VAR: - page->pg_var_d = (WT_COL *)((uint8_t *)page + sizeof(WT_PAGE)); - page->pg_var_entries = alloc_entries; + page->pg_var = (WT_COL *)((uint8_t *)page + sizeof(WT_PAGE)); + page->entries = alloc_entries; break; case WT_PAGE_ROW_LEAF: - page->pg_row_d = (WT_ROW *)((uint8_t *)page + sizeof(WT_PAGE)); - page->pg_row_entries = alloc_entries; + page->pg_row = (WT_ROW *)((uint8_t *)page + sizeof(WT_PAGE)); + page->entries = alloc_entries; break; WT_ILLEGAL_VALUE(session); } @@ -333,9 +333,10 @@ __inmem_col_var( WT_CELL *cell; WT_CELL_UNPACK *unpack, _unpack; const WT_PAGE_HEADER *dsk; + size_t size; uint64_t rle; - size_t bytes_allocated; uint32_t i, indx, n, repeat_off; + void *p; btree = S2BT(session); dsk = page->dsk; @@ -343,7 +344,6 @@ __inmem_col_var( repeats = NULL; repeat_off = 0; unpack = &_unpack; - bytes_allocated = 0; /* * Walk the page, building references: the page contains unsorted value @@ -351,7 +351,7 @@ __inmem_col_var( * (WT_CELL_VALUE_OVFL) or deleted items (WT_CELL_DEL). */ indx = 0; - cip = page->pg_var_d; + cip = page->pg_var; WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { __wt_cell_unpack(cell, unpack); WT_COL_PTR_SET(cip, WT_PAGE_DISK_OFFSET(page, cell)); @@ -367,12 +367,14 @@ __inmem_col_var( if (rle > 1) { if (repeats == NULL) { __inmem_col_var_repeats(session, page, &n); - WT_RET(__wt_realloc_def(session, - &bytes_allocated, n + 1, &repeats)); + size = sizeof(WT_COL_VAR_REPEAT) + + (n + 1) * sizeof(WT_COL_RLE); + WT_RET(__wt_calloc(session, 1, size, &p)); + *sizep += size; - page->pg_var_repeats = repeats; + page->u.col_var.repeats = p; page->pg_var_nrepeats = n; - *sizep += bytes_allocated; + repeats = page->pg_var_repeats; } repeats[repeat_off].indx = indx; repeats[repeat_off].recno = recno; @@ -569,7 +571,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) unpack = &_unpack; /* Walk the page, building indices. */ - rip = page->pg_row_d; + rip = page->pg_row; WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { __wt_cell_unpack(cell, unpack); switch (unpack->type) { diff --git a/src/third_party/wiredtiger/src/btree/bt_random.c b/src/third_party/wiredtiger/src/btree/bt_random.c new file mode 100644 index 00000000000..c5948ec4ab5 --- /dev/null +++ b/src/third_party/wiredtiger/src/btree/bt_random.c @@ -0,0 +1,432 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_row_random_leaf -- + * Return a random key from a row-store leaf page. + */ +int +__wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) +{ + WT_INSERT *ins, **start, **stop; + WT_INSERT_HEAD *ins_head; + WT_PAGE *page; + uint64_t samples; + uint32_t choice, entries, i; + int level; + + page = cbt->ref->page; + start = stop = NULL; /* [-Wconditional-uninitialized] */ + entries = 0; /* [-Wconditional-uninitialized] */ + + __cursor_pos_clear(cbt); + + /* If the page has disk-based entries, select from them. */ + if (page->entries != 0) { + cbt->compare = 0; + cbt->slot = __wt_random(&session->rnd) % page->entries; + + /* + * The real row-store search function builds the key, so we + * have to as well. + */ + return (__wt_row_leaf_key(session, + page, page->pg_row + cbt->slot, cbt->tmp, false)); + } + + /* + * If the tree is new (and not empty), it might have a large insert + * list. + * + * Walk down the list until we find a level with at least 50 entries, + * that's where we'll start rolling random numbers. The value 50 is + * used to ignore levels with only a few entries, that is, levels which + * are potentially badly skewed. + */ + F_SET(cbt, WT_CBT_SEARCH_SMALLEST); + if ((ins_head = WT_ROW_INSERT_SMALLEST(page)) == NULL) + return (WT_NOTFOUND); + for (level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { + start = &ins_head->head[level]; + for (entries = 0, stop = start; + *stop != NULL; stop = &(*stop)->next[level]) + ++entries; + + if (entries > 50) + break; + } + + /* + * If it's a tiny list and we went all the way to level 0, correct the + * level; entries is correctly set. + */ + if (level < 0) + level = 0; + + /* + * Step down the skip list levels, selecting a random chunk of the name + * space at each level. + */ + for (samples = entries; level > 0; samples += entries) { + /* + * There are (entries) or (entries + 1) chunks of the name space + * considered at each level. They are: between start and the 1st + * element, between the 1st and 2nd elements, and so on to the + * last chunk which is the name space after the stop element on + * the current level. This last chunk of name space may or may + * not be there: as we descend the levels of the skip list, this + * chunk may appear, depending if the next level down has + * entries logically after the stop point in the current level. + * We can't ignore those entries: because of the algorithm used + * to determine the depth of a skiplist, there may be a large + * number of entries "revealed" by descending a level. + * + * If the next level down has more items after the current stop + * point, there are (entries + 1) chunks to consider, else there + * are (entries) chunks. + */ + if (*(stop - 1) == NULL) + choice = __wt_random(&session->rnd) % entries; + else + choice = __wt_random(&session->rnd) % (entries + 1); + + if (choice == entries) { + /* + * We selected the name space after the stop element on + * this level. Set the start point to the current stop + * point, descend a level and move the stop element to + * the end of the list, that is, the end of the newly + * discovered name space, counting entries as we go. + */ + start = stop; + --start; + --level; + for (entries = 0, stop = start; + *stop != NULL; stop = &(*stop)->next[level]) + ++entries; + } else { + /* + * We selected another name space on the level. Move the + * start pointer the selected number of entries forward + * to the start of the selected chunk (if the selected + * number is 0, start won't move). Set the stop pointer + * to the next element in the list and drop both start + * and stop down a level. + */ + for (i = 0; i < choice; ++i) + start = &(*start)->next[level]; + stop = &(*start)->next[level]; + + --start; + --stop; + --level; + + /* Count the entries in the selected name space. */ + for (entries = 0, + ins = *start; ins != *stop; ins = ins->next[level]) + ++entries; + } + } + + /* + * When we reach the bottom level, entries will already be set. Select + * a random entry from the name space and return it. + * + * It should be impossible for the entries count to be 0 at this point, + * but check for it out of paranoia and to quiet static testing tools. + */ + if (entries > 0) + entries = __wt_random(&session->rnd) % entries; + for (ins = *start; entries > 0; --entries) + ins = ins->next[0]; + + cbt->ins = ins; + cbt->ins_head = ins_head; + cbt->compare = 0; + + /* + * Random lookups in newly created collections can be slow if a page + * consists of a large skiplist. Schedule the page for eviction if we + * encounter a large skiplist. This worthwhile because applications + * that take a sample often take many samples, so the overhead of + * traversing the skip list each time accumulates to real time. + */ + if (samples > 5000) + __wt_page_evict_soon(session, cbt->ref); + + return (0); +} + +/* + * __wt_random_descent -- + * Find a random page in a tree for either sampling or eviction. + */ +int +__wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, bool eviction) +{ + WT_BTREE *btree; + WT_DECL_RET; + WT_PAGE *page; + WT_PAGE_INDEX *pindex; + WT_REF *current, *descent; + uint32_t flags, i, entries, retry; + + *refp = NULL; + + btree = S2BT(session); + current = NULL; + retry = 100; + + /* Eviction should not be tapped to do eviction. */ + if (eviction) + flags = WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | + WT_READ_NO_WAIT | WT_READ_NOTFOUND_OK | WT_READ_RESTART_OK; + else + flags = WT_READ_RESTART_OK; + + if (0) { +restart: /* + * Discard the currently held page and restart the search from + * the root. + */ + WT_RET(__wt_page_release(session, current, flags)); + } + + /* Search the internal pages of the tree. */ + current = &btree->root; + for (;;) { + page = current->page; + if (!WT_PAGE_IS_INTERNAL(page)) + break; + + WT_INTL_INDEX_GET(session, page, pindex); + entries = pindex->entries; + + /* Eviction just wants any random child. */ + if (eviction) { + descent = pindex->index[ + __wt_random(&session->rnd) % entries]; + goto descend; + } + + /* + * There may be empty pages in the tree, and they're useless to + * us. If we don't find a non-empty page in "entries" random + * guesses, take the first non-empty page in the tree. If the + * search page contains nothing other than empty pages, restart + * from the root some number of times before giving up. + * + * Random sampling is looking for a key/value pair on a random + * leaf page, and so will accept any page that contains a valid + * key/value pair, so on-disk is fine, but deleted is not. + */ + descent = NULL; + for (i = 0; i < entries; ++i) { + descent = + pindex->index[__wt_random(&session->rnd) % entries]; + if (descent->state == WT_REF_MEM || + descent->state == WT_REF_DISK) + break; + } + if (i == entries) + for (i = 0; i < entries; ++i) { + descent = pindex->index[i]; + if (descent->state == WT_REF_MEM || + descent->state == WT_REF_DISK) + break; + } + if (i == entries || descent == NULL) { + if (--retry > 0) + goto restart; + + WT_RET(__wt_page_release(session, current, flags)); + return (WT_NOTFOUND); + } + + /* + * Swap the current page for the child page. If the page splits + * while we're retrieving it, restart the search at the root. + * + * On other error, simply return, the swap call ensures we're + * holding nothing on failure. + */ +descend: if ((ret = + __wt_page_swap(session, current, descent, flags)) == 0) { + current = descent; + continue; + } + if (eviction && (ret == WT_NOTFOUND || ret == WT_RESTART)) + break; + if (ret == WT_RESTART) + goto restart; + return (ret); + } + + /* + * There is no point starting with the root page: the walk will exit + * immediately. In that case we aren't holding a hazard pointer so + * there is nothing to release. + */ + if (!eviction || !__wt_ref_is_root(current)) + *refp = current; + return (0); +} + +/* + * __wt_btcur_next_random -- + * Move to a random record in the tree. There are two algorithms, one + * where we select a record at random from the whole tree on each + * retrieval and one where we first select a record at random from the + * whole tree, and then subsequently sample forward from that location. + * The sampling approach allows us to select reasonably uniform random + * points from unbalanced trees. + */ +int +__wt_btcur_next_random(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION_IMPL *session; + WT_UPDATE *upd; + wt_off_t size; + uint64_t n, skip; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cbt->iface.session; + + /* + * Only supports row-store: applications can trivially select a random + * value from a column-store, if there were any reason to do so. + */ + if (btree->type != BTREE_ROW) + WT_RET_MSG(session, ENOTSUP, + "WT_CURSOR.next_random only supported by row-store tables"); + + WT_STAT_CONN_INCR(session, cursor_next); + WT_STAT_DATA_INCR(session, cursor_next); + + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + +#ifdef HAVE_DIAGNOSTIC + /* + * Under some conditions we end up using the underlying cursor.next to + * walk through the object. Since there are multiple calls, we can hit + * the cursor-order checks, turn them off. + */ + __wt_cursor_key_order_reset(cbt); +#endif + /* + * If we don't have a current position in the tree, or if retrieving + * random values without sampling, pick a roughly random leaf page in + * the tree and return an entry from it. + */ + if (cbt->ref == NULL || cbt->next_random_sample_size == 0) { + WT_ERR(__cursor_func_init(cbt, true)); + WT_WITH_PAGE_INDEX(session, + ret = __wt_random_descent(session, &cbt->ref, false)); + if (ret == 0) + goto random_page_entry; + + /* + * Random descent may return not-found: the tree might be empty + * or have so many deleted items we didn't find any valid pages. + * We can't return WT_NOTFOUND to the application unless a tree + * is really empty, fallback to skipping through tree pages. + */ + WT_ERR_NOTFOUND_OK(ret); + } + + /* + * Cursor through the tree, skipping past the sample size of the leaf + * pages in the tree between each random key return to compensate for + * unbalanced trees. + * + * If the random descent attempt failed, we don't have a configured + * sample size, use 100 for no particular reason. + */ + if (cbt->next_random_sample_size == 0) + cbt->next_random_sample_size = 100; + + /* + * If the random descent attempt failed, or it's our first skip attempt, + * we haven't yet set the pages to skip, do it now. + * + * Use the underlying file size divided by its block allocation size as + * our guess of leaf pages in the file (this can be entirely wrong, as + * it depends on how many pages are in this particular checkpoint, how + * large the leaf and internal pages really are, and other factors). + * Then, divide that value by the configured sample size and increment + * the final result to make sure tiny files don't leave us with a skip + * value of 0. + * + * !!! + * Ideally, the number would be prime to avoid restart issues. + */ + if (cbt->next_random_leaf_skip == 0) { + WT_ERR(btree->bm->size(btree->bm, session, &size)); + cbt->next_random_leaf_skip = (uint64_t) + ((size / btree->allocsize) / + cbt->next_random_sample_size) + 1; + } + + /* + * Be paranoid about loop termination: first, if the last leaf page + * skipped was also the last leaf page in the tree, skip may be set to + * zero on return along with the NULL WT_REF end-of-walk condition. + * Second, if a tree has no valid pages at all (the condition after + * initial creation), we might make no progress at all, or finally, if + * a tree has only deleted pages, we'll make progress, but never get a + * useful WT_REF. And, of course, the tree can switch from one of these + * states to another without warning. Decrement skip regardless of what + * is happening in the search, guarantee we eventually quit. + * + * Pages read for data sampling aren't "useful"; don't update the read + * generation of pages already in memory, and if a page is read, set + * its generation to a low value so it is evicted quickly. + */ + for (skip = cbt->next_random_leaf_skip; cbt->ref == NULL || skip > 0;) { + n = skip; + WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip, + WT_READ_NO_GEN | WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + if (n == skip) { + if (skip == 0) + break; + --skip; + } + } + + /* + * We can't return WT_NOTFOUND to the application unless a tree is + * really empty, fallback to a random entry from the first page in the + * tree that has anything at all. + */ + if (cbt->ref == NULL) + WT_ERR(__wt_btcur_next(cbt, false)); + +random_page_entry: + /* + * Select a random entry from the leaf page. If it's not valid, move to + * the next entry, if that doesn't work, move to the previous entry. + */ + WT_ERR(__wt_row_random_leaf(session, cbt)); + if (__wt_cursor_valid(cbt, &upd)) + WT_ERR(__wt_kv_return(session, cbt, upd)); + else { + if ((ret = __wt_btcur_next(cbt, false)) == WT_NOTFOUND) + ret = __wt_btcur_prev(cbt, false); + WT_ERR(ret); + } + return (0); + +err: WT_TRET(__cursor_reset(cbt)); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 90188498535..ae1f8427b25 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -90,7 +90,7 @@ __col_instantiate(WT_SESSION_IMPL *session, { /* Search the page and add updates. */ WT_RET(__wt_col_search(session, recno, ref, cbt)); - WT_RET(__wt_col_modify(session, cbt, recno, NULL, upd, false)); + WT_RET(__wt_col_modify(session, cbt, recno, NULL, upd, false, false)); return (0); } @@ -104,7 +104,7 @@ __row_instantiate(WT_SESSION_IMPL *session, { /* Search the page and add updates. */ WT_RET(__wt_row_search(session, key, ref, cbt, true)); - WT_RET(__wt_row_modify(session, cbt, key, NULL, upd, false)); + WT_RET(__wt_row_modify(session, cbt, key, NULL, upd, false, false)); return (0); } @@ -324,7 +324,7 @@ __evict_force_check(WT_SESSION_IMPL *session, WT_REF *ref) * If this session has more than one hazard pointer, eviction will fail * and there is no point trying. */ - if (__wt_hazard_count(session, page) > 1) + if (__wt_hazard_count(session, ref) > 1) return (false); /* If we can do an in-memory split, do it. */ @@ -369,6 +369,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref) size_t addr_size; uint32_t previous_state; const uint8_t *addr; + bool timer; btree = S2BT(session); page = NULL; @@ -408,10 +409,11 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref) * There's an address, read or map the backing disk page and build an * in-memory version of the page. */ - if (!F_ISSET(session, WT_SESSION_INTERNAL)) + timer = !F_ISSET(session, WT_SESSION_INTERNAL); + if (timer) __wt_epoch(session, &start); WT_ERR(__wt_bt_read(session, &tmp, addr, addr_size)); - if (!F_ISSET(session, WT_SESSION_INTERNAL)) { + if (timer) { __wt_epoch(session, &stop); WT_STAT_CONN_INCR(session, cache_read_app_count); WT_STAT_CONN_INCRV(session, cache_read_app_time, @@ -590,8 +592,9 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags */ if (LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(session, WT_SESSION_NO_EVICTION) || - (F_ISSET(btree, WT_BTREE_NO_EVICTION) && - !F_ISSET(btree, WT_BTREE_NO_RECONCILE))) + btree->lsm_primary || + (btree->evict_disabled > 0 && + !F_ISSET(btree, WT_BTREE_ALLOW_SPLITS))) goto skip_evict; /* diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 29380459b94..68848c7c8f5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -265,7 +265,7 @@ __rebalance_row_leaf_key(WT_SESSION_IMPL *session, */ WT_RET(__wt_bt_read(session, rs->tmp1, addr, addr_len)); WT_RET(__wt_page_inmem(session, NULL, rs->tmp1->data, 0, 0, &page)); - ret = __wt_row_leaf_key_copy(session, page, &page->pg_row_d[0], key); + ret = __wt_row_leaf_key_copy(session, page, &page->pg_row[0], key); __wt_page_out(session, &page); return (ret); } @@ -406,12 +406,10 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_BTREE *btree; WT_DECL_RET; WT_REBALANCE_STUFF *rs, _rstuff; - bool evict_reset; WT_UNUSED(cfg); btree = S2BT(session); - evict_reset = false; /* * If the tree has never been written to disk, we're done, rebalance @@ -433,14 +431,6 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) /* Set the internal page tree type. */ rs->type = btree->root.page->type; - /* - * Get exclusive access to the file. (Not required, the only page in the - * cache is the root page, and that cannot be evicted; however, this way - * eviction ignores the tree entirely.) - */ - WT_ERR(__wt_evict_file_exclusive_on(session)); - evict_reset = true; - /* Recursively walk the tree. */ switch (rs->type) { case WT_PAGE_ROW_INT: @@ -471,10 +461,7 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) btree->root.page = rs->root; rs->root = NULL; -err: if (evict_reset) - __wt_evict_file_exclusive_off(session); - - /* Discard any leftover root page we created. */ +err: /* Discard any leftover root page we created. */ if (rs->root != NULL) { __wt_page_modify_clear(session, rs->root); __wt_page_out(session, &rs->root); diff --git a/src/third_party/wiredtiger/src/btree/bt_ret.c b/src/third_party/wiredtiger/src/btree/bt_ret.c index 8ef2db67e7b..f17fa1b85d1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ret.c +++ b/src/third_party/wiredtiger/src/btree/bt_ret.c @@ -9,65 +9,22 @@ #include "wt_internal.h" /* - * __wt_kv_return -- - * Return a page referenced key/value pair to the application. + * __key_return -- + * Change the cursor to reference an internal return key. */ -int -__wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +static inline int +__key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) { - WT_BTREE *btree; - WT_CELL *cell; - WT_CELL_UNPACK unpack; WT_CURSOR *cursor; WT_ITEM *tmp; WT_PAGE *page; WT_ROW *rip; - uint8_t v; - - btree = S2BT(session); page = cbt->ref->page; cursor = &cbt->iface; - switch (page->type) { - case WT_PAGE_COL_FIX: - /* - * The interface cursor's record has usually been set, but that - * isn't universally true, specifically, cursor.search_near may - * call here without first setting the interface cursor. - */ - cursor->recno = cbt->recno; - - /* If the cursor references a WT_UPDATE item, return it. */ - if (upd != NULL) { - cursor->value.data = WT_UPDATE_DATA(upd); - cursor->value.size = upd->size; - return (0); - } - - /* Take the value from the original page. */ - v = __bit_getv_recno(cbt->ref, cursor->recno, btree->bitcnt); - return (__wt_buf_set(session, &cursor->value, &v, 1)); - case WT_PAGE_COL_VAR: - /* - * The interface cursor's record has usually been set, but that - * isn't universally true, specifically, cursor.search_near may - * call here without first setting the interface cursor. - */ - cursor->recno = cbt->recno; - - /* If the cursor references a WT_UPDATE item, return it. */ - if (upd != NULL) { - cursor->value.data = WT_UPDATE_DATA(upd); - cursor->value.size = upd->size; - return (0); - } - - /* Take the value from the original page cell. */ - cell = WT_COL_PTR(page, &page->pg_var_d[cbt->slot]); - break; - case WT_PAGE_ROW_LEAF: - rip = &page->pg_row_d[cbt->slot]; + if (page->type == WT_PAGE_ROW_LEAF) { + rip = &page->pg_row[cbt->slot]; /* * If the cursor references a WT_INSERT item, take its key. @@ -79,7 +36,10 @@ __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) if (cbt->ins != NULL) { cursor->key.data = WT_INSERT_KEY(cbt->ins); cursor->key.size = WT_INSERT_KEY_SIZE(cbt->ins); - } else if (cbt->compare == 0) { + return (0); + } + + if (cbt->compare == 0) { /* * If not in an insert list and there's an exact match, * the row-store search function built the key we want @@ -97,16 +57,51 @@ __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) cursor->key.data = cbt->row_key->data; cursor->key.size = cbt->row_key->size; - } else - WT_RET(__wt_row_leaf_key( - session, page, rip, &cursor->key, false)); - - /* If the cursor references a WT_UPDATE item, return it. */ - if (upd != NULL) { - cursor->value.data = WT_UPDATE_DATA(upd); - cursor->value.size = upd->size; return (0); } + return (__wt_row_leaf_key( + session, page, rip, &cursor->key, false)); + } + + /* + * WT_PAGE_COL_FIX, WT_PAGE_COL_VAR: + * The interface cursor's record has usually been set, but that + * isn't universally true, specifically, cursor.search_near may call + * here without first setting the interface cursor. + */ + cursor->recno = cbt->recno; + return (0); +} + +/* + * __value_return -- + * Change the cursor to reference an internal return value. + */ +static inline int +__value_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK unpack; + WT_CURSOR *cursor; + WT_PAGE *page; + WT_ROW *rip; + uint8_t v; + + btree = S2BT(session); + + page = cbt->ref->page; + cursor = &cbt->iface; + + /* If the cursor references a WT_UPDATE item, return it. */ + if (upd != NULL) { + cursor->value.data = WT_UPDATE_DATA(upd); + cursor->value.size = upd->size; + return (0); + } + + if (page->type == WT_PAGE_ROW_LEAF) { + rip = &page->pg_row[cbt->slot]; /* Simple values have their location encoded in the WT_ROW. */ if (__wt_row_leaf_value(page, rip, &cursor->value)) @@ -121,13 +116,65 @@ __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) cursor->value.size = 0; return (0); } - break; - WT_ILLEGAL_VALUE(session); + __wt_cell_unpack(cell, &unpack); + return (__wt_page_cell_data_ref( + session, page, &unpack, &cursor->value)); + } - /* The value is an on-page cell, unpack and expand it as necessary. */ - __wt_cell_unpack(cell, &unpack); - WT_RET(__wt_page_cell_data_ref(session, page, &unpack, &cursor->value)); + if (page->type == WT_PAGE_COL_VAR) { + /* Take the value from the original page cell. */ + cell = WT_COL_PTR(page, &page->pg_var[cbt->slot]); + __wt_cell_unpack(cell, &unpack); + return (__wt_page_cell_data_ref( + session, page, &unpack, &cursor->value)); + } + + /* WT_PAGE_COL_FIX: Take the value from the original page. */ + v = __bit_getv_recno(cbt->ref, cursor->recno, btree->bitcnt); + return (__wt_buf_set(session, &cursor->value, &v, 1)); +} + +/* + * __wt_key_return -- + * Change the cursor to reference an internal return key. + */ +int +__wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) +{ + WT_CURSOR *cursor; + + cursor = &cbt->iface; + + /* + * We may already have an internal key, in which case the cursor may + * not be set up to get another copy (for example, when we rely on a + * search-function result). + */ + F_CLR(cursor, WT_CURSTD_KEY_EXT); + if (!F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_RET(__key_return(session, cbt)); + F_SET(cursor, WT_CURSTD_KEY_INT); + } + return (0); +} + +/* + * __wt_kv_return -- + * Return a page referenced key/value pair to the application. + */ +int +__wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +{ + WT_CURSOR *cursor; + + cursor = &cbt->iface; + + WT_RET(__wt_key_return(session, cbt)); + + F_CLR(cursor, WT_CURSTD_VALUE_EXT); + WT_RET(__value_return(session, cbt, upd)); + F_SET(cursor, WT_CURSTD_VALUE_INT); return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index f269c2d7f43..165f932afb2 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -166,13 +166,11 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_DECL_RET; WT_STUFF *ss, stuff; uint32_t i, leaf_cnt; - bool evict_reset; WT_UNUSED(cfg); btree = S2BT(session); bm = btree->bm; - evict_reset = false; WT_CLEAR(stuff); ss = &stuff; @@ -183,13 +181,6 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_ERR(__wt_scr_alloc(session, 0, &ss->tmp1)); WT_ERR(__wt_scr_alloc(session, 0, &ss->tmp2)); - /* - * Salvage handles its own page eviction; get exclusive access to the - * file, have eviction ignore the tree entirely. - */ - WT_ERR(__wt_evict_file_exclusive_on(session)); - evict_reset = true; - /* * Step 1: * Inform the underlying block manager that we're salvaging the file. @@ -350,9 +341,6 @@ err: WT_TRET(bm->salvage_end(bm, session)); if (ss->root_ref.page != NULL) __wt_ref_out(session, &ss->root_ref); - if (evict_reset) - __wt_evict_file_exclusive_off(session); - /* Discard the leaf and overflow page memory. */ WT_TRET(__slvg_cleanup(session, ss)); @@ -603,9 +591,9 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, */ WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, 0, &page)); WT_ERR(__wt_row_leaf_key_copy(session, - page, &page->pg_row_d[0], &trk->row_start)); - WT_ERR(__wt_row_leaf_key_copy(session, page, - &page->pg_row_d[page->pg_row_entries - 1], &trk->row_stop)); + page, &page->pg_row[0], &trk->row_start)); + WT_ERR(__wt_row_leaf_key_copy(session, + page, &page->pg_row[page->entries - 1], &trk->row_stop)); __wt_verbose(session, WT_VERB_SALVAGE, "%s start key %s", @@ -1235,7 +1223,7 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) WT_PAGE *page; WT_SALVAGE_COOKIE *cookie, _cookie; uint64_t recno, skip, take; - uint32_t *entriesp, save_entries; + uint32_t save_entries; cookie = &_cookie; WT_CLEAR(*cookie); @@ -1244,11 +1232,8 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) WT_RET(__wt_page_in(session, ref, 0)); page = ref->page; - entriesp = page->type == WT_PAGE_COL_VAR ? - &page->pg_var_entries : &page->pg_fix_entries; - - save_col_var = page->pg_var_d; - save_entries = *entriesp; + save_col_var = page->pg_var; + save_entries = page->entries; /* * Calculate the number of K/V entries we are going to skip, and @@ -1300,11 +1285,11 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) /* Write the new version of the leaf page to disk. */ WT_ERR(__slvg_modify_init(session, page)); - WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR)); + WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR, NULL)); /* Reset the page. */ - page->pg_var_d = save_col_var; - *entriesp = save_entries; + page->pg_var = save_col_var; + page->entries = save_entries; ret = __wt_page_release(session, ref, 0); if (ret == 0) @@ -1973,14 +1958,14 @@ __slvg_row_build_leaf( /* We should have selected some entries, but not the entire page. */ WT_ASSERT(session, skip_start + skip_stop > 0 && - skip_start + skip_stop < page->pg_row_entries); + skip_start + skip_stop < page->entries); /* * Take a copy of this page's first key to define the start of * its range. The key may require processing, otherwise, it's * a copy from the page. */ - rip = page->pg_row_d + skip_start; + rip = page->pg_row + skip_start; WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); WT_ERR(__wt_row_ikey_incr( session, ref->home, 0, key->data, key->size, ref)); @@ -1988,14 +1973,14 @@ __slvg_row_build_leaf( /* Set the referenced flag on overflow pages we're using. */ if (trk->trk_ovfl_cnt != 0) WT_ERR(__slvg_row_ovfl(session, - trk, page, skip_start, page->pg_row_entries - skip_stop)); + trk, page, skip_start, page->entries - skip_stop)); /* * Change the page to reflect the correct record count: there is no * need to copy anything on the page itself, the entries value limits * the number of page items. */ - page->pg_row_entries -= skip_stop; + page->entries -= skip_stop; cookie->skip = skip_start; /* @@ -2011,10 +1996,10 @@ __slvg_row_build_leaf( /* Write the new version of the leaf page to disk. */ WT_ERR(__slvg_modify_init(session, page)); - WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR)); + WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR, NULL)); /* Reset the page. */ - page->pg_row_entries += skip_stop; + page->entries += skip_stop; /* * Discard our hazard pointer and evict the page, updating the @@ -2081,7 +2066,7 @@ __slvg_row_ovfl(WT_SESSION_IMPL *session, * We're merging a row-store page, and we took some number of records, * figure out which (if any) overflow records we used. */ - for (rip = page->pg_row_d + start; start < stop; ++start, ++rip) { + for (rip = page->pg_row + start; start < stop; ++start, ++rip) { copy = WT_ROW_KEY_COPY(rip); (void)__wt_row_leaf_key_info( page, copy, NULL, &cell, NULL, NULL); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 017c820ea29..92be2125b88 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -10,8 +10,8 @@ #define WT_MEM_TRANSFER(from_decr, to_incr, len) do { \ size_t __len = (len); \ - from_decr += __len; \ - to_incr += __len; \ + (from_decr) += __len; \ + (to_incr) += __len; \ } while (0) /* @@ -53,6 +53,16 @@ __split_oldest_gen(WT_SESSION_IMPL *session) return (oldest); } +/* + * __wt_split_obsolete -- + * Check if it is safe to free / evict based on split generation. + */ +bool +__wt_split_obsolete(WT_SESSION_IMPL *session, uint64_t split_gen) +{ + return (split_gen < __split_oldest_gen(session)); +} + /* * __split_stash_add -- * Add a new entry into the session's split stash list. @@ -109,7 +119,7 @@ __wt_split_stash_discard(WT_SESSION_IMPL *session) ++i, ++stash) { if (stash->p == NULL) continue; - else if (stash->split_gen >= oldest) + if (stash->split_gen >= oldest) break; /* * It's a bad thing if another thread is in this memory after @@ -177,7 +187,7 @@ __split_safe_free(WT_SESSION_IMPL *session, exclusive = true; if (exclusive) { - __wt_free(session, p); + __wt_overwrite_and_free_len(session, p, s); return (0); } @@ -187,7 +197,7 @@ __split_safe_free(WT_SESSION_IMPL *session, #ifdef HAVE_DIAGNOSTIC /* * __split_verify_intl_key_order -- - * Verify the key order on an internal page after a split, diagnostic only. + * Verify the key order on an internal page after a split. */ static void __split_verify_intl_key_order(WT_SESSION_IMPL *session, WT_PAGE *page) @@ -239,6 +249,46 @@ __split_verify_intl_key_order(WT_SESSION_IMPL *session, WT_PAGE *page) break; } } + +/* + * __split_verify_root -- + * Verify a root page involved in a split. + */ +static int +__split_verify_root(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_DECL_RET; + WT_REF *ref; + + /* The split is complete and live, verify all of the pages involved. */ + __split_verify_intl_key_order(session, page); + + WT_INTL_FOREACH_BEGIN(session, page, ref) { + /* + * An eviction thread might be attempting to evict the page + * (the WT_REF may be WT_REF_LOCKED), or it may be a disk based + * page (the WT_REF may be WT_REF_READING), or it may be in + * some other state. Acquire a hazard pointer for any + * in-memory pages so we know the state of the page. + * + * Ignore pages not in-memory (deleted, on-disk, being read), + * there's no in-memory structure to check. + */ + if ((ret = __wt_page_in(session, + ref, WT_READ_CACHE | WT_READ_NO_EVICT)) == WT_NOTFOUND) + continue; + WT_ERR(ret); + + __split_verify_intl_key_order(session, ref->page); + + WT_ERR(__wt_page_release(session, ref, WT_READ_NO_EVICT)); + } WT_INTL_FOREACH_END; + + return (0); + +err: /* Something really bad just happened. */ + WT_PANIC_RET(session, ret, "fatal error during page split"); +} #endif /* @@ -390,12 +440,12 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, } /* - * __split_ref_step1 -- + * __split_ref_prepare -- * Prepare a set of WT_REFs for a move. */ static void -__split_ref_step1( - WT_SESSION_IMPL *session, WT_PAGE_INDEX *pindex, bool skip_first) +__split_ref_prepare(WT_SESSION_IMPL *session, + WT_PAGE_INDEX *pindex, uint64_t split_gen, bool skip_first) { WT_PAGE *child; WT_REF *child_ref, *ref; @@ -418,30 +468,25 @@ __split_ref_step1( child = ref->page; /* - * Block eviction and splits in newly created pages. + * Block eviction in newly created pages. * * Once the split is live, newly created internal pages might be * evicted and their WT_REF structures freed. If that happened * before all threads exit the index of the page that previously * "owned" the WT_REF, a thread might see a freed WT_REF. To - * ensure that doesn't happen, the newly created page's modify - * structure has a field with a transaction ID that's checked - * before any internal page is evicted. Unfortunately, we don't - * know the correct value until we update the original page's - * index (we need a transaction ID from after that update), but - * the act of updating the original page's index is what allows - * the eviction to happen. + * ensure that doesn't happen, the newly created page contains + * the current split generation and can't be evicted until + * all readers have left the old generation. * - * Split blocking was because historic versions of the split - * code didn't update the WT_REF.home field until after the - * split was live, so the WT_REF.home fields being updated could - * split again before the update, there's a race between splits - * as to which would update them first. The current code updates - * the WT_REF.home fields before going live (in this function), - * this shouldn't be an issue, but for now splits remain turned - * off. + * Historic, we also blocked splits in newly created pages + * because we didn't update the WT_REF.home field until after + * the split was live, so the WT_REF.home fields being updated + * could split again before the update, there's a race between + * splits as to which would update them first. The current code + * updates the WT_REF.home fields before going live (in this + * function), this isn't an issue. */ - F_SET_ATOMIC(child, WT_PAGE_SPLIT_BLOCK); + child->pg_intl_split_gen = split_gen; /* * We use a page flag to prevent the child from splitting from @@ -464,64 +509,6 @@ __split_ref_step1( } } -/* - * __split_ref_step2 -- - * Allow the newly created children to be evicted or split. - */ -static int -__split_ref_step2( - WT_SESSION_IMPL *session, WT_PAGE_INDEX *pindex, bool skip_first) -{ - WT_DECL_RET; - WT_PAGE *child; - WT_REF *ref; - uint32_t i; - - /* - * The split has gone live, enable eviction and splits on the newly - * created internal pages. - */ - WT_WRITE_BARRIER(); - - for (i = skip_first ? 1 : 0; i < pindex->entries; ++i) { - ref = pindex->index[i]; - - /* - * We don't hold hazard pointers on created pages, they cannot - * be evicted because the page-modify transaction value set as - * they were created prevents eviction. (See above, we reset - * that value as part of fixing up the page.) But, an eviction - * thread might be attempting to evict the page (the WT_REF may - * be WT_REF_LOCKED), or it may be a disk based page (the WT_REF - * may be WT_REF_READING), or it may be in some other state. - * Acquire a hazard pointer for any in-memory pages so we know - * the state of the page. Ignore pages not in-memory (deleted, - * on-disk, being read), there's no in-memory structure to fix. - */ - if ((ret = __wt_page_in(session, - ref, WT_READ_CACHE | WT_READ_NO_EVICT)) == WT_NOTFOUND) - continue; - WT_ERR(ret); - - child = ref->page; - - /* The child can now be evicted or split. */ - F_CLR_ATOMIC(child, WT_PAGE_SPLIT_BLOCK); - -#ifdef HAVE_DIAGNOSTIC - WT_WITH_PAGE_INDEX(session, - __split_verify_intl_key_order(session, child)); -#endif - - WT_ERR(__wt_hazard_clear(session, child)); - } - - return (0); - -err: /* Something really bad just happened. */ - WT_PANIC_RET(session, ret, "fatal error resolving a split"); -} - /* * __split_root -- * Split the root page in-memory, deepening the tree. @@ -653,8 +640,12 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) /* Start making real changes to the tree, errors are fatal. */ complete = WT_ERR_PANIC; - /* Prepare the WT_REFs for the move. */ - __split_ref_step1(session, alloc_index, false); + /* + * Prepare the WT_REFs for the move: this requires a stable split + * generation to block splits in newly created pages, so get one. + */ + WT_ENTER_PAGE_INDEX(session); + __split_ref_prepare(session, alloc_index, session->split_gen, false); /* * Confirm the root page's index hasn't moved, then update it, which @@ -662,20 +653,27 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) */ WT_ASSERT(session, WT_INTL_INDEX_GET_SAFE(root) == pindex); WT_INTL_INDEX_SET(root, alloc_index); + alloc_index = NULL; + + WT_LEAVE_PAGE_INDEX(session); + + /* + * Get a generation for this split, mark the root page. This must be + * after the new index is swapped into place in order to know that no + * readers are looking at the old index. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + root->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, - __split_verify_intl_key_order(session, root)); + ret = __split_verify_root(session, root)); + WT_ERR(ret); #endif - /* Finalize the WT_REFs we moved. */ - WT_ERR(__split_ref_step2(session, alloc_index, false)); - /* The split is complete and correct, ignore benign errors. */ + /* The split is complete and verified, ignore benign errors. */ complete = WT_ERR_IGNORE; - /* We've installed the allocated page-index, ensure error handling. */ - alloc_index = NULL; - /* * We can't free the previous root's index, there may be threads using * it. Add to the session's discard list, to be freed once we know no @@ -686,7 +684,6 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) * fails, we don't roll back that change, because threads may already * be using the new index. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); size = sizeof(WT_PAGE_INDEX) + pindex->entries * sizeof(WT_REF *); WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); root_decr += size; @@ -846,10 +843,13 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, WT_INTL_INDEX_SET(parent, alloc_index); alloc_index = NULL; -#ifdef HAVE_DIAGNOSTIC - WT_WITH_PAGE_INDEX(session, - __split_verify_intl_key_order(session, parent)); -#endif + /* + * Get a generation for this split, mark the page. This must be after + * the new index is swapped into place in order to know that no readers + * are looking at the old index. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + parent->pg_intl_split_gen = split_gen; /* * If discarding the page's original WT_REF field, reset it to split. @@ -869,16 +869,25 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, __wt_free(session, ref->page_del); } + /* + * Set the discarded WT_REF state to split, ensuring we don't + * race with any discard of the WT_REF deleted fields. + */ WT_PUBLISH(ref->state, WT_REF_SPLIT); + + /* + * Push out the change: not required for correctness, but stops + * threads spinning on incorrect page references. + */ + WT_FULL_BARRIER(); } - /* - * Push out the changes: not required for correctness, but don't let - * threads spin on incorrect page references longer than necessary. - */ - WT_FULL_BARRIER(); +#ifdef HAVE_DIAGNOSTIC + WT_WITH_PAGE_INDEX(session, + __split_verify_intl_key_order(session, parent)); +#endif - /* The split is complete and correct, ignore benign errors. */ + /* The split is complete and verified, ignore benign errors. */ complete = WT_ERR_IGNORE; /* @@ -908,7 +917,6 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, * * Acquire a new split generation. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); for (i = 0, deleted_refs = scr->mem; i < deleted_entries; ++i) { next_ref = pindex->index[deleted_refs[i]]; WT_ASSERT(session, next_ref->state == WT_REF_SPLIT); @@ -1160,16 +1168,34 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) /* Start making real changes to the tree, errors are fatal. */ complete = WT_ERR_PANIC; - /* Prepare the WT_REFs for the move. */ - __split_ref_step1(session, alloc_index, true); + /* + * Prepare the WT_REFs for the move: this requires a stable split + * generation to block splits in newly created pages, so get one. + */ + WT_ENTER_PAGE_INDEX(session); + __split_ref_prepare(session, alloc_index, session->split_gen, true); /* Split into the parent. */ - WT_ERR(__split_parent(session, page_ref, alloc_index->index, - alloc_index->entries, parent_incr, false, false)); + if ((ret = __split_parent(session, page_ref, alloc_index->index, + alloc_index->entries, parent_incr, false, false)) == 0) { + /* + * Confirm the page's index hasn't moved, then update it, which + * makes the split visible to threads descending the tree. + */ + WT_ASSERT(session, WT_INTL_INDEX_GET_SAFE(page) == pindex); + WT_INTL_INDEX_SET(page, replace_index); + } - /* Confirm the page's index hasn't moved, then update it. */ - WT_ASSERT(session, WT_INTL_INDEX_GET_SAFE(page) == pindex); - WT_INTL_INDEX_SET(page, replace_index); + WT_LEAVE_PAGE_INDEX(session); + WT_ERR(ret); + + /* + * Get a generation for this split, mark the parent page. This must be + * after the new index is swapped into place in order to know that no + * readers are looking at the old index. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + page->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, @@ -1178,18 +1204,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) __split_verify_intl_key_order(session, page)); #endif - /* Finalize the WT_REFs we moved. */ - WT_ERR(__split_ref_step2(session, alloc_index, true)); - - /* The split is complete and correct, ignore benign errors. */ + /* The split is complete and verified, ignore benign errors. */ complete = WT_ERR_IGNORE; - /* - * Push out the changes: not required for correctness, but no reason - * to wait. - */ - WT_FULL_BARRIER(); - /* * We don't care about the page-index we allocated, all we needed was * the array of WT_REF structures, which has now been split into the @@ -1207,7 +1224,6 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * back that change, because threads may already be using the new parent * page. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); size = sizeof(WT_PAGE_INDEX) + pindex->entries * sizeof(WT_REF *); WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); page_decr += size; @@ -1240,12 +1256,12 @@ err: switch (complete) { } /* - * __split_internal_lock -- + * __split_internal_lock_worker -- * Lock an internal page. */ static int -__split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, - WT_PAGE **parentp, bool *hazardp) +__split_internal_lock_worker(WT_SESSION_IMPL *session, + WT_REF *ref, bool trylock, WT_PAGE **parentp, bool *hazardp) { WT_DECL_RET; WT_PAGE *parent; @@ -1284,17 +1300,19 @@ __split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, for (;;) { parent = ref->home; - /* Skip pages that aren't ready to split. */ - if (F_ISSET_ATOMIC(parent, WT_PAGE_SPLIT_BLOCK)) - return (EBUSY); + /* + * The page will be marked dirty, and we can only lock a page + * with a modify structure. + */ + WT_RET(__wt_page_modify_init(session, parent)); if (trylock) - WT_RET(__wt_try_writelock(session, &parent->page_lock)); + WT_RET(WT_PAGE_TRYLOCK(session, parent)); else - __wt_writelock(session, &parent->page_lock); + WT_PAGE_LOCK(session, parent); if (parent == ref->home) break; - __wt_writeunlock(session, &parent->page_lock); + WT_PAGE_UNLOCK(session, parent); } /* @@ -1317,7 +1335,33 @@ __split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, *parentp = parent; return (0); -err: __wt_writeunlock(session, &parent->page_lock); +err: WT_PAGE_UNLOCK(session, parent); + return (ret); +} + +/* + * __split_internal_lock -- + * Lock an internal page. + */ +static int +__split_internal_lock(WT_SESSION_IMPL *session, + WT_REF *ref, bool trylock, WT_PAGE **parentp, bool *hazardp) +{ + WT_DECL_RET; + + /* + * There's no lock on our parent page and we're about to acquire one, + * which implies using the WT_REF.home field to reference our parent + * page. As a child of the parent page, we prevent its eviction, but + * that's a weak guarantee. If the parent page splits, and our WT_REF + * were to move with the split, the WT_REF.home field might change + * underneath us and we could race, and end up attempting to access + * an evicted page. Set the session page-index generation so if the + * parent splits, it still can't be evicted. + */ + WT_WITH_PAGE_INDEX(session, + ret = __split_internal_lock_worker( + session, ref, trylock, parentp, hazardp)); return (ret); } @@ -1331,9 +1375,9 @@ __split_internal_unlock(WT_SESSION_IMPL *session, WT_PAGE *parent, bool hazard) WT_DECL_RET; if (hazard) - ret = __wt_hazard_clear(session, parent); + ret = __wt_hazard_clear(session, parent->pg_intl_parent_ref); - __wt_writeunlock(session, &parent->page_lock); + WT_PAGE_UNLOCK(session, parent); return (ret); } @@ -1547,7 +1591,7 @@ __split_multi_inmem( /* Apply the modification. */ WT_ERR(__wt_col_modify( - session, &cbt, recno, NULL, upd, false)); + session, &cbt, recno, NULL, upd, false, true)); break; case WT_PAGE_ROW_LEAF: /* Build a key. */ @@ -1569,7 +1613,7 @@ __split_multi_inmem( /* Apply the modification. */ WT_ERR(__wt_row_modify( - session, &cbt, key, NULL, upd, false)); + session, &cbt, key, NULL, upd, false, true)); break; WT_ILLEGAL_VALUE_ERR(session); } @@ -1770,9 +1814,9 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) /* Find the last item on the page. */ if (type == WT_PAGE_ROW_LEAF) - ins_head = page->pg_row_entries == 0 ? + ins_head = page->entries == 0 ? WT_ROW_INSERT_SMALLEST(page) : - WT_ROW_INSERT_SLOT(page, page->pg_row_entries - 1); + WT_ROW_INSERT_SLOT(page, page->entries - 1); else ins_head = WT_COL_APPEND(page); moved_ins = WT_SKIP_LAST(ins_head); @@ -1822,7 +1866,7 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) key->size = WT_INSERT_KEY_SIZE(ins); } else WT_ERR(__wt_row_leaf_key( - session, page, &page->pg_row_d[0], key, true)); + session, page, &page->pg_row[0], key, true)); WT_ERR(__wt_row_ikey(session, 0, key->data, key->size, child)); parent_incr += sizeof(WT_IKEY) + key->size; __wt_scr_free(session, &key); @@ -2086,8 +2130,7 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_PAGE *parent; bool hazard; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: split-insert", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: split-insert", (void *)ref); WT_RET(__split_internal_lock(session, ref, true, &parent, &hazard)); if ((ret = __split_insert(session, ref)) != 0) { @@ -2178,8 +2221,7 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing) WT_PAGE *parent; bool hazard; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: split-multi", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: split-multi", (void *)ref); WT_RET(__split_internal_lock(session, ref, false, &parent, &hazard)); if ((ret = __split_multi(session, ref, closing)) != 0 || closing) { @@ -2207,8 +2249,7 @@ __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref) WT_PAGE *parent; bool hazard; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: reverse-split", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: reverse-split", (void *)ref); WT_RET(__split_internal_lock(session, ref, false, &parent, &hazard)); ret = __split_parent(session, ref, NULL, 0, 0, false, true); @@ -2229,8 +2270,7 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi) page = ref->page; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: split-rewrite", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: split-rewrite", (void *)ref); /* * This isn't a split: a reconciliation failed because we couldn't write @@ -2266,7 +2306,7 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi) * reconciliation, do it now. */ __wt_page_modify_clear(session, page); - __wt_ref_out(session, ref); + __wt_ref_out_int(session, ref, true); /* Swap the new page into place. */ ref->page = new->page; diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index 06428b87f6e..0da0e0807bd 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -40,6 +40,8 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) WT_STAT_SET(session, stats, btree_maxleafpage, btree->maxleafpage); WT_STAT_SET(session, stats, btree_maxleafvalue, btree->maxleafvalue); + WT_STAT_SET(session, stats, cache_bytes_dirty, + __wt_btree_dirty_inuse(session)); WT_STAT_SET(session, stats, cache_bytes_inuse, __wt_btree_bytes_inuse(session)); @@ -104,8 +106,7 @@ __stat_page(WT_SESSION_IMPL *session, WT_PAGE *page, WT_DSRC_STATS **stats) switch (page->type) { case WT_PAGE_COL_FIX: WT_STAT_INCR(session, stats, btree_column_fix); - WT_STAT_INCRV( - session, stats, btree_entries, page->pg_fix_entries); + WT_STAT_INCRV(session, stats, btree_entries, page->entries); break; case WT_PAGE_COL_INT: WT_STAT_INCR(session, stats, btree_column_internal); diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 6d4ad9d0d0f..ead6ccc4ac0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -78,6 +78,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; uint64_t oldest_id, saved_pinned_id; uint32_t flags; + bool timer; conn = S2C(session); btree = S2BT(session); @@ -88,7 +89,8 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) internal_bytes = leaf_bytes = 0; internal_pages = leaf_pages = 0; - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + timer = WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT); + if (timer) __wt_epoch(session, &start); switch (syncop) { @@ -133,11 +135,11 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (__wt_page_is_modified(page) && WT_TXNID_LT(page->modify->update_txn, oldest_id)) { if (txn->isolation == WT_ISO_READ_COMMITTED) - WT_ERR(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); leaf_bytes += page->memory_footprint; ++leaf_pages; - WT_ERR(__wt_reconcile( - session, walk, NULL, WT_CHECKPOINTING)); + WT_ERR(__wt_reconcile(session, + walk, NULL, WT_CHECKPOINTING, NULL)); } } break; @@ -155,7 +157,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * the checkpoint are included. */ if (txn->isolation == WT_ISO_READ_COMMITTED) - WT_ERR(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); /* * We cannot check the tree modified flag in the case of a @@ -186,9 +188,9 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * to grow significantly larger than the configured maximum * size. */ - F_SET(btree, WT_BTREE_NO_RECONCILE); + F_SET(btree, WT_BTREE_ALLOW_SPLITS); ret = __wt_evict_file_exclusive_on(session); - F_CLR(btree, WT_BTREE_NO_RECONCILE); + F_CLR(btree, WT_BTREE_ALLOW_SPLITS); WT_ERR(ret); __wt_evict_file_exclusive_off(session); @@ -233,7 +235,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) ++leaf_pages; } WT_ERR(__wt_reconcile( - session, walk, NULL, WT_CHECKPOINTING)); + session, walk, NULL, WT_CHECKPOINTING, NULL)); } break; case WT_SYNC_CLOSE: @@ -242,7 +244,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) break; } - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { + if (timer) { __wt_epoch(session, &end); __wt_verbose(session, WT_VERB_CHECKPOINT, "__sync_file WT_SYNC_%s wrote: %" PRIu64 diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 340f9bb6f0e..7475811adc5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -216,13 +216,11 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) ckpt->raw.data, ckpt->raw.size, root_addr, &root_addr_size, true)); - /* - * Ignore trees with no root page. - * Verify, then discard the checkpoint from the cache. - */ - if (root_addr_size != 0 && - (ret = __wt_btree_tree_open( - session, root_addr, root_addr_size)) == 0) { + /* Skip trees with no root page. */ + if (root_addr_size != 0) { + WT_ERR(__wt_btree_tree_open( + session, root_addr, root_addr_size)); + if (WT_VRFY_DUMP(vs)) WT_ERR(__wt_msg(session, "Root: %s %s", __wt_addr_string(session, @@ -230,14 +228,38 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) __wt_page_type_string( btree->root.page->type))); + __wt_evict_file_exclusive_off(session); + + /* Verify the tree. */ WT_WITH_PAGE_INDEX(session, ret = __verify_tree(session, &btree->root, vs)); + /* + * We have an exclusive lock on the handle, but we're + * swapping root pages in-and-out of that handle, and + * there's a race with eviction entering the tree and + * seeing an invalid root page. Eviction must work on + * trees being verified (else we'd have to do our own + * eviction), lock eviction out whenever we're loading + * a new root page. This loops works because we are + * called with eviction locked out, so we release the + * lock at the top of the loop and re-acquire it here. + */ + WT_TRET(__wt_evict_file_exclusive_on(session)); WT_TRET(__wt_cache_op(session, WT_SYNC_DISCARD)); } /* Unload the checkpoint. */ WT_TRET(bm->checkpoint_unload(bm, session)); + + /* + * We've finished one checkpoint's verification (verification, + * then cache eviction and checkpoint unload): if any errors + * occurred, quit. Done this way because otherwise we'd need + * at least two more state variables on error, one to know if + * we need to discard the tree from the cache and one to know + * if we need to unload the checkpoint. + */ WT_ERR(ret); /* Display the tree shape. */ @@ -252,7 +274,7 @@ err: /* Inform the underlying block manager we're done. */ /* Discard the list of checkpoints. */ if (ckptbase != NULL) - __wt_meta_ckptlist_free(session, ckptbase); + __wt_meta_ckptlist_free(session, &ckptbase); /* Free allocated memory. */ __wt_scr_free(session, &vs->max_key); @@ -386,7 +408,7 @@ recno_chk: if (recno != vs->record_total + 1) } switch (page->type) { case WT_PAGE_COL_FIX: - vs->record_total += page->pg_fix_entries; + vs->record_total += page->entries; break; case WT_PAGE_COL_VAR: recno = 0; @@ -614,7 +636,7 @@ __verify_row_leaf_key_order( * If a tree is empty (just created), it won't have keys; if there * are no keys, we're done. */ - if (page->pg_row_entries == 0) + if (page->entries == 0) return (0); /* @@ -624,7 +646,7 @@ __verify_row_leaf_key_order( */ if (vs->max_addr->size != 0) { WT_RET(__wt_row_leaf_key_copy( - session, page, page->pg_row_d, vs->tmp1)); + session, page, page->pg_row, vs->tmp1)); /* * Compare the key against the largest key we've seen so far. @@ -653,7 +675,7 @@ __verify_row_leaf_key_order( /* Update the largest key we've seen to the last key on this page. */ WT_RET(__wt_row_leaf_key_copy(session, page, - page->pg_row_d + (page->pg_row_entries - 1), vs->max_key)); + page->pg_row + (page->entries - 1), vs->max_key)); (void)__wt_page_addr_string(session, ref, vs->max_addr); return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 3a6fd8261ba..a4071c44aee 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -203,7 +203,8 @@ __verify_dsk_row( WT_ITEM *last; enum { FIRST, WAS_KEY, WAS_VALUE } last_cell_type; void *huffman; - uint32_t cell_num, cell_type, i, key_cnt, prefix; + size_t prefix; + uint32_t cell_num, cell_type, i, key_cnt; uint8_t *end; int cmp; @@ -343,8 +344,9 @@ __verify_dsk_row( if (cell_num > 1 && prefix > last->size) WT_ERR_VRFY(session, "key %" PRIu32 " on page at %s has a prefix " - "compression count of %" PRIu32 ", larger than " - "the length of the previous key, %" WT_SIZET_FMT, + "compression count of %" WT_SIZET_FMT + ", larger than the length of the previous key, %" + WT_SIZET_FMT, cell_num, tag, prefix, last->size); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index fb0d2296823..86484feb7c9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -17,54 +17,60 @@ __ref_index_slot(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp, uint32_t *slotp) { WT_PAGE_INDEX *pindex; - uint32_t i; + WT_REF **start, **stop, **p, **t; + uint32_t entries, slot; - /* - * Copy the parent page's index value: the page can split at any time, - * but the index's value is always valid, even if it's not up-to-date. - */ -retry: WT_INTL_INDEX_GET(session, ref->home, pindex); + for (;;) { + /* + * Copy the parent page's index value: the page can split at + * any time, but the index's value is always valid, even if + * it's not up-to-date. + */ + WT_INTL_INDEX_GET(session, ref->home, pindex); + entries = pindex->entries; - /* - * Use the page's reference hint: it should be correct unless the page - * split before our slot. If the page splits after our slot, the hint - * will point earlier in the array than our actual slot, so the first - * loop is from the hint to the end of the list, and the second loop - * is from the start of the list to the end of the list. (The second - * loop overlaps the first, but that only happen in cases where we've - * split the tree and aren't going to find our slot at all, that's not - * worth optimizing.) - * - * It's not an error for the reference hint to be wrong, it just means - * the first retrieval (which sets the hint for subsequent retrievals), - * is slower. - */ - i = ref->pindex_hint; - if (i < pindex->entries && pindex->index[i] == ref) { - *pindexp = pindex; - *slotp = i; - return; - } - while (++i < pindex->entries) - if (pindex->index[i] == ref) { - *pindexp = pindex; - *slotp = ref->pindex_hint = i; - return; - } - for (i = 0; i < pindex->entries; ++i) - if (pindex->index[i] == ref) { - *pindexp = pindex; - *slotp = ref->pindex_hint = i; - return; + /* + * Use the page's reference hint: it should be correct unless + * there was a split or delete in the parent before our slot. + * If the hint is wrong, it can be either too big or too small, + * but often only by a small amount. Search up and down the + * index starting from the hint. + * + * It's not an error for the reference hint to be wrong, it + * just means the first retrieval (which sets the hint for + * subsequent retrievals), is slower. + */ + slot = ref->pindex_hint; + if (slot >= entries) + slot = entries - 1; + if (pindex->index[slot] == ref) + goto found; + for (start = &pindex->index[0], + stop = &pindex->index[entries - 1], + p = t = &pindex->index[slot]; + p > start || t < stop;) { + if (p > start && *--p == ref) { + slot = (uint32_t)(p - start); + goto found; + } + if (t < stop && *++t == ref) { + slot = (uint32_t)(t - start); + goto found; + } } - /* - * If we don't find our reference, the page split and our home pointer - * references the wrong page. When internal pages split, their WT_REF - * structure home values are updated; yield and wait for that to happen. - */ - __wt_yield(); - goto retry; + /* + * If we don't find our reference, the page split and our home + * pointer references the wrong page. When internal pages + * split, their WT_REF structure home values are updated; yield + * and wait for that to happen. + */ + __wt_yield(); + } + +found: WT_ASSERT(session, pindex->index[slot] == ref); + *pindexp = pindex; + *slotp = slot; } /* @@ -334,9 +340,7 @@ __tree_walk_internal(WT_SESSION_IMPL *session, * Take a copy of any held page and clear the return value. Remember * the hazard pointer we're currently holding. * - * We may be passed a pointer to btree->evict_page that we are clearing - * here. We check when discarding pages that we're not discarding that - * page, so this clear must be done before the page is released. + * Clear the returned value, it makes future error handling easier. */ couple = couple_orig = ref = *refp; *refp = NULL; @@ -344,16 +348,19 @@ __tree_walk_internal(WT_SESSION_IMPL *session, /* If no page is active, begin a walk from the start/end of the tree. */ if (ref == NULL) { restart: /* - * We can reach here with a NULL or root reference; the release + * We can be here with a NULL or root WT_REF; the page release * function handles them internally, don't complicate this code * by calling them out. */ WT_ERR(__wt_page_release(session, couple, flags)); - couple = couple_orig = ref = &btree->root; - if (ref->page == NULL) - goto done; + /* + * We're not supposed to walk trees without root pages. As this + * has not always been the case, assert to debug that change. + */ + WT_ASSERT(session, btree->root.page != NULL); + couple = couple_orig = ref = &btree->root; initial_descent = true; goto descend; } @@ -431,8 +438,8 @@ restart: /* /* * Move to the next slot, and set the reference hint if * it's wrong (used when we continue the walk). We don't - * update those hints when splitting, so it's common for - * them to be incorrect in some workloads. + * always update the hints when splitting, it's expected + * for them to be incorrect in some workloads. */ ref = pindex->index[slot]; if (ref->pindex_hint != slot) diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index a7920da5267..8b758453288 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -17,7 +17,8 @@ static int __col_insert_alloc( */ int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) + uint64_t recno, WT_ITEM *value, + WT_UPDATE *upd_arg, bool is_remove, bool exclusive) { WT_BTREE *btree; WT_DECL_RET; @@ -103,7 +104,7 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Serialize the update. */ WT_ERR(__wt_update_serial( - session, page, &cbt->ins->upd, &upd, upd_size)); + session, page, &cbt->ins->upd, &upd, upd_size, false)); } else { /* Allocate the append/update list reference as necessary. */ if (append) { @@ -115,9 +116,8 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, page, mod->mod_col_update, ins_headp, 1); ins_headp = &mod->mod_col_update[0]; } else { - WT_PAGE_ALLOC_AND_SWAP(session, - page, mod->mod_col_update, ins_headp, - page->pg_var_entries); + WT_PAGE_ALLOC_AND_SWAP(session, page, + mod->mod_col_update, ins_headp, page->entries); ins_headp = &mod->mod_col_update[cbt->slot]; } @@ -186,11 +186,11 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, if (append) WT_ERR(__wt_col_append_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, &cbt->recno, skipdepth)); + &ins, ins_size, &cbt->recno, skipdepth, exclusive)); else WT_ERR(__wt_insert_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, skipdepth)); + &ins, ins_size, skipdepth, exclusive)); } /* If the update was successful, add it to the in-memory log. */ diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c index 64ee9e94f4c..c72d66f8796 100644 --- a/src/third_party/wiredtiger/src/btree/col_srch.c +++ b/src/third_party/wiredtiger/src/btree/col_srch.c @@ -240,8 +240,8 @@ leaf_only: cbt->compare = 1; return (0); } - if (recno >= current->ref_recno + page->pg_fix_entries) { - cbt->recno = current->ref_recno + page->pg_fix_entries; + if (recno >= current->ref_recno + page->entries) { + cbt->recno = current->ref_recno + page->entries; goto past_end; } else { cbt->recno = recno; @@ -257,8 +257,7 @@ leaf_only: } if ((cip = __col_var_search(current, recno, NULL)) == NULL) { cbt->recno = __col_var_last_recno(current); - cbt->slot = page->pg_var_entries == 0 ? - 0 : page->pg_var_entries - 1; + cbt->slot = page->entries == 0 ? 0 : page->entries - 1; goto past_end; } else { cbt->recno = recno; diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index 99ee34a6c5d..032fdf7d897 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -26,7 +26,7 @@ __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) btree = S2BT(session); - if (page->pg_row_entries == 0) { /* Just checking... */ + if (page->entries == 0) { /* Just checking... */ F_SET_ATOMIC(page, WT_PAGE_BUILD_KEYS); return (0); } @@ -51,15 +51,15 @@ __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) */ WT_RET(__wt_scr_alloc(session, 0, &key)); WT_RET(__wt_scr_alloc(session, - (uint32_t)__bitstr_size(page->pg_row_entries), &tmp)); + (uint32_t)__bitstr_size(page->entries), &tmp)); memset(tmp->mem, 0, tmp->memsize); if ((gap = btree->key_gap) == 0) gap = 1; - __inmem_row_leaf_slots(tmp->mem, 0, page->pg_row_entries, gap); + __inmem_row_leaf_slots(tmp->mem, 0, page->entries, gap); /* Instantiate the keys. */ - for (rip = page->pg_row_d, i = 0; i < page->pg_row_entries; ++rip, ++i) + for (rip = page->pg_row, i = 0; i < page->entries; ++rip, ++i) if (__bit_test(tmp->mem, i)) WT_ERR(__wt_row_leaf_key_work( session, page, rip, key, true)); @@ -282,7 +282,7 @@ switch_and_jump: /* Switching to a forward roll. */ * the tracking cache. */ if (slot_offset == 0) { - __wt_readlock(session, btree->ovfl_lock); + __wt_readlock(session, &btree->ovfl_lock); copy = WT_ROW_KEY_COPY(rip); if (!__wt_row_leaf_key_info(page, copy, NULL, &cell, &keyb->data, &keyb->size)) { @@ -290,7 +290,7 @@ switch_and_jump: /* Switching to a forward roll. */ ret = __wt_dsk_cell_data_ref(session, WT_PAGE_ROW_LEAF, unpack, keyb); } - __wt_readunlock(session, btree->ovfl_lock); + __wt_readunlock(session, &btree->ovfl_lock); WT_ERR(ret); break; } diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index a1c214e5b8b..0dd562ed1aa 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -15,18 +15,13 @@ int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_CONNECTION_IMPL *conn; + WT_DECL_RET; WT_PAGE_MODIFY *modify; - conn = S2C(session); - WT_RET(__wt_calloc_one(session, &modify)); - /* - * Select a spinlock for the page; let the barrier immediately below - * keep things from racing too badly. - */ - modify->page_lock = ++conn->page_lock_cnt % WT_PAGE_LOCKS; + /* Initialize the spinlock for the page. */ + WT_ERR(__wt_spin_init(session, &modify->page_lock, "btree page")); /* * Multiple threads of control may be searching and deciding to modify @@ -37,8 +32,8 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) if (__wt_atomic_cas_ptr(&page->modify, NULL, modify)) __wt_cache_page_inmem_incr(session, page, sizeof(*modify)); else - __wt_free(session, modify); - return (0); +err: __wt_free(session, modify); + return (ret); } /* @@ -47,7 +42,8 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) */ int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) + WT_ITEM *key, WT_ITEM *value, + WT_UPDATE *upd_arg, bool is_remove, bool exclusive) { WT_DECL_RET; WT_INSERT *ins; @@ -85,9 +81,8 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, if (cbt->compare == 0) { if (cbt->ins == NULL) { /* Allocate an update array as necessary. */ - WT_PAGE_ALLOC_AND_SWAP(session, - page, mod->mod_row_update, - upd_entry, page->pg_row_entries); + WT_PAGE_ALLOC_AND_SWAP(session, page, + mod->mod_row_update, upd_entry, page->entries); /* Set the WT_UPDATE array reference. */ upd_entry = &mod->mod_row_update[cbt->slot]; @@ -133,7 +128,7 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Serialize the update. */ WT_ERR(__wt_update_serial( - session, page, upd_entry, &upd, upd_size)); + session, page, upd_entry, &upd, upd_size, exclusive)); } else { /* * Allocate the insert array as necessary. @@ -147,10 +142,10 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, * slot. That's hard, so we set a flag. */ WT_PAGE_ALLOC_AND_SWAP(session, page, - mod->mod_row_insert, ins_headp, page->pg_row_entries + 1); + mod->mod_row_insert, ins_headp, page->entries + 1); ins_slot = F_ISSET(cbt, WT_CBT_SEARCH_SMALLEST) ? - page->pg_row_entries: cbt->slot; + page->entries: cbt->slot; ins_headp = &mod->mod_row_insert[ins_slot]; /* Allocate the WT_INSERT_HEAD structure as necessary. */ @@ -208,7 +203,7 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Insert the WT_INSERT structure. */ WT_ERR(__wt_insert_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, skipdepth)); + &ins, ins_size, skipdepth, exclusive)); } if (logged) diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index d4e82c458d4..9c3d467340e 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -486,14 +486,14 @@ leaf_only: if (insert && descend_right) { cbt->append_tree = 1; - if (page->pg_row_entries == 0) { - cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); + if (page->entries == 0) { + cbt->slot = WT_ROW_SLOT(page, page->pg_row); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->slot = WT_ROW_SLOT(page, - page->pg_row_d + (page->pg_row_entries - 1)); + page->pg_row + (page->entries - 1)); ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } @@ -511,11 +511,11 @@ leaf_only: * doing the tests and error handling inside the loop costs about 5%. */ base = 0; - limit = page->pg_row_entries; + limit = page->entries; if (collator == NULL && srch_key->size <= WT_COMPARE_SHORT_MAXLEN) for (; limit != 0; limit >>= 1) { indx = base + (limit >> 1); - rip = page->pg_row_d + indx; + rip = page->pg_row + indx; WT_ERR( __wt_row_leaf_key(session, page, rip, item, true)); @@ -529,7 +529,7 @@ leaf_only: else if (collator == NULL) for (; limit != 0; limit >>= 1) { indx = base + (limit >> 1); - rip = page->pg_row_d + indx; + rip = page->pg_row + indx; WT_ERR( __wt_row_leaf_key(session, page, rip, item, true)); @@ -547,7 +547,7 @@ leaf_only: else for (; limit != 0; limit >>= 1) { indx = base + (limit >> 1); - rip = page->pg_row_d + indx; + rip = page->pg_row + indx; WT_ERR( __wt_row_leaf_key(session, page, rip, item, true)); @@ -591,13 +591,13 @@ leaf_match: cbt->compare = 0; */ if (base == 0) { cbt->compare = 1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); + cbt->slot = WT_ROW_SLOT(page, page->pg_row); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->compare = -1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row_d + (base - 1)); + cbt->slot = WT_ROW_SLOT(page, page->pg_row + (base - 1)); ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } @@ -623,215 +623,3 @@ leaf_match: cbt->compare = 0; err: WT_TRET(__wt_page_release(session, current, 0)); return (ret); } - -/* - * __wt_row_random_leaf -- - * Return a random key from a row-store leaf page. - */ -int -__wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) -{ - WT_INSERT *ins, **start, **stop; - WT_INSERT_HEAD *ins_head; - WT_PAGE *page; - uint64_t samples; - uint32_t choice, entries, i; - int level; - - page = cbt->ref->page; - start = stop = NULL; /* [-Wconditional-uninitialized] */ - entries = 0; /* [-Wconditional-uninitialized] */ - - __cursor_pos_clear(cbt); - - /* If the page has disk-based entries, select from them. */ - if (page->pg_row_entries != 0) { - cbt->compare = 0; - cbt->slot = __wt_random(&session->rnd) % page->pg_row_entries; - - /* - * The real row-store search function builds the key, so we - * have to as well. - */ - return (__wt_row_leaf_key(session, - page, page->pg_row_d + cbt->slot, cbt->tmp, false)); - } - - /* - * If the tree is new (and not empty), it might have a large insert - * list. - * - * Walk down the list until we find a level with at least 50 entries, - * that's where we'll start rolling random numbers. The value 50 is - * used to ignore levels with only a few entries, that is, levels which - * are potentially badly skewed. - */ - F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - if ((ins_head = WT_ROW_INSERT_SMALLEST(page)) == NULL) - return (WT_NOTFOUND); - for (level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { - start = &ins_head->head[level]; - for (entries = 0, stop = start; - *stop != NULL; stop = &(*stop)->next[level]) - ++entries; - - if (entries > 50) - break; - } - - /* - * If it's a tiny list and we went all the way to level 0, correct the - * level; entries is correctly set. - */ - if (level < 0) - level = 0; - - /* - * Step down the skip list levels, selecting a random chunk of the name - * space at each level. - */ - for (samples = entries; level > 0; samples += entries) { - /* - * There are (entries) or (entries + 1) chunks of the name space - * considered at each level. They are: between start and the 1st - * element, between the 1st and 2nd elements, and so on to the - * last chunk which is the name space after the stop element on - * the current level. This last chunk of name space may or may - * not be there: as we descend the levels of the skip list, this - * chunk may appear, depending if the next level down has - * entries logically after the stop point in the current level. - * We can't ignore those entries: because of the algorithm used - * to determine the depth of a skiplist, there may be a large - * number of entries "revealed" by descending a level. - * - * If the next level down has more items after the current stop - * point, there are (entries + 1) chunks to consider, else there - * are (entries) chunks. - */ - if (*(stop - 1) == NULL) - choice = __wt_random(&session->rnd) % entries; - else - choice = __wt_random(&session->rnd) % (entries + 1); - - if (choice == entries) { - /* - * We selected the name space after the stop element on - * this level. Set the start point to the current stop - * point, descend a level and move the stop element to - * the end of the list, that is, the end of the newly - * discovered name space, counting entries as we go. - */ - start = stop; - --start; - --level; - for (entries = 0, stop = start; - *stop != NULL; stop = &(*stop)->next[level]) - ++entries; - } else { - /* - * We selected another name space on the level. Move the - * start pointer the selected number of entries forward - * to the start of the selected chunk (if the selected - * number is 0, start won't move). Set the stop pointer - * to the next element in the list and drop both start - * and stop down a level. - */ - for (i = 0; i < choice; ++i) - start = &(*start)->next[level]; - stop = &(*start)->next[level]; - - --start; - --stop; - --level; - - /* Count the entries in the selected name space. */ - for (entries = 0, - ins = *start; ins != *stop; ins = ins->next[level]) - ++entries; - } - } - - /* - * When we reach the bottom level, entries will already be set. Select - * a random entry from the name space and return it. - * - * It should be impossible for the entries count to be 0 at this point, - * but check for it out of paranoia and to quiet static testing tools. - */ - if (entries > 0) - entries = __wt_random(&session->rnd) % entries; - for (ins = *start; entries > 0; --entries) - ins = ins->next[0]; - - cbt->ins = ins; - cbt->ins_head = ins_head; - cbt->compare = 0; - - /* - * Random lookups in newly created collections can be slow if a page - * consists of a large skiplist. Schedule the page for eviction if we - * encounter a large skiplist. This worthwhile because applications - * that take a sample often take many samples, so the overhead of - * traversing the skip list each time accumulates to real time. - */ - if (samples > 5000) - __wt_page_evict_soon(session, cbt->ref); - - return (0); -} - -/* - * __wt_row_random_descent -- - * Find a random leaf page in a row-store tree. - */ -int -__wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) -{ - WT_BTREE *btree; - WT_DECL_RET; - WT_PAGE *page; - WT_PAGE_INDEX *pindex; - WT_REF *current, *descent; - - btree = S2BT(session); - current = NULL; - - if (0) { -restart: /* - * Discard the currently held page and restart the search from - * the root. - */ - WT_RET(__wt_page_release(session, current, 0)); - } - - /* Search the internal pages of the tree. */ - current = &btree->root; - for (;;) { - page = current->page; - if (page->type != WT_PAGE_ROW_INT) - break; - - WT_INTL_INDEX_GET(session, page, pindex); - descent = pindex->index[ - __wt_random(&session->rnd) % pindex->entries]; - - /* - * Swap the current page for the child page. If the page splits - * while we're retrieving it, restart the search at the root. - * - * On other error, simply return, the swap call ensures we're - * holding nothing on failure. - */ - if ((ret = __wt_page_swap( - session, current, descent, WT_READ_RESTART_OK)) == 0) { - current = descent; - continue; - } - if (ret == WT_RESTART) - goto restart; - return (ret); - } - - cbt->ref = current; - return (0); -} diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 41f50957809..41da4225f3a 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -63,6 +63,7 @@ int __wt_las_create(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; + WT_DECL_RET; uint32_t session_flags; const char *drop_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_drop), "force=true", NULL }; @@ -80,7 +81,9 @@ __wt_las_create(WT_SESSION_IMPL *session) * * Discard any previous incarnation of the table. */ - WT_RET(__wt_session_drop(session, WT_LAS_URI, drop_cfg)); + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_drop(session, WT_LAS_URI, drop_cfg)); + WT_RET(ret); /* Re-create the table. */ WT_RET(__wt_session_create(session, WT_LAS_URI, WT_LAS_FORMAT)); diff --git a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c index ddfa2bdaeb8..a9be9ced1c6 100644 --- a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c +++ b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c @@ -1,4 +1,6 @@ #if defined(__powerpc64__) +#include "wt_internal.h" + #define CRC_TABLE #include "crc32_constants.h" @@ -68,8 +70,6 @@ out: } #endif -#include "wt_internal.h" - /* * __wt_checksum_hw -- * WiredTiger: return a checksum for a chunk of memory. diff --git a/src/third_party/wiredtiger/src/checksum/software/checksum.c b/src/third_party/wiredtiger/src/checksum/software/checksum.c index 30362584a3e..65ed74bbe06 100644 --- a/src/third_party/wiredtiger/src/checksum/software/checksum.c +++ b/src/third_party/wiredtiger/src/checksum/software/checksum.c @@ -1101,6 +1101,7 @@ static const uint32_t g_crc_slicing[8][256] = { */ uint32_t __wt_checksum_sw(const void *chunk, size_t len) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { uint32_t crc, next; size_t nqwords; diff --git a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c index f77d6768d42..28b46594220 100644 --- a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c +++ b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c @@ -6,8 +6,20 @@ * Author(s): Hendrik Brueckner * */ +#include "wt_internal.h" + #include #include + +#if defined(HAVE_CRC32_HARDWARE) + +#include + +/* RHEL 7 has kernel support, but does not define this constant in the lib c headers. */ +#ifndef HWCAP_S390_VX +#define HWCAP_S390_VX 2048 +#endif + #include "crc32-s390x.h" #include "slicing-consts.h" @@ -69,8 +81,6 @@ unsigned int __wt_crc32c_le(unsigned int crc, const unsigned char *buf, size_t l /* Main CRC-32 functions */ DEFINE_CRC32_VX(__wt_crc32c_le_vx, __wt_crc32c_le_vgfm_16, __wt_crc32c_le) -#include "wt_internal.h" - /* * __wt_checksum_hw -- * WiredTiger: return a checksum for a chunk of memory. @@ -81,6 +91,8 @@ __wt_checksum_hw(const void *chunk, size_t len) return (~__wt_crc32c_le_vx(0xffffffff, chunk, len)); } +#endif + /* * __wt_checksum_init -- * WiredTiger: detect CRC hardware and set the checksum function. @@ -89,8 +101,14 @@ void __wt_checksum_init(void) { #if defined(HAVE_CRC32_HARDWARE) - __wt_process.checksum = __wt_checksum_hw; -#else + unsigned long caps = getauxval(AT_HWCAP); + + if (caps & HWCAP_S390_VX) + __wt_process.checksum = __wt_checksum_hw; + else + __wt_process.checksum = __wt_checksum_sw; + +#else /* !HAVE_CRC32_HARDWARE */ __wt_process.checksum = __wt_checksum_sw; #endif } diff --git a/src/third_party/wiredtiger/src/config/config.c b/src/third_party/wiredtiger/src/config/config.c index 3416153d160..a47dfe76aec 100644 --- a/src/third_party/wiredtiger/src/config/config.c +++ b/src/third_party/wiredtiger/src/config/config.c @@ -711,7 +711,7 @@ __wt_config_getones(WT_SESSION_IMPL *session, /* * __wt_config_getones_none -- * Get the value for a given string key from a single config string. - * Treat "none" as empty. + * Treat "none" as empty. */ int __wt_config_getones_none(WT_SESSION_IMPL *session, diff --git a/src/third_party/wiredtiger/src/config/config_api.c b/src/third_party/wiredtiger/src/config/config_api.c index 76a51903588..c1299baaafe 100644 --- a/src/third_party/wiredtiger/src/config/config_api.c +++ b/src/third_party/wiredtiger/src/config/config_api.c @@ -158,24 +158,25 @@ wiredtiger_config_validate(WT_SESSION *wt_session, } /* - * __conn_foc_add -- + * __wt_conn_foc_add -- * Add a new entry into the connection's free-on-close list. */ -static int -__conn_foc_add(WT_SESSION_IMPL *session, const void *p) +void +__wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p) { WT_CONNECTION_IMPL *conn; conn = S2C(session); /* - * Our caller is expected to be holding any locks we need. + * Callers of this function are expected to be holding the connection's + * api_lock. + * + * All callers of this function currently ignore errors. */ - WT_RET(__wt_realloc_def( - session, &conn->foc_size, conn->foc_cnt + 1, &conn->foc)); - - conn->foc[conn->foc_cnt++] = (void *)p; - return (0); + if (__wt_realloc_def( + session, &conn->foc_size, conn->foc_cnt + 1, &conn->foc) == 0) + conn->foc[conn->foc_cnt++] = (void *)p; } /* @@ -214,7 +215,7 @@ __wt_configure_method(WT_SESSION_IMPL *session, WT_CONFIG_ENTRY *entry; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - size_t cnt; + size_t cnt, len; char *newcheck_name, *p; /* @@ -275,12 +276,10 @@ __wt_configure_method(WT_SESSION_IMPL *session, */ WT_ERR(__wt_calloc_one(session, &entry)); entry->method = (*epp)->method; - WT_ERR(__wt_calloc_def(session, - strlen((*epp)->base) + strlen(",") + strlen(config) + 1, &p)); - (void)strcpy(p, (*epp)->base); - (void)strcat(p, ","); - (void)strcat(p, config); + len = strlen((*epp)->base) + strlen(",") + strlen(config) + 1; + WT_ERR(__wt_calloc_def(session, len, &p)); entry->base = p; + WT_ERR(__wt_snprintf(p, len, "%s,%s", (*epp)->base, config)); /* * There may be a default value in the config argument passed in (for @@ -328,12 +327,12 @@ __wt_configure_method(WT_SESSION_IMPL *session, * order to avoid freeing chunks of memory twice. Again, this isn't a * commonly used API and it shouldn't ever happen, just leak it. */ - (void)__conn_foc_add(session, entry->base); - (void)__conn_foc_add(session, entry); - (void)__conn_foc_add(session, checks); - (void)__conn_foc_add(session, newcheck->type); - (void)__conn_foc_add(session, newcheck->checks); - (void)__conn_foc_add(session, newcheck_name); + __wt_conn_foc_add(session, entry->base); + __wt_conn_foc_add(session, entry); + __wt_conn_foc_add(session, checks); + __wt_conn_foc_add(session, newcheck->type); + __wt_conn_foc_add(session, newcheck->checks); + __wt_conn_foc_add(session, newcheck_name); /* * Instead of using locks to protect configuration information, assume diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c index 7fe78d06ba7..5abe7556a03 100644 --- a/src/third_party/wiredtiger/src/config/config_collapse.c +++ b/src/third_party/wiredtiger/src/config/config_collapse.c @@ -372,6 +372,7 @@ __config_merge_cmp(const void *a, const void *b) int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_CONFIG_MERGE merge; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index d57bc418c93..f152fbacad4 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -24,6 +24,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_load_extension[] = { }; static const WT_CONFIG_CHECK confchk_WT_CONNECTION_open_session[] = { + { "ignore_cache_size", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," "\"snapshot\"]", @@ -146,11 +147,12 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -161,6 +163,14 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, + { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction[] = { { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," @@ -223,6 +233,9 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -281,7 +294,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { { "source", "string", NULL, NULL, NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "type", "string", NULL, NULL, NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, @@ -344,6 +357,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { }; static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = { + { "ignore_cache_size", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," "\"snapshot\"]", @@ -369,6 +383,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_snapshot[] = { { "drop", "category", NULL, NULL, confchk_WT_SESSION_snapshot_drop_subconfigs, 4 }, + { "include_updates", "boolean", NULL, NULL, NULL, 0 }, { "name", "string", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -398,6 +413,9 @@ static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { }; static const WT_CONFIG_CHECK confchk_file_config[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -448,7 +466,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, @@ -456,6 +474,9 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { }; static const WT_CONFIG_CHECK confchk_file_meta[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -509,7 +530,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, @@ -534,6 +555,9 @@ static const WT_CONFIG_CHECK confchk_index_meta[] = { }; static const WT_CONFIG_CHECK confchk_lsm_meta[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -590,7 +614,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, @@ -657,6 +681,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -725,11 +750,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -742,6 +768,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -810,11 +837,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -828,6 +856,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -890,11 +919,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -908,6 +938,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -970,11 +1001,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -1017,13 +1049,13 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_CONNECTION_load_extension, 4 }, { "WT_CONNECTION.open_session", - "isolation=read-committed", - confchk_WT_CONNECTION_open_session, 1 + "ignore_cache_size=false,isolation=read-committed", + confchk_WT_CONNECTION_open_session, 2 }, { "WT_CONNECTION.reconfigure", "async=(enabled=false,ops_max=1024,threads=2),cache_overhead=8," "cache_size=100MB,checkpoint=(log_size=0,wait=0),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," + "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1047,6 +1079,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "append=false,overwrite=true", confchk_WT_CURSOR_reconfigure, 2 }, + { "WT_SESSION.alter", + "access_pattern_hint=none,cache_resident=false", + confchk_WT_SESSION_alter, 2 + }, { "WT_SESSION.begin_transaction", "isolation=,name=,priority=0,snapshot=,sync=", confchk_WT_SESSION_begin_transaction, 5 @@ -1068,12 +1104,13 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_SESSION_compact, 1 }, { "WT_SESSION.create", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checksum=uncompressed," - "colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=," - "name=),exclusive=false,extractor=,format=btree,huffman_key=," - "huffman_value=,ignore_in_memory_cache_size=false,immutable=false" - ",internal_item_max=0,internal_key_max=0," + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,colgroups=,collator=,columns=,dictionary=0" + ",encryption=(keyid=,name=),exclusive=false,extractor=," + "format=btree,huffman_key=,huffman_value=," + "ignore_in_memory_cache_size=false,immutable=false," + "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," @@ -1082,8 +1119,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," "prefix_compression_min=4,source=,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=75,type=file,value_format=u", - confchk_WT_SESSION_create, 41 + "split_deepen_per_child=0,split_pct=90,type=file,value_format=u", + confchk_WT_SESSION_create, 42 }, { "WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," @@ -1116,8 +1153,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL, 0 }, { "WT_SESSION.reconfigure", - "isolation=read-committed", - confchk_WT_SESSION_reconfigure, 1 + "ignore_cache_size=false,isolation=read-committed", + confchk_WT_SESSION_reconfigure, 2 }, { "WT_SESSION.rename", "", @@ -1136,8 +1173,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_SESSION_salvage, 1 }, { "WT_SESSION.snapshot", - "drop=(all=false,before=,names=,to=),name=", - confchk_WT_SESSION_snapshot, 2 + "drop=(all=false,before=,names=,to=),include_updates=false,name=", + confchk_WT_SESSION_snapshot, 3 }, { "WT_SESSION.strerror", "", @@ -1165,35 +1202,35 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_colgroup_meta, 5 }, { "file.config", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checksum=uncompressed," - "collator=,columns=,dictionary=0,encryption=(keyid=,name=)," - "format=btree,huffman_key=,huffman_value=," - "ignore_in_memory_cache_size=false,internal_item_max=0," - "internal_key_max=0,internal_key_truncate=true," - "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," - "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," - "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0," - "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" - ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75," - "value_format=u", - confchk_file_config, 34 - }, - { "file.meta", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checkpoint=," - "checkpoint_lsn=,checksum=uncompressed,collator=,columns=," - "dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key=" - ",huffman_value=,id=,ignore_in_memory_cache_size=false," + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_key=," + "huffman_value=,ignore_in_memory_cache_size=false," "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=75,value_format=u," - "version=(major=0,minor=0)", - confchk_file_meta, 38 + "split_deepen_per_child=0,split_pct=90,value_format=u", + confchk_file_config, 35 + }, + { "file.meta", + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checkpoint=,checkpoint_lsn=,checksum=uncompressed,collator=," + "columns=,dictionary=0,encryption=(keyid=,name=),format=btree," + "huffman_key=,huffman_value=,id=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0," + "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" + ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u,version=(major=0,minor=0)", + confchk_file_meta, 39 }, { "index.meta", "app_metadata=,collator=,columns=,extractor=,immutable=false," @@ -1201,23 +1238,23 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_index_meta, 10 }, { "lsm.meta", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checksum=uncompressed," - "chunks=,collator=,columns=,dictionary=0,encryption=(keyid=," - "name=),format=btree,huffman_key=,huffman_value=," - "ignore_in_memory_cache_size=false,internal_item_max=0," - "internal_key_max=0,internal_key_truncate=true," - "internal_page_max=4KB,key_format=u,key_gap=10,last=," - "leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," - "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8," - "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB," - "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB," - "old_chunks=,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75," - "value_format=u", - confchk_lsm_meta, 38 + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,chunks=,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_key=," + "huffman_value=,ignore_in_memory_cache_size=false," + "internal_item_max=0,internal_key_max=0," + "internal_key_truncate=true,internal_page_max=4KB,key_format=u," + "key_gap=10,last=,leaf_item_max=0,leaf_key_max=0," + "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=true)," + "lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16," + "bloom_config=,bloom_hash_count=8,bloom_oldest=false," + "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_max=15," + "merge_min=0),memory_page_max=5MB,old_chunks=," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u", + confchk_lsm_meta, 39 }, { "table.meta", "app_metadata=,colgroups=,collator=,columns=,key_format=u," @@ -1226,62 +1263,64 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "wiredtiger_open", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 39 + confchk_wiredtiger_open, 40 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 40 + confchk_wiredtiger_open_all, 41 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1289,20 +1328,20 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 34 + confchk_wiredtiger_open_basecfg, 35 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1310,7 +1349,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 33 + confchk_wiredtiger_open_usercfg, 34 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/config/config_ext.c b/src/third_party/wiredtiger/src/config/config_ext.c index 56c0018f8c3..88f1390843a 100644 --- a/src/third_party/wiredtiger/src/config/config_ext.c +++ b/src/third_party/wiredtiger/src/config/config_ext.c @@ -8,23 +8,10 @@ #include "wt_internal.h" -/* - * __wt_ext_config_parser_open -- - * WT_EXTENSION_API->config_parser_open implementation - */ -int -__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) -{ - WT_UNUSED(wt_ext); - return (wiredtiger_config_parser_open( - wt_session, config, len, config_parserp)); -} - /* * __wt_ext_config_get -- * Given a NULL-terminated list of configuration strings, find the final - * value for a given string key (external API version). + * value for a given string key (external API version). */ int __wt_ext_config_get(WT_EXTENSION_API *wt_api, @@ -43,3 +30,65 @@ __wt_ext_config_get(WT_EXTENSION_API *wt_api, return (WT_NOTFOUND); return (__wt_config_gets(session, cfg, key, cval)); } + +/* + * __wt_ext_config_get_string -- + * Given a configuration string, find the value for a given string key + * (external API version). + */ +int +__wt_ext_config_get_string(WT_EXTENSION_API *wt_api, + WT_SESSION *wt_session, const char *config, const char *key, + WT_CONFIG_ITEM *cval) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + + conn = (WT_CONNECTION_IMPL *)wt_api->conn; + if ((session = (WT_SESSION_IMPL *)wt_session) == NULL) + session = conn->default_session; + + return (__wt_config_getones(session, config, key, cval)); +} + +/* + * __wt_ext_config_parser_open -- + * WT_EXTENSION_API->config_parser_open implementation + */ +int +__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) +{ + WT_UNUSED(wt_ext); + return (wiredtiger_config_parser_open( + wt_session, config, len, config_parserp)); +} + +/* + * __wt_ext_config_parser_open_arg -- + * WT_EXTENSION_API->config_parser_open_arg implementation + */ +int +__wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, + WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, + WT_CONFIG_PARSER **config_parserp) +{ + const char **cfg, *p; + size_t len; + + WT_UNUSED(wt_ext); + + /* Find the last non-NULL entry in the configuration stack. */ + if ((cfg = (const char **)cfg_arg) == NULL || *cfg == NULL) { + p = NULL; + len = 0; + } else { + while (cfg[1] != NULL) + ++cfg; + p = *cfg; + len = strlen(p); + } + + return (wiredtiger_config_parser_open( + wt_session, p, len, config_parserp)); +} diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index a6e0f57f02e..68d45678965 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -239,8 +239,6 @@ __conn_add_compressor(WT_CONNECTION *wt_conn, WT_NAMED_COMPRESSOR *ncomp; WT_SESSION_IMPL *session; - WT_UNUSED(name); - WT_UNUSED(compressor); ncomp = NULL; conn = (WT_CONNECTION_IMPL *)wt_conn; @@ -756,8 +754,11 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.scr_free = __wt_ext_scr_free; conn->extension_api.collator_config = ext_collator_config; conn->extension_api.collate = ext_collate; - conn->extension_api.config_parser_open = __wt_ext_config_parser_open; conn->extension_api.config_get = __wt_ext_config_get; + conn->extension_api.config_get_string = __wt_ext_config_get_string; + conn->extension_api.config_parser_open = __wt_ext_config_parser_open; + conn->extension_api.config_parser_open_arg = + __wt_ext_config_parser_open_arg; conn->extension_api.metadata_insert = __wt_ext_metadata_insert; conn->extension_api.metadata_remove = __wt_ext_metadata_remove; conn->extension_api.metadata_search = __wt_ext_metadata_search; @@ -789,40 +790,75 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) return (&conn->extension_api); } +/* + * __conn_builtin_init -- + * Initialize and configure a builtin extension. + */ +static int +__conn_builtin_init(WT_CONNECTION_IMPL *conn, const char *name, + int (*extension_init)(WT_CONNECTION *, WT_CONFIG_ARG *), + const char *cfg[]) +{ + WT_CONFIG_ITEM all_configs, cval; + WT_DECL_RET; + WT_SESSION_IMPL *session; + char *config; + const char *ext_cfg[] = { NULL, NULL }; + + session = conn->default_session; + + WT_RET(__wt_config_gets( + session, cfg, "builtin_extension_config", &all_configs)); + WT_CLEAR(cval); + WT_RET_NOTFOUND_OK(__wt_config_subgets( + session, &all_configs, name, &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &config)); + ext_cfg[0] = config; + + ret = extension_init(&conn->iface, (WT_CONFIG_ARG *)ext_cfg); + __wt_free(session, config); + + return (ret); +} + #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif /* - * __conn_load_default_extensions -- + * __conn_builtin_extensions -- * Load extensions that are enabled via --with-builtins */ static int -__conn_load_default_extensions(WT_CONNECTION_IMPL *conn) +__conn_builtin_extensions(WT_CONNECTION_IMPL *conn, const char *cfg[]) { - WT_UNUSED(conn); - #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - WT_RET(lz4_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "lz4", lz4_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - WT_RET(snappy_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "snappy", snappy_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - WT_RET(zlib_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zlib", zlib_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - WT_RET(zstd_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zstd", zstd_extension_init, cfg)); #endif + + /* Avoid warnings if no builtin extensions are configured. */ + WT_UNUSED(conn); + WT_UNUSED(cfg); + WT_UNUSED(__conn_builtin_init); + return (0); } @@ -839,10 +875,11 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_DLH *dlh; int (*load)(WT_CONNECTION *, WT_CONFIG_ARG *); bool is_local; - const char *init_name, *terminate_name; + const char *ext_config, *init_name, *terminate_name; + const char *ext_cfg[2]; dlh = NULL; - init_name = terminate_name = NULL; + ext_config = init_name = terminate_name = NULL; is_local = strcmp(path, "local") == 0; /* Ensure that the load matches the phase of startup we are in. */ @@ -872,8 +909,14 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_ERR( __wt_dlsym(session, dlh, terminate_name, false, &dlh->terminate)); + WT_CLEAR(cval); + WT_ERR_NOTFOUND_OK(__wt_config_gets(session, cfg, "config", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &ext_config)); + ext_cfg[0] = ext_config; + ext_cfg[1] = NULL; + /* Call the load function last, it simplifies error handling. */ - WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)cfg)); + WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)ext_cfg)); /* Link onto the environment's list of open libraries. */ __wt_spin_lock(session, &S2C(session)->api_lock); @@ -883,6 +926,7 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, err: if (dlh != NULL) WT_TRET(__wt_dlclose(session, dlh)); + __wt_free(session, ext_config); __wt_free(session, init_name); __wt_free(session, terminate_name); return (ret); @@ -1618,8 +1662,8 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR_MSG(session, EINVAL, "Creating a new database is incompatible with " "read-only configuration"); - len = (size_t)snprintf(buf, sizeof(buf), - "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING); + WT_ERR(__wt_snprintf_len_set(buf, sizeof(buf), &len, + "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING)); WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); WT_ERR(__wt_fsync(session, fh, true)); } else { @@ -1754,6 +1798,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "checkpoint", WT_VERB_CHECKPOINT }, { "compact", WT_VERB_COMPACT }, { "evict", WT_VERB_EVICT }, + { "evict_stuck", WT_VERB_EVICT_STUCK }, { "evictserver", WT_VERB_EVICTSERVER }, { "fileops", WT_VERB_FILEOPS }, { "handleops", WT_VERB_HANDLEOPS }, @@ -1767,6 +1812,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "rebalance", WT_VERB_REBALANCE }, { "reconcile", WT_VERB_RECONCILE }, { "recovery", WT_VERB_RECOVERY }, + { "recovery_progress", WT_VERB_RECOVERY_PROGRESS }, { "salvage", WT_VERB_SALVAGE }, { "shared_cache", WT_VERB_SHARED_CACHE }, { "split", WT_VERB_SPLIT }, @@ -1942,6 +1988,16 @@ __conn_set_file_system( CONNECTION_API_CALL(conn, session, set_file_system, config, cfg); WT_UNUSED(cfg); + /* + * You can only configure a file system once, and attempting to do it + * again probably means the extension argument didn't have early-load + * set and we've already configured the default file system. + */ + if (conn->file_system != NULL) + WT_ERR_MSG(session, EPERM, + "filesystem already configured; custom filesystems should " + "enable \"early_load\" configuration"); + conn->file_system = file_system; err: API_END_RET(session, ret); @@ -2130,6 +2186,15 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, if (cval.val) F_SET(conn, WT_CONN_READONLY); + /* Configure error messages so we get them right early. */ + WT_ERR(__wt_config_gets(session, cfg, "error_prefix", &cval)); + if (cval.len != 0) + WT_ERR(__wt_strndup( + session, cval.str, cval.len, &conn->error_prefix)); + + /* Set the database home so extensions have access to it. */ + WT_ERR(__conn_home(session, home, cfg)); + /* * Load early extensions before doing further initialization (one early * extension is to configure a file system). @@ -2153,6 +2218,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR( __conn_chk_file_system(session, F_ISSET(conn, WT_CONN_READONLY))); + /* Make sure no other thread of control already owns this database. */ + WT_ERR(__conn_single(session, cfg)); + /* * Capture the config_base setting file for later use. Again, if the * application doesn't want us to read the base configuration file, @@ -2162,18 +2230,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_config_gets(session, cfg, "config_base", &cval)); config_base_set = cval.val != 0; - /* Configure error messages so we get them right early. */ - WT_ERR(__wt_config_gets(session, cfg, "error_prefix", &cval)); - if (cval.len != 0) - WT_ERR(__wt_strndup( - session, cval.str, cval.len, &conn->error_prefix)); - - /* Get the database home. */ - WT_ERR(__conn_home(session, home, cfg)); - - /* Make sure no other thread of control already owns this database. */ - WT_ERR(__conn_single(session, cfg)); - /* * Build the real configuration stack, in the following order (where * later entries override earlier entries): @@ -2194,10 +2250,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_scr_alloc(session, 0, &i3)); cfg[0] = WT_CONFIG_BASE(session, wiredtiger_open_all); cfg[1] = NULL; - WT_ERR_TEST(snprintf(version, sizeof(version), + WT_ERR(__wt_snprintf(version, sizeof(version), "version=(major=%d,minor=%d)", - WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR) >= - (int)sizeof(version), ENOMEM); + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR)); __conn_config_append(cfg, version); /* Ignore the base_config file if config_base_set is false. */ @@ -2265,9 +2320,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, } WT_ERR(__wt_verbose_config(session, cfg)); - WT_ERR(__wt_config_gets(session, cfg, "hazard_max", &cval)); - conn->hazard_max = (uint32_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "session_max", &cval)); conn->session_size = (uint32_t)cval.val + WT_EXTRA_INTERNAL_SESSIONS; @@ -2355,7 +2407,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * everything else to be in place, and the extensions call back into the * library. */ - WT_ERR(__conn_load_default_extensions(conn)); + WT_ERR(__conn_builtin_extensions(conn, cfg)); WT_ERR(__conn_load_extensions(session, cfg, false)); /* diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index fe5f94ea03d..12279709191 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -143,7 +143,8 @@ __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) if (reconfigure) WT_RET(__wt_thread_group_resize( session, &conn->evict_threads, - conn->evict_threads_min, conn->evict_threads_max, + conn->evict_threads_min, + conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL)); return (0); @@ -186,8 +187,8 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET_MSG(session, EINVAL, "eviction target must be lower than the eviction trigger"); - WT_RET(__wt_cond_auto_alloc(session, "cache eviction server", - false, 10000, WT_MILLION, &cache->evict_cond)); + WT_RET(__wt_cond_auto_alloc(session, + "cache eviction server", 10000, WT_MILLION, &cache->evict_cond)); WT_RET(__wt_spin_init(session, &cache->evict_pass_lock, "evict pass")); WT_RET(__wt_spin_init(session, &cache->evict_queue_lock, "cache eviction queue")); @@ -311,7 +312,7 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) cache->bytes_dirty_intl + cache->bytes_dirty_leaf, cache->pages_dirty_intl + cache->pages_dirty_leaf); - WT_TRET(__wt_cond_auto_destroy(session, &cache->evict_cond)); + __wt_cond_destroy(session, &cache->evict_cond); __wt_spin_destroy(session, &cache->evict_pass_lock); __wt_spin_destroy(session, &cache->evict_queue_lock); __wt_spin_destroy(session, &cache->evict_walk_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index 79c2fc23da5..3f7a770b762 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -32,7 +32,7 @@ */ #define WT_CACHE_POOL_APP_EVICT_MULTIPLIER 3 #define WT_CACHE_POOL_APP_WAIT_MULTIPLIER 6 -#define WT_CACHE_POOL_READ_MULTIPLIER 1 +#define WT_CACHE_POOL_READ_MULTIPLIER 1 static void __cache_pool_adjust( WT_SESSION_IMPL *, uint64_t, uint64_t, bool, bool *); @@ -104,8 +104,8 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) TAILQ_INIT(&cp->cache_pool_qh); WT_ERR(__wt_spin_init( session, &cp->cache_pool_lock, "cache shared pool")); - WT_ERR(__wt_cond_alloc(session, - "cache pool server", false, &cp->cache_pool_cond)); + WT_ERR(__wt_cond_alloc( + session, "cache pool server", &cp->cache_pool_cond)); __wt_process.cache_pool = cp; __wt_verbose(session, @@ -225,7 +225,7 @@ err: __wt_spin_unlock(session, &__wt_process.spinlock); __wt_free(session, pool_name); if (ret != 0 && created) { __wt_free(session, cp->name); - WT_TRET(__wt_cond_destroy(session, &cp->cache_pool_cond)); + __wt_cond_destroy(session, &cp->cache_pool_cond); __wt_free(session, cp); } return (ret); @@ -391,7 +391,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) __wt_free(session, cp->name); __wt_spin_destroy(session, &cp->cache_pool_lock); - WT_TRET(__wt_cond_destroy(session, &cp->cache_pool_cond)); + __wt_cond_destroy(session, &cp->cache_pool_cond); __wt_free(session, cp); } @@ -418,8 +418,9 @@ static void __cache_pool_balance(WT_SESSION_IMPL *session, bool forward) { WT_CACHE_POOL *cp; - bool adjusted; uint64_t bump_threshold, highest; + int i; + bool adjusted; cp = __wt_process.cache_pool; adjusted = false; @@ -438,11 +439,17 @@ __cache_pool_balance(WT_SESSION_IMPL *session, bool forward) /* * Actively attempt to: - * - Reduce the amount allocated, if we are over the budget + * - Reduce the amount allocated, if we are over the budget. * - Increase the amount used if there is capacity and any pressure. + * Don't keep trying indefinitely, if we aren't succeeding in reducing + * the cache in use re-assessing the participants' states is necessary. + * We are also holding a lock across this process, which can slow + * participant shutdown if we spend a long time balancing. */ - while (F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && - F_ISSET(S2C(session)->cache, WT_CACHE_POOL_RUN)) { + for (i = 0; + i < 2 * WT_CACHE_POOL_BUMP_THRESHOLD && + F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && + F_ISSET(S2C(session)->cache, WT_CACHE_POOL_RUN); i++) { __cache_pool_adjust( session, highest, bump_threshold, forward, &adjusted); /* @@ -565,7 +572,7 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *entry; uint64_t adjustment, highest_percentile, pressure, reserved, smallest; u_int pct_full; - bool busy, pool_full, grow; + bool busy, decrease_ok, grow, pool_full; *adjustedp = false; cp = __wt_process.cache_pool; @@ -611,6 +618,34 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, if (cache->cp_skip_count > 0 && --cache->cp_skip_count > 0) continue; + /* + * The bump threshold decreases as we try longer to balance + * the pool. Adjust how aggressively we free space from + * participants depending on how long we have been trying. + */ + decrease_ok = false; + /* + * Any participant is a candidate if we have been trying + * for long enough. + */ + if (bump_threshold == 0) + decrease_ok = true; + /* + * Participants that aren't doing application eviction and + * are showing a reasonable amount of usage are excluded + * even if we have been trying for a while. + */ + else if (bump_threshold < WT_CACHE_POOL_BUMP_THRESHOLD / 3 && + (!busy && highest > 1)) + decrease_ok = true; + /* + * Any participant that is proportionally less busy is a + * candidate from the first attempt. + */ + else if (highest > 1 && + pressure < WT_CACHE_POOL_REDUCE_THRESHOLD) + decrease_ok = true; + /* * If the entry is currently allocated less than the reserved * size, increase its allocation. This should only happen if: @@ -624,17 +659,12 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, * Conditions for reducing the amount of resources for an * entry: * - the pool is full, - * - application threads are not busy doing eviction already, * - this entry has more than the minimum amount of space in * use, - * - the read pressure in this entry is below the threshold, - * other entries need more cache, the entry has more than - * the minimum space and there is no available space in the - * pool. + * - it was determined that this slot is a good candidate */ - } else if (pool_full && !busy && - entry->cache_size > reserved && - pressure < WT_CACHE_POOL_REDUCE_THRESHOLD && highest > 1) { + } else if (pool_full && + entry->cache_size > reserved && decrease_ok) { grow = false; /* * Don't drop the size down too much - or it can @@ -733,7 +763,7 @@ __wt_cache_pool_server(void *arg) F_ISSET(cache, WT_CACHE_POOL_RUN)) { if (cp->currently_used <= cp->size) __wt_cond_wait( - session, cp->cache_pool_cond, WT_MILLION); + session, cp->cache_pool_cond, WT_MILLION, NULL); /* * Re-check pool run flag - since we want to avoid getting the diff --git a/src/third_party/wiredtiger/src/conn/conn_ckpt.c b/src/third_party/wiredtiger/src/conn/conn_ckpt.c index 1d18c128c5b..dea261162c2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_ckpt.c +++ b/src/third_party/wiredtiger/src/conn/conn_ckpt.c @@ -64,6 +64,16 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, bool *startp) return (0); } +/* + * __ckpt_server_run_chk -- + * Check to decide if the checkpoint server should continue running. + */ +static bool +__ckpt_server_run_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_SERVER_CHECKPOINT)); +} + /* * __ckpt_server -- * The checkpoint server thread. @@ -80,14 +90,18 @@ __ckpt_server(void *arg) conn = S2C(session); wt_session = (WT_SESSION *)session; - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_CHECKPOINT)) { + for (;;) { /* * Wait... * NOTE: If the user only configured logsize, then usecs * will be 0 and this wait won't return until signalled. */ - __wt_cond_wait(session, conn->ckpt_cond, conn->ckpt_usecs); + __wt_cond_wait(session, + conn->ckpt_cond, conn->ckpt_usecs, __ckpt_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__ckpt_server_run_chk(session)) + break; /* * Checkpoint the database if the connection is marked dirty. @@ -115,7 +129,8 @@ __ckpt_server(void *arg) * it so we don't do another checkpoint * immediately. */ - __wt_cond_wait(session, conn->ckpt_cond, 1); + __wt_cond_wait( + session, conn->ckpt_cond, 1, NULL); } } else WT_STAT_CONN_INCR(session, txn_checkpoint_skipped); @@ -154,8 +169,7 @@ __ckpt_server_start(WT_CONNECTION_IMPL *conn) "checkpoint-server", true, session_flags, &conn->ckpt_session)); session = conn->ckpt_session; - WT_RET(__wt_cond_alloc( - session, "checkpoint server", false, &conn->ckpt_cond)); + WT_RET(__wt_cond_alloc(session, "checkpoint server", &conn->ckpt_cond)); /* * Start the thread. @@ -219,7 +233,7 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join(session, conn->ckpt_tid)); conn->ckpt_tid_set = false; } - WT_TRET(__wt_cond_destroy(session, &conn->ckpt_cond)); + __wt_cond_destroy(session, &conn->ckpt_cond); /* Close the server thread's session. */ if (conn->ckpt_session != NULL) { diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index ac72e330b67..1816e66b0b7 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -12,37 +12,47 @@ * __conn_dhandle_destroy -- * Destroy a data handle. */ -static void +static int __conn_dhandle_destroy(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) { + WT_DECL_RET; + + WT_WITH_DHANDLE(session, dhandle, ret = __wt_btree_discard(session)); + __wt_rwlock_destroy(session, &dhandle->rwlock); __wt_free(session, dhandle->name); __wt_free(session, dhandle->checkpoint); - __wt_free(session, dhandle->handle); __wt_spin_destroy(session, &dhandle->close_lock); __wt_stat_dsrc_discard(session, dhandle); __wt_overwrite_and_free(session, dhandle); + return (ret); } /* - * __conn_dhandle_alloc -- + * __wt_conn_dhandle_alloc -- * Allocate a new data handle and return it linked into the connection's * list. */ -static int -__conn_dhandle_alloc(WT_SESSION_IMPL *session, - const char *uri, const char *checkpoint, WT_DATA_HANDLE **dhandlep) +int +__wt_conn_dhandle_alloc( + WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) { WT_BTREE *btree; WT_DATA_HANDLE *dhandle; WT_DECL_RET; uint64_t bucket; - *dhandlep = NULL; + /* + * Ensure no one beat us to creating the handle now that we hold the + * write lock. + */ + if ((ret = + __wt_conn_dhandle_find(session, uri, checkpoint)) != WT_NOTFOUND) + return (ret); WT_RET(__wt_calloc_one(session, &dhandle)); - WT_ERR(__wt_rwlock_alloc(session, &dhandle->rwlock, "data handle")); + WT_ERR(__wt_rwlock_init(session, &dhandle->rwlock)); dhandle->name_hash = __wt_hash_city64(uri, strlen(uri)); WT_ERR(__wt_strdup(session, uri, &dhandle->name)); WT_ERR(__wt_strdup(session, checkpoint, &dhandle->checkpoint)); @@ -75,10 +85,10 @@ __conn_dhandle_alloc(WT_SESSION_IMPL *session, bucket = dhandle->name_hash % WT_HASH_ARRAY_SIZE; WT_CONN_DHANDLE_INSERT(S2C(session), dhandle, bucket); - *dhandlep = dhandle; + session->dhandle = dhandle; return (0); -err: __conn_dhandle_destroy(session, dhandle); +err: WT_TRET(__conn_dhandle_destroy(session, dhandle)); return (ret); } @@ -122,10 +132,7 @@ __wt_conn_dhandle_find( } } - WT_RET(__conn_dhandle_alloc(session, uri, checkpoint, &dhandle)); - - session->dhandle = dhandle; - return (0); + return (WT_NOTFOUND); } /* @@ -153,11 +160,11 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_RET(__wt_evict_file_exclusive_on(session)); /* - * If we don't already have the schema lock, make it an error to try - * to acquire it. The problem is that we are holding an exclusive - * lock on the handle, and if we attempt to acquire the schema lock - * we might deadlock with a thread that has the schema lock and wants - * a handle lock (specifically, checkpoint). + * If we don't already have the schema lock, make it an error to try to + * acquire it. The problem is that we are holding an exclusive lock on + * the handle, and if we attempt to acquire the schema lock we might + * deadlock with a thread that has the schema lock and wants a handle + * lock. */ no_schema_lock = false; if (!F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)) { @@ -197,6 +204,7 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) } WT_TRET(__wt_btree_close(session)); + F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* * If we marked a handle dead it will be closed by sweep, via @@ -306,7 +314,8 @@ __wt_conn_btree_open( F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && !LF_ISSET(WT_DHANDLE_LOCK_ONLY)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_CLOSING)); + WT_ASSERT(session, + !F_ISSET(S2C(session), WT_CONN_CLOSING_NO_MORE_OPENS)); /* * If the handle is already open, it has to be closed so it can be @@ -400,10 +409,7 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, return (ret == EBUSY ? 0 : ret); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - if (WT_META_TRACKING(session)) - WT_TRET(__wt_meta_track_handle_lock(session, false)); - else - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_btree(session)); return (ret); } @@ -419,12 +425,11 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; + WT_DECL_RET; uint64_t bucket; conn = S2C(session); - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); - /* * If we're given a URI, then we walk only the hash list for that * name. If we don't have a URI we walk the entire dhandle list. @@ -432,29 +437,85 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, if (uri != NULL) { bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; - TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { + + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, + &conn->dhhash[bucket], hashq)); + if (dhandle == NULL) + return (0); + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || dhandle->checkpoint != NULL || strcmp(uri, dhandle->name) != 0) continue; - WT_RET(__conn_btree_apply_internal( - session, dhandle, file_func, name_func, cfg)); + WT_ERR(__conn_btree_apply_internal(session, + dhandle, file_func, name_func, cfg)); } } else { - TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); + if (dhandle == NULL) + return (0); + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || dhandle->checkpoint != NULL || !WT_PREFIX_MATCH(dhandle->name, "file:") || - WT_IS_METADATA(session, dhandle)) + WT_IS_METADATA(dhandle)) continue; - WT_RET(__conn_btree_apply_internal( - session, dhandle, file_func, name_func, cfg)); + WT_ERR(__conn_btree_apply_internal(session, + dhandle, file_func, name_func, cfg)); } } - return (0); +err: WT_DHANDLE_RELEASE(dhandle); + return (ret); +} + +/* + * __conn_dhandle_close_one -- + * Lock and, if necessary, close a data handle. + */ +static int +__conn_dhandle_close_one(WT_SESSION_IMPL *session, + const char *uri, const char *checkpoint, bool force) +{ + WT_DECL_RET; + + /* + * Lock the handle exclusively. If this is part of schema-changing + * operation (indicated by metadata tracking being enabled), hold the + * lock for the duration of the operation. + */ + WT_RET(__wt_session_get_btree(session, uri, checkpoint, + NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); + if (WT_META_TRACKING(session)) + WT_RET(__wt_meta_track_handle_lock(session, false)); + + /* + * We have an exclusive lock, which means there are no cursors open at + * this point. Close the handle, if necessary. + */ + if (F_ISSET(session->dhandle, WT_DHANDLE_OPEN)) { + __wt_meta_track_sub_on(session); + ret = __wt_conn_btree_sync_and_close(session, false, force); + + /* + * If the close succeeded, drop any locks it acquired. If + * there was a failure, this function will fail and the whole + * transaction will be rolled back. + */ + if (ret == 0) + ret = __wt_meta_track_sub_off(session); + } + + if (!WT_META_TRACKING(session)) + WT_TRET(__wt_session_release_btree(session)); + + return (ret); } /* @@ -473,46 +534,26 @@ __wt_conn_dhandle_close_all( conn = S2C(session); - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + WT_ASSERT(session, + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, session->dhandle == NULL); + /* + * Lock the live handle first. This ordering is important: we rely on + * locking the live handle to fail fast if the tree is busy (e.g., with + * cursors open or in a checkpoint). + */ + WT_ERR(__conn_dhandle_close_one(session, uri, NULL, force)); + bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { if (strcmp(dhandle->name, uri) != 0 || + dhandle->checkpoint == NULL || F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; - session->dhandle = dhandle; - - /* Lock the handle exclusively. */ - WT_ERR(__wt_session_get_btree(session, - dhandle->name, dhandle->checkpoint, - NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); - if (WT_META_TRACKING(session)) - WT_ERR(__wt_meta_track_handle_lock(session, false)); - - /* - * We have an exclusive lock, which means there are no cursors - * open at this point. Close the handle, if necessary. - */ - if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - __wt_meta_track_sub_on(session); - ret = __wt_conn_btree_sync_and_close( - session, false, force); - - /* - * If the close succeeded, drop any locks it acquired. - * If there was a failure, this function will fail and - * the whole transaction will be rolled back. - */ - if (ret == 0) - ret = __wt_meta_track_sub_off(session); - } - - if (!WT_META_TRACKING(session)) - WT_TRET(__wt_session_release_btree(session)); - - WT_ERR(ret); + WT_ERR(__conn_dhandle_close_one( + session, dhandle->name, dhandle->checkpoint, force)); } err: session->dhandle = NULL; @@ -534,7 +575,8 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, bool final) dhandle = session->dhandle; bucket = dhandle->name_hash % WT_HASH_ARRAY_SIZE; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + WT_ASSERT(session, + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, dhandle != conn->cache->evict_file_next); /* Check if the handle was reacquired by a session while we waited. */ @@ -583,7 +625,7 @@ __wt_conn_dhandle_discard_single( } /* Try to remove the handle, protected by the data handle lock. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __conn_dhandle_remove(session, final)); if (set_pass_intr) (void)__wt_atomic_subv32(&S2C(session)->cache->pass_intr, 1); @@ -594,7 +636,7 @@ __wt_conn_dhandle_discard_single( */ if (ret == 0 || final) { __conn_btree_config_clear(session); - __conn_dhandle_destroy(session, dhandle); + WT_TRET(__conn_dhandle_destroy(session, dhandle)); session->dhandle = NULL; } @@ -627,7 +669,7 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) */ restart: TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; WT_WITH_DHANDLE(session, dhandle, diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 3571cc60115..5e4a8c29adc 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -53,25 +53,18 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) /* Spinlocks. */ WT_RET(__wt_spin_init(session, &conn->api_lock, "api")); WT_SPIN_INIT_TRACKED(session, &conn->checkpoint_lock, checkpoint); - WT_SPIN_INIT_TRACKED(session, &conn->dhandle_lock, handle_list); WT_RET(__wt_spin_init(session, &conn->encryptor_lock, "encryptor")); WT_RET(__wt_spin_init(session, &conn->fh_lock, "file list")); WT_RET(__wt_spin_init(session, &conn->las_lock, "lookaside table")); WT_SPIN_INIT_TRACKED(session, &conn->metadata_lock, metadata); WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_SPIN_INIT_TRACKED(session, &conn->schema_lock, schema); - WT_SPIN_INIT_TRACKED(session, &conn->table_lock, table); WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ - WT_RET(__wt_rwlock_alloc( - session, &conn->hot_backup_lock, "hot backup")); - - WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS, &conn->page_lock)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->page_lock); - for (i = 0; i < WT_PAGE_LOCKS; ++i) - WT_RET( - __wt_spin_init(session, &conn->page_lock[i], "btree page")); + WT_RET(__wt_rwlock_init(session, &conn->dhandle_lock)); + WT_RET(__wt_rwlock_init(session, &conn->hot_backup_lock)); + WT_RET(__wt_rwlock_init(session, &conn->table_lock)); /* Setup the spin locks for the LSM manager queues. */ WT_RET(__wt_spin_init(session, @@ -81,7 +74,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init( session, &conn->lsm_manager.switch_lock, "LSM switch queue lock")); WT_RET(__wt_cond_alloc( - session, "LSM worker cond", false, &conn->lsm_manager.work_cond)); + session, "LSM worker cond", &conn->lsm_manager.work_cond)); /* * Generation numbers. @@ -111,16 +104,14 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) * __wt_connection_destroy -- * Destroy the connection's underlying WT_CONNECTION_IMPL structure. */ -int +void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) { - WT_DECL_RET; WT_SESSION_IMPL *session; - u_int i; /* Check there's something to destroy. */ if (conn == NULL) - return (0); + return; session = conn->default_session; @@ -137,7 +128,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->api_lock); __wt_spin_destroy(session, &conn->block_lock); __wt_spin_destroy(session, &conn->checkpoint_lock); - __wt_spin_destroy(session, &conn->dhandle_lock); + __wt_rwlock_destroy(session, &conn->dhandle_lock); __wt_spin_destroy(session, &conn->encryptor_lock); __wt_spin_destroy(session, &conn->fh_lock); __wt_rwlock_destroy(session, &conn->hot_backup_lock); @@ -145,16 +136,8 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->metadata_lock); __wt_spin_destroy(session, &conn->reconfig_lock); __wt_spin_destroy(session, &conn->schema_lock); - __wt_spin_destroy(session, &conn->table_lock); + __wt_rwlock_destroy(session, &conn->table_lock); __wt_spin_destroy(session, &conn->turtle_lock); - for (i = 0; i < WT_PAGE_LOCKS; ++i) - __wt_spin_destroy(session, &conn->page_lock[i]); - __wt_free(session, conn->page_lock); - - /* Destroy the file-system configuration. */ - if (conn->file_system != NULL && conn->file_system->terminate != NULL) - WT_TRET(conn->file_system->terminate( - conn->file_system, (WT_SESSION *)session)); /* Free allocated memory. */ __wt_free(session, conn->cfg); @@ -164,5 +147,4 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_stat_connection_discard(session, conn); __wt_free(NULL, conn); - return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 8198b3a1a02..d2ed314fd2e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -174,7 +174,7 @@ __logmgr_config( WT_RET(__logmgr_sync_cfg(session, cfg)); if (conn->log_cond != NULL) - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); return (0); } @@ -237,7 +237,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) * We can only archive files if a hot backup is not in progress or * if we are the backup. */ - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); locked = true; if (!conn->hot_backup || backup_file != 0) { for (i = 0; i < logcount; i++) { @@ -248,7 +248,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) session, WT_LOG_FILENAME, lognum)); } } - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); locked = false; /* @@ -260,7 +260,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) if (0) err: __wt_err(session, ret, "log archive server error"); if (locked) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); return (ret); } @@ -341,7 +341,7 @@ __wt_log_truncate_files( conn = S2C(session); if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_RUN) && + if (F_ISSET(conn, WT_CONN_SERVER_LOG) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) WT_RET_MSG(session, EINVAL, "Attempt to archive manually while a server is running"); @@ -355,9 +355,9 @@ __wt_log_truncate_files( __wt_verbose(session, WT_VERB_LOG, "log_truncate_files: Archive once up to %" PRIu32, backup_file); - __wt_writelock(session, log->log_archive_lock); + __wt_writelock(session, &log->log_archive_lock); ret = __log_archive_once(session, backup_file); - __wt_writeunlock(session, log->log_archive_lock); + __wt_writeunlock(session, &log->log_archive_lock); return (ret); } @@ -382,7 +382,7 @@ __log_file_server(void *arg) conn = S2C(session); log = conn->log; locked = false; - while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * If there is a log file to close, make sure any outstanding * write operations have completed, then fsync and close it. @@ -433,7 +433,7 @@ __log_file_server(void *arg) */ if (!conn->hot_backup) { __wt_readlock( - session, conn->hot_backup_lock); + session, &conn->hot_backup_lock); if (!conn->hot_backup) WT_ERR_ERROR_OK( __wt_ftruncate(session, @@ -441,7 +441,7 @@ __log_file_server(void *arg) close_end_lsn.l.offset), ENOTSUP); __wt_readunlock( - session, conn->hot_backup_lock); + session, &conn->hot_backup_lock); } WT_SET_LSN(&close_end_lsn, close_end_lsn.l.file + 1, 0); @@ -505,8 +505,7 @@ __log_file_server(void *arg) locked = false; __wt_spin_unlock(session, &log->log_sync_lock); } else { - __wt_cond_auto_signal( - session, conn->log_wrlsn_cond); + __wt_cond_signal(session, conn->log_wrlsn_cond); /* * We do not want to wait potentially a second * to process this. Yield to give the wrlsn @@ -517,8 +516,9 @@ __log_file_server(void *arg) continue; } } + /* Wait until the next event. */ - __wt_cond_wait(session, conn->log_file_cond, WT_MILLION / 10); + __wt_cond_wait(session, conn->log_file_cond, 100000, NULL); } if (0) { @@ -708,7 +708,7 @@ __log_wrlsn_server(void *arg) log = conn->log; yield = 0; WT_INIT_LSN(&prev); - while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * Write out any log record buffers if anything was done * since last time. Only call the function to walk the @@ -730,12 +730,8 @@ __log_wrlsn_server(void *arg) if (yield++ < WT_THOUSAND) __wt_yield(); else - /* - * Send in false because if we did any work we would - * not be on this path. - */ __wt_cond_auto_wait( - session, conn->log_wrlsn_cond, did_work); + session, conn->log_wrlsn_cond, did_work, NULL); } /* * On close we need to do this one more time because there could @@ -787,7 +783,7 @@ __log_server(void *arg) * takes to sync out an earlier file. */ did_work = true; - while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * Slots depend on future activity. Force out buffered * writes in case we are idle. This cannot be part of the @@ -814,10 +810,11 @@ __log_server(void *arg) * agreed not to rename or remove any files in * the database directory. */ - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (!conn->hot_backup) ret = __log_prealloc_once(session); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock( + session, &conn->hot_backup_lock); WT_ERR(ret); } @@ -826,10 +823,10 @@ __log_server(void *arg) */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) { if (__wt_try_writelock( - session, log->log_archive_lock) == 0) { + session, &log->log_archive_lock) == 0) { ret = __log_archive_once(session, 0); __wt_writeunlock( - session, log->log_archive_lock); + session, &log->log_archive_lock); WT_ERR(ret); } else __wt_verbose(session, WT_VERB_LOG, @@ -839,10 +836,9 @@ __log_server(void *arg) } /* Wait until the next event. */ - __wt_epoch(session, &start); - __wt_cond_auto_wait_signal(session, - conn->log_cond, did_work, &signalled); + __wt_cond_auto_wait_signal( + session, conn->log_cond, did_work, NULL, &signalled); __wt_epoch(session, &now); timediff = WT_TIMEDIFF_MS(now, start); } @@ -884,8 +880,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_spin_init(session, &log->log_sync_lock, "log sync")); WT_RET(__wt_spin_init(session, &log->log_writelsn_lock, "log write LSN")); - WT_RET(__wt_rwlock_alloc(session, - &log->log_archive_lock, "log archive lock")); + WT_RET(__wt_rwlock_init(session, &log->log_archive_lock)); if (FLD_ISSET(conn->direct_io, WT_DIRECT_IO_LOG)) log->allocsize = (uint32_t) WT_MAX(conn->buffer_alignment, WT_LOG_ALIGN); @@ -904,10 +899,8 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_INIT_LSN(&log->write_lsn); WT_INIT_LSN(&log->write_start_lsn); log->fileid = 0; - WT_RET(__wt_cond_alloc( - session, "log sync", false, &log->log_sync_cond)); - WT_RET(__wt_cond_alloc( - session, "log write", false, &log->log_write_cond)); + WT_RET(__wt_cond_alloc(session, "log sync", &log->log_sync_cond)); + WT_RET(__wt_cond_alloc(session, "log write", &log->log_write_cond)); WT_RET(__wt_log_open(session)); WT_RET(__wt_log_slot_init(session)); @@ -930,6 +923,8 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); + F_SET(conn, WT_CONN_SERVER_LOG); + /* * Start the log close thread. It is not configurable. * If logging is enabled, this thread runs. @@ -937,8 +932,8 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) session_flags = WT_SESSION_NO_DATA_HANDLES; WT_RET(__wt_open_internal_session(conn, "log-close-server", false, session_flags, &conn->log_file_session)); - WT_RET(__wt_cond_alloc(conn->log_file_session, - "log close server", false, &conn->log_file_cond)); + WT_RET(__wt_cond_alloc( + conn->log_file_session, "log close server", &conn->log_file_cond)); /* * Start the log file close thread. @@ -954,8 +949,7 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) WT_RET(__wt_open_internal_session(conn, "log-wrlsn-server", false, session_flags, &conn->log_wrlsn_session)); WT_RET(__wt_cond_auto_alloc(conn->log_wrlsn_session, - "log write lsn server", false, 10000, WT_MILLION, - &conn->log_wrlsn_cond)); + "log write lsn server", 10000, WT_MILLION, &conn->log_wrlsn_cond)); WT_RET(__wt_thread_create(conn->log_wrlsn_session, &conn->log_wrlsn_tid, __log_wrlsn_server, conn->log_wrlsn_session)); conn->log_wrlsn_tid_set = true; @@ -969,13 +963,13 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) if (conn->log_session != NULL) { WT_ASSERT(session, conn->log_cond != NULL); WT_ASSERT(session, conn->log_tid_set == true); - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); } else { /* The log server gets its own session. */ WT_RET(__wt_open_internal_session(conn, "log-server", false, session_flags, &conn->log_session)); WT_RET(__wt_cond_auto_alloc(conn->log_session, - "log server", false, 50000, WT_MILLION, &conn->log_cond)); + "log server", 50000, WT_MILLION, &conn->log_cond)); /* * Start the thread. @@ -1001,6 +995,8 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn = S2C(session); + F_CLR(conn, WT_CONN_SERVER_LOG); + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) { /* * We always set up the log_path so printlog can work without @@ -1011,7 +1007,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) return (0); } if (conn->log_tid_set) { - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); WT_TRET(__wt_thread_join(session, conn->log_tid)); conn->log_tid_set = false; } @@ -1026,7 +1022,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn->log_file_session = NULL; } if (conn->log_wrlsn_tid_set) { - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); + __wt_cond_signal(session, conn->log_wrlsn_cond); WT_TRET(__wt_thread_join(session, conn->log_wrlsn_tid)); conn->log_wrlsn_tid_set = false; } @@ -1047,12 +1043,12 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) } /* Destroy the condition variables now that all threads are stopped */ - WT_TRET(__wt_cond_auto_destroy(session, &conn->log_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log_file_cond)); - WT_TRET(__wt_cond_auto_destroy(session, &conn->log_wrlsn_cond)); + __wt_cond_destroy(session, &conn->log_cond); + __wt_cond_destroy(session, &conn->log_file_cond); + __wt_cond_destroy(session, &conn->log_wrlsn_cond); - WT_TRET(__wt_cond_destroy(session, &conn->log->log_sync_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log->log_write_cond)); + __wt_cond_destroy(session, &conn->log->log_sync_cond); + __wt_cond_destroy(session, &conn->log->log_write_cond); __wt_rwlock_destroy(session, &conn->log->log_archive_lock); __wt_spin_destroy(session, &conn->log->log_lock); __wt_spin_destroy(session, &conn->log->log_slot_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 69b50147bf5..eb3c79422a0 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -21,16 +21,9 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) session = conn->default_session; WT_ASSERT(session, session->iface.connection == &conn->iface); - /* - * Tell internal server threads to run: this must be set before opening - * any sessions. - */ - F_SET(conn, WT_CONN_SERVER_RUN | WT_CONN_LOG_SERVER_RUN); - /* WT_SESSION_IMPL array. */ WT_RET(__wt_calloc(session, conn->session_size, sizeof(WT_SESSION_IMPL), &conn->sessions)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->sessions); /* * Open the default session. We open this before starting service @@ -95,13 +88,22 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) for (;;) { WT_TRET(__wt_txn_update_oldest(session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); - if (txn_global->oldest_id == txn_global->current) + if (txn_global->oldest_id == txn_global->current && + txn_global->metadata_pinned == txn_global->current) break; __wt_yield(); } - /* Clear any pending async ops. */ + /* Shut down the subsystems, ensuring workers see the state change. */ + F_SET(conn, WT_CONN_CLOSING); + WT_FULL_BARRIER(); + + /* + * Clear any pending async operations and shut down the async worker + * threads and system before closing LSM. + */ WT_TRET(__wt_async_flush(session)); + WT_TRET(__wt_async_destroy(session)); /* * Shut down server threads other than the eviction server, which is @@ -109,15 +111,20 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) * btree handles, so take care in ordering shutdown to make sure they * exit before files are closed. */ - F_CLR(conn, WT_CONN_SERVER_RUN); - WT_TRET(__wt_async_destroy(session)); WT_TRET(__wt_lsm_manager_destroy(session)); - WT_TRET(__wt_sweep_destroy(session)); - F_SET(conn, WT_CONN_CLOSING); + /* + * Once the async and LSM threads exit, we shouldn't be opening any + * more files. + */ + F_SET(conn, WT_CONN_CLOSING_NO_MORE_OPENS); + WT_FULL_BARRIER(); WT_TRET(__wt_checkpoint_server_destroy(session)); WT_TRET(__wt_statlog_destroy(session, true)); + WT_TRET(__wt_sweep_destroy(session)); + + /* The eviction server is shut down last. */ WT_TRET(__wt_evict_destroy(session)); /* Shut down the lookaside table, after all eviction is complete. */ @@ -126,7 +133,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* Close open data handles. */ WT_TRET(__wt_conn_dhandle_discard(session)); - /* Shut down metadata tracking, required before creating tables. */ + /* Shut down metadata tracking. */ WT_TRET(__wt_meta_track_destroy(session)); /* @@ -140,7 +147,6 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE)) WT_TRET(__wt_txn_checkpoint_log( session, true, WT_TXN_LOG_CKPT_STOP, NULL)); - F_CLR(conn, WT_CONN_LOG_SERVER_RUN); WT_TRET(__wt_logmgr_destroy(session)); /* Free memory for collators, compressors, data sources. */ @@ -159,15 +165,6 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* Discard transaction state. */ __wt_txn_global_destroy(session); - /* Close extensions, first calling any unload entry point. */ - while ((dlh = TAILQ_FIRST(&conn->dlhqh)) != NULL) { - TAILQ_REMOVE(&conn->dlhqh, dlh, q); - - if (dlh->terminate != NULL) - WT_TRET(dlh->terminate(wt_conn)); - WT_TRET(__wt_dlclose(session, dlh)); - } - /* Close the lock file, opening up the database to other connections. */ if (conn->lock_fh != NULL) WT_TRET(__wt_close(session, &conn->lock_fh)); @@ -199,8 +196,22 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) __wt_free(session, s->hazard); } + /* Destroy the file-system configuration. */ + if (conn->file_system != NULL && conn->file_system->terminate != NULL) + WT_TRET(conn->file_system->terminate( + conn->file_system, (WT_SESSION *)session)); + + /* Close extensions, first calling any unload entry point. */ + while ((dlh = TAILQ_FIRST(&conn->dlhqh)) != NULL) { + TAILQ_REMOVE(&conn->dlhqh, dlh, q); + + if (dlh->terminate != NULL) + WT_TRET(dlh->terminate(wt_conn)); + WT_TRET(__wt_dlclose(session, dlh)); + } + /* Destroy the handle. */ - WT_TRET(__wt_connection_destroy(conn)); + __wt_connection_destroy(conn); return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index 3bcdfd7ecb1..3234d51cd4c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -409,7 +409,6 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) struct timespec ts; struct tm *tm, _tm; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; WT_FSTREAM *log_stream; conn = S2C(session); @@ -446,12 +445,9 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) * Lock the schema and walk the list of open handles, dumping * any that match the list of object sources. */ - if (conn->stat_sources != NULL) { - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_conn_btree_apply( + if (conn->stat_sources != NULL) + WT_RET(__wt_conn_btree_apply( session, NULL, __statlog_apply, NULL, NULL)); - WT_RET(ret); - } /* * Walk the list of open LSM trees, dumping any that match the @@ -485,8 +481,7 @@ __statlog_on_close(WT_SESSION_IMPL *session) if (!FLD_ISSET(conn->stat_flags, WT_STAT_ON_CLOSE)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) + if (F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) WT_RET_MSG(session, EINVAL, "Attempt to log statistics while a server is running"); @@ -497,6 +492,16 @@ err: __wt_scr_free(session, &tmp); return (ret); } +/* + * __statlog_server_run_chk -- + * Check to decide if the statistics log server should continue running. + */ +static bool +__statlog_server_run_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_SERVER_STATISTICS)); +} + /* * __statlog_server -- * The statistics server thread. @@ -525,10 +530,14 @@ __statlog_server(void *arg) WT_ERR(__wt_buf_init(session, &path, strlen(conn->stat_path) + 128)); WT_ERR(__wt_buf_init(session, &tmp, strlen(conn->stat_path) + 128)); - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) { + for (;;) { /* Wait until the next event. */ - __wt_cond_wait(session, conn->stat_cond, conn->stat_usecs); + __wt_cond_wait(session, conn->stat_cond, + conn->stat_usecs, __statlog_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__statlog_server_run_chk(session)) + break; if (WT_STAT_ENABLED(session)) WT_ERR(__statlog_log_one(session, &path, &tmp)); @@ -563,7 +572,7 @@ __statlog_start(WT_CONNECTION_IMPL *conn) session = conn->stat_session; WT_RET(__wt_cond_alloc( - session, "statistics log server", false, &conn->stat_cond)); + session, "statistics log server", &conn->stat_cond)); /* * Start the thread. @@ -639,7 +648,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_TRET(__wt_thread_join(session, conn->stat_tid)); conn->stat_tid_set = false; } - WT_TRET(__wt_cond_destroy(session, &conn->stat_cond)); + __wt_cond_destroy(session, &conn->stat_cond); /* Log a set of statistics on shutdown if configured. */ if (is_close) diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index dba37fa2eb0..e273f1d08e5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -10,7 +10,7 @@ #define WT_DHANDLE_CAN_DISCARD(dhandle) \ (!F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_OPEN) && \ - dhandle->session_inuse == 0 && dhandle->session_ref == 0) + (dhandle)->session_inuse == 0 && (dhandle)->session_ref == 0) /* * __sweep_mark -- @@ -26,7 +26,7 @@ __sweep_mark(WT_SESSION_IMPL *session, time_t now) conn = S2C(session); TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; /* @@ -81,7 +81,7 @@ __sweep_expire_one(WT_SESSION_IMPL *session) * handle list lock so that connection-level handle searches * never need to retry. */ - WT_RET(__wt_try_writelock(session, dhandle->rwlock)); + WT_RET(__wt_try_writelock(session, &dhandle->rwlock)); /* Only sweep clean trees where all updates are visible. */ if (btree->modified || @@ -95,7 +95,7 @@ __sweep_expire_one(WT_SESSION_IMPL *session) */ ret = __wt_conn_btree_sync_and_close(session, false, true); -err: __wt_writeunlock(session, dhandle->rwlock); +err: __wt_writeunlock(session, &dhandle->rwlock); return (ret); } @@ -122,7 +122,7 @@ __sweep_expire(WT_SESSION_IMPL *session, time_t now) if (conn->open_btree_count < conn->sweep_handles_min) break; - if (WT_IS_METADATA(session, dhandle) || + if (WT_IS_METADATA(dhandle) || !F_ISSET(dhandle, WT_DHANDLE_OPEN) || dhandle->session_inuse != 0 || dhandle->timeofdeath == 0 || @@ -188,7 +188,7 @@ __sweep_remove_one(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) WT_DECL_RET; /* Try to get exclusive access. */ - WT_RET(__wt_try_writelock(session, dhandle->rwlock)); + WT_RET(__wt_try_writelock(session, &dhandle->rwlock)); /* * If there are no longer any references to the handle in any @@ -205,7 +205,7 @@ __sweep_remove_one(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) * don't retry the discard until it times out again. */ if (ret != 0) { -err: __wt_writeunlock(session, dhandle->rwlock); +err: __wt_writeunlock(session, &dhandle->rwlock); } return (ret); @@ -228,12 +228,12 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) dhandle != NULL; dhandle = dhandle_next) { dhandle_next = TAILQ_NEXT(dhandle, q); - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; if (!WT_DHANDLE_CAN_DISCARD(dhandle)) continue; - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __sweep_remove_one(session, dhandle)); if (ret == 0) WT_STAT_CONN_INCR(session, dh_sweep_remove); @@ -245,6 +245,16 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) return (ret == EBUSY ? 0 : ret); } +/* + * __sweep_server_run_chk -- + * Check to decide if the checkpoint server should continue running. + */ +static bool +__sweep_server_run_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_SERVER_SWEEP)); +} + /* * __sweep_server -- * The handle sweep server thread. @@ -266,11 +276,15 @@ __sweep_server(void *arg) /* * Sweep for dead and excess handles. */ - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_SWEEP)) { + for (;;) { /* Wait until the next event. */ - __wt_cond_wait(session, - conn->sweep_cond, conn->sweep_interval * WT_MILLION); + __wt_cond_wait(session, conn->sweep_cond, + conn->sweep_interval * WT_MILLION, __sweep_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__sweep_server_run_chk(session)) + break; + __wt_seconds(session, &now); WT_STAT_CONN_INCR(session, dh_sweeps); @@ -390,7 +404,7 @@ __wt_sweep_create(WT_SESSION_IMPL *session) session = conn->sweep_session; WT_RET(__wt_cond_alloc( - session, "handle sweep server", false, &conn->sweep_cond)); + session, "handle sweep server", &conn->sweep_cond)); WT_RET(__wt_thread_create( session, &conn->sweep_tid, __sweep_server, session)); @@ -418,7 +432,7 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join(session, conn->sweep_tid)); conn->sweep_tid_set = 0; } - WT_TRET(__wt_cond_destroy(session, &conn->sweep_cond)); + __wt_cond_destroy(session, &conn->sweep_cond); if (conn->sweep_session != NULL) { wt_session = &conn->sweep_session->iface; diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 3585082644f..61ced8d11e7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -144,8 +144,8 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, * Start the backup and fill in the cursor's list. Acquire the schema * lock, we need a consistent view when creating a copy. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __backup_start(session, cb, cfg))); WT_ERR(ret); @@ -230,10 +230,10 @@ __backup_start( * We are holding the checkpoint and schema locks so schema operations * will not see the backup file list until it is complete and valid. */ - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup = true; conn->hot_backup_list = NULL; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); /* We're the lock holder, we own cleanup. */ F_SET(cb, WT_CURBACKUP_LOCKER); @@ -297,9 +297,9 @@ err: /* Close the hot backup file. */ if (ret == 0) { WT_ASSERT(session, dest != NULL); WT_TRET(__wt_fs_rename(session, WT_BACKUP_TMP, dest, false)); - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup_list = cb->list; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); } return (ret); @@ -319,9 +319,9 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) conn = S2C(session); /* Release all btree names held by the backup. */ - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup_list = NULL; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); if (cb->list != NULL) { for (i = 0; cb->list[i] != NULL; ++i) __wt_free(session, cb->list[i]); @@ -332,9 +332,9 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) WT_TRET(__wt_backup_file_remove(session)); /* Checkpoint deletion can proceed, as can the next hot backup. */ - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup = false; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); return (ret); } @@ -346,13 +346,9 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) static int __backup_all(WT_SESSION_IMPL *session) { - WT_DECL_RET; - /* Build a list of the file objects that need to be copied. */ - WT_WITH_HANDLE_LIST_LOCK(session, ret = - __wt_meta_apply_all(session, NULL, __backup_list_uri_append, NULL)); - - return (ret); + return (__wt_meta_apply_all( + session, NULL, __backup_list_uri_append, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index 2d3f3ffd176..4001188e21c 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -49,8 +49,6 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_CONFIG, iface) == 0); - WT_UNUSED(uri); - WT_RET(__wt_calloc_one(session, &cconfig)); cursor = &cconfig->iface; diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 458e0f1b1f0..131d1ffa930 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -12,13 +12,11 @@ * __curds_txn_enter -- * Do transactional initialization when starting an operation. */ -static int +static void __curds_txn_enter(WT_SESSION_IMPL *session) { session->ncursors++; /* XXX */ - WT_RET(__wt_txn_cursor_op(session)); - - return (0); + __wt_txn_cursor_op(session); } /* @@ -187,7 +185,7 @@ __curds_next(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_next); WT_STAT_DATA_INCR(session, cursor_next); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); ret = __curds_cursor_resolve(cursor, source->next(source)); @@ -215,7 +213,7 @@ __curds_prev(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_prev); WT_STAT_DATA_INCR(session, cursor_prev); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); ret = __curds_cursor_resolve(cursor, source->prev(source)); @@ -267,7 +265,7 @@ __curds_search(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = __curds_cursor_resolve(cursor, source->search(source)); @@ -295,7 +293,7 @@ __curds_search_near(WT_CURSOR *cursor, int *exact) WT_STAT_CONN_INCR(session, cursor_search_near); WT_STAT_DATA_INCR(session, cursor_search_near); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = @@ -321,7 +319,7 @@ __curds_insert(WT_CURSOR *cursor) CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_STAT_CONN_INCR(session, cursor_insert); WT_STAT_DATA_INCR(session, cursor_insert); @@ -358,7 +356,7 @@ __curds_update(WT_CURSOR *cursor) WT_STAT_DATA_INCR(session, cursor_update); WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); WT_ERR(__curds_value_set(cursor)); @@ -389,7 +387,7 @@ __curds_remove(WT_CURSOR *cursor) WT_STAT_DATA_INCR(session, cursor_remove); WT_STAT_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = __curds_cursor_resolve(cursor, source->remove(source)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 9fc466f4c76..bc54f10f2d6 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -8,29 +8,6 @@ #include "wt_internal.h" -/* - * WT_BTREE_CURSOR_SAVE_AND_RESTORE - * Save the cursor's key/value data/size fields, call an underlying btree - * function, and then consistently handle failure and success. - */ -#define WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, f, ret) do { \ - WT_ITEM __key_copy = (cursor)->key; \ - uint64_t __recno = (cursor)->recno; \ - WT_ITEM __value_copy = (cursor)->value; \ - if (((ret) = (f)) == 0) { \ - F_CLR(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); \ - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); \ - } else { \ - if (F_ISSET(cursor, WT_CURSTD_KEY_EXT)) { \ - (cursor)->recno = __recno; \ - WT_ITEM_SET((cursor)->key, __key_copy); \ - } \ - if (F_ISSET(cursor, WT_CURSTD_VALUE_EXT)) \ - WT_ITEM_SET((cursor)->value, __value_copy); \ - F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); \ - } \ -} while (0) - /* * __curfile_compare -- * WT_CURSOR->compare method for the btree cursor type. @@ -109,9 +86,12 @@ __curfile_next(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, next, cbt->btree); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((ret = __wt_btcur_next(cbt, false)) == 0) - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + WT_ERR(__wt_btcur_next(cbt, false)); + + /* Next maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -131,9 +111,12 @@ __wt_curfile_next_random(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, next, cbt->btree); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((ret = __wt_btcur_next_random(cbt)) == 0) - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + WT_ERR(__wt_btcur_next_random(cbt)); + + /* Next-random maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -152,9 +135,12 @@ __curfile_prev(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, prev, cbt->btree); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((ret = __wt_btcur_prev(cbt, false)) == 0) - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + WT_ERR(__wt_btcur_prev(cbt, false)); + + /* Prev maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -175,7 +161,10 @@ __curfile_reset(WT_CURSOR *cursor) ret = __wt_btcur_reset(cbt); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + /* Reset maintains no position, key or value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == 0 && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); err: API_END_RET(session, ret); } @@ -194,10 +183,15 @@ __curfile_search(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_CHECKKEY(cursor); WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_search(cbt), ret); + WT_ERR(__wt_btcur_search(cbt)); + + /* Search maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -216,11 +210,15 @@ __curfile_search_near(WT_CURSOR *cursor, int *exact) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search_near, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_CHECKKEY(cursor); WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE( - cursor, __wt_btcur_search_near(cbt, exact), ret); + WT_ERR(__wt_btcur_search_near(cbt, exact)); + + /* Search-near maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -238,38 +236,33 @@ __curfile_insert(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL(cursor, session, insert, cbt->btree); + if (!F_ISSET(cursor, WT_CURSTD_APPEND)) - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NEEDVALUE(cursor); + WT_CURSOR_CHECKKEY(cursor); + WT_CURSOR_CHECKVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_insert(cbt), ret); + WT_ERR(__wt_btcur_insert(cbt)); /* - * Insert is the one cursor operation that doesn't end with the cursor - * pointing to an on-page item (except for column-store appends, where - * we are returning a key). That is, the application's cursor continues - * to reference the application's memory after a successful cursor call, - * which isn't true anywhere else. We don't want to have to explain that - * scoping corner case, so we reset the application's cursor so it can - * free the referenced memory and continue on without risking subsequent - * core dumps. + * Insert maintains no position, key or value (except for column-store + * appends, where we are returning a key). */ - if (ret == 0) { - if (!F_ISSET(cursor, WT_CURSTD_APPEND)) - F_CLR(cursor, WT_CURSTD_KEY_INT); - F_CLR(cursor, WT_CURSTD_VALUE_INT); - } + WT_ASSERT(session, + (F_ISSET(cursor, WT_CURSTD_APPEND) && + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT) || + (!F_ISSET(cursor, WT_CURSTD_APPEND) && + F_MASK(cursor, WT_CURSTD_KEY_SET) == 0)); err: CURSOR_UPDATE_API_END(session, ret); return (ret); } /* - * __curfile_update -- - * WT_CURSOR->update method for the btree cursor type. + * __wt_curfile_insert_check -- + * WT_CURSOR->insert_check method for the btree cursor type. */ -static int -__curfile_update(WT_CURSOR *cursor) +int +__wt_curfile_insert_check(WT_CURSOR *cursor) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; @@ -278,21 +271,21 @@ __curfile_update(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL(cursor, session, update, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NEEDVALUE(cursor); + WT_CURSOR_CHECKKEY(cursor); + WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_update(cbt), ret); + ret = __wt_btcur_insert_check(cbt); err: CURSOR_UPDATE_API_END(session, ret); return (ret); } /* - * __wt_curfile_update_check -- - * WT_CURSOR->update_check method for the btree cursor type. + * __curfile_update -- + * WT_CURSOR->update method for the btree cursor type. */ -int -__wt_curfile_update_check(WT_CURSOR *cursor) +static int +__curfile_update(WT_CURSOR *cursor) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; @@ -301,11 +294,15 @@ __wt_curfile_update_check(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL(cursor, session, update, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_CURSOR_CHECKKEY(cursor); + WT_CURSOR_CHECKVALUE(cursor); + + WT_ERR(__wt_btcur_update(cbt)); - WT_BTREE_CURSOR_SAVE_AND_RESTORE( - cursor, __wt_btcur_update_check(cbt), ret); + /* Update maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: CURSOR_UPDATE_API_END(session, ret); return (ret); @@ -325,24 +322,21 @@ __curfile_remove(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_REMOVE_API_CALL(cursor, session, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_CHECKKEY(cursor); WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_remove(cbt), ret); + WT_ERR(__wt_btcur_remove(cbt)); /* - * After a successful remove, copy the key: the value is not available. + * Remove with a search-key is fire-and-forget, no position and no key. + * Remove starting from a position maintains the position and a key. + * We don't know which it was at this layer, so can only assert the key + * is not set at all, or internal. There's never a value. */ - if (ret == 0) { - if (F_ISSET(cursor, WT_CURSTD_KEY_INT) && - !WT_DATA_IN_ITEM(&(cursor)->key)) { - WT_ERR(__wt_buf_set(session, &cursor->key, - cursor->key.data, cursor->key.size)); - F_CLR(cursor, WT_CURSTD_KEY_INT); - F_SET(cursor, WT_CURSTD_KEY_EXT); - } - F_CLR(cursor, WT_CURSTD_VALUE_SET); - } + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == 0 || + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + WT_ASSERT(session, F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); err: CURSOR_UPDATE_API_END(session, ret); return (ret); @@ -492,7 +486,14 @@ __curfile_create(WT_SESSION_IMPL *session, WT_STAT_DATA_INCR(session, cursor_create); if (0) { -err: WT_TRET(__curfile_close(cursor)); +err: /* + * Our caller expects to release the data handle if we fail. + * Disconnect it from the cursor before closing. + */ + if (session->dhandle != NULL) + __wt_cursor_dhandle_decr_use(session); + cbt->btree = NULL; + WT_TRET(__curfile_close(cursor)); *cursorp = NULL; } @@ -559,7 +560,7 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, * get the handle while holding the checkpoint lock. */ if (LF_ISSET(WT_DHANDLE_EXCLUSIVE) && checkpoint_wait) - WT_WITH_CHECKPOINT_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, ret = __wt_session_get_btree_ckpt( session, uri, cfg, flags)); else diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index eb5e15ae5c3..6fc01c0421f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -240,7 +240,17 @@ __curindex_search(WT_CURSOR *cursor) found_key = child->key; if (found_key.size < cursor->key.size) WT_ERR(WT_NOTFOUND); - found_key.size = cursor->key.size; + + /* + * Custom collators expect to see complete keys, pass an item containing + * all the visible fields so it unpacks correctly. + */ + if (cindex->index->collator != NULL && + !F_ISSET(cursor, WT_CURSTD_RAW_SEARCH)) + WT_ERR(__wt_struct_repack(session, child->key_format, + cindex->iface.key_format, &child->key, &found_key)); + else + found_key.size = cursor->key.size; WT_ERR(__wt_compare( session, cindex->index->collator, &cursor->key, &found_key, &cmp)); @@ -281,33 +291,48 @@ __curindex_search_near(WT_CURSOR *cursor, int *exact) * (usually) doesn't contain the primary key, so it is just a prefix of * any matching index key. That said, if there is an exact match, we * want to find the first matching index entry and set exact equal to - * zero. Do a search_near, step to the next entry if we land on one - * that is too small, then check that the prefix matches. + * zero. + * + * Do a search_near, and if we find an entry that is too small, step to + * the next one. In the unlikely event of a search past the end of the + * tree, go back to the last key. */ __wt_cursor_set_raw_key(child, &cursor->key); WT_ERR(child->search_near(child, &cmp)); - if (cmp < 0) - WT_ERR(child->next(child)); + if (cmp < 0) { + if ((ret = child->next(child)) == WT_NOTFOUND) + ret = child->prev(child); + WT_ERR(ret); + } /* * We expect partial matches, and want the smallest record with a key * greater than or equal to the search key. * - * If the key we find is shorter than the search key, it can't possibly - * match. + * If the found key starts with the search key, we indicate a match by + * setting exact equal to zero. * - * The only way for the key to be exactly equal is if there is an index - * on the primary key, because otherwise the primary key columns will - * be appended to the index key, but we don't disallow that (odd) case. + * The compare function expects application-supplied keys to come first + * so we flip the sign of the result to match what callers expect. */ found_key = child->key; - if (found_key.size < cursor->key.size) - WT_ERR(WT_NOTFOUND); - found_key.size = cursor->key.size; + if (found_key.size > cursor->key.size) { + /* + * Custom collators expect to see complete keys, pass an item + * containing all the visible fields so it unpacks correctly. + */ + if (cindex->index->collator != NULL) + WT_ERR(__wt_struct_repack(session, + cindex->child->key_format, cindex->iface.key_format, + &child->key, &found_key)); + else + found_key.size = cursor->key.size; + } WT_ERR(__wt_compare( session, cindex->index->collator, &cursor->key, &found_key, exact)); + *exact = -*exact; WT_ERR(__curindex_move(cindex)); @@ -515,8 +540,8 @@ __wt_curindex_open(WT_SESSION_IMPL *session, WT_ERR(__curindex_open_colgroups(session, cindex, cfg)); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) - __wt_json_column_init( - cursor, table->key_format, &idx->colconf, &table->colconf); + __wt_json_column_init(cursor, uri, table->key_format, + &idx->colconf, &table->colconf); if (0) { err: WT_TRET(__curindex_close(cursor)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 806436ebf38..80afaf798dc 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -185,7 +185,7 @@ __curjoin_iter_set_entry(WT_CURSOR_JOIN_ITER *iter, u_int entry_pos) size = strlen(to_dup->internal_uri) + 3; WT_ERR(__wt_calloc(session, size, 1, &uri)); - snprintf(uri, size, "%s()", to_dup->internal_uri); + WT_ERR(__wt_snprintf(uri, size, "%s()", to_dup->internal_uri)); if ((c = iter->cursor) == NULL || !WT_STREQ(c->uri, uri)) { iter->cursor = NULL; if (c != NULL) @@ -270,7 +270,7 @@ again: iter->positioned = true; return (ret); } - else if (ret == WT_NOTFOUND) { + if (ret == WT_NOTFOUND) { WT_RET(__curjoin_iter_close_all(iter->child)); entry->subjoin->iter = NULL; iter->child = NULL; @@ -326,8 +326,7 @@ __curjoin_close(WT_CURSOR *cursor) JOINABLE_CURSOR_API_CALL(cursor, session, close, NULL); __wt_schema_release_table(session, cjoin->table); - /* These are owned by the table */ - cursor->internal_uri = NULL; + /* This is owned by the table */ cursor->key_format = NULL; if (cjoin->projection != NULL) { __wt_free(session, cjoin->projection); @@ -519,8 +518,7 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, } if (disjunction && end == endmax) return (WT_NOTFOUND); - else - return (0); + return (0); } typedef struct { @@ -921,7 +919,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, "cursors"); /* Get a consistent view of our subordinate cursors if appropriate. */ - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); if (F_ISSET((WT_CURSOR *)cjoin, WT_CURSTD_RAW)) config = &raw_cfg[0]; @@ -931,7 +929,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if ((proj = cjoin->projection) != NULL) { size = strlen(urimain) + strlen(proj) + 1; WT_ERR(__wt_calloc(session, size, 1, &mainbuf)); - snprintf(mainbuf, size, "%s%s", urimain, proj); + WT_ERR(__wt_snprintf(mainbuf, size, "%s%s", urimain, proj)); urimain = mainbuf; } WT_ERR(__wt_open_cursor(session, urimain, (WT_CURSOR *)cjoin, config, @@ -976,8 +974,8 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if (!iterable && F_ISSET(je, WT_CURJOIN_ENTRY_BLOOM)) { if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) WT_ERR_MSG(session, EINVAL, - "join cursors with Bloom filters cannot be " - "used with read-uncommitted isolation"); + "join cursors with Bloom filters cannot be " + "used with read-uncommitted isolation"); if (je->bloom == NULL) { /* * Look for compatible filters to be shared, @@ -1150,8 +1148,8 @@ __curjoin_open_main(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, newsize = strlen(cjoin->table->name) + idx->colconf.len + 1; WT_ERR(__wt_calloc(session, 1, newsize, &main_uri)); - snprintf(main_uri, newsize, "%s%.*s", - cjoin->table->name, (int)idx->colconf.len, idx->colconf.str); + WT_ERR(__wt_snprintf(main_uri, newsize, "%s%.*s", + cjoin->table->name, (int)idx->colconf.len, idx->colconf.str)); WT_ERR(__wt_open_cursor(session, main_uri, (WT_CURSOR *)cjoin, raw_cfg, &entry->main)); if (idx->extractor == NULL) { @@ -1164,7 +1162,8 @@ __curjoin_open_main(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, */ len = strlen(entry->main->value_format) + 3; WT_ERR(__wt_calloc(session, len, 1, &newformat)); - snprintf(newformat, len, "%s0x", entry->main->value_format); + WT_ERR(__wt_snprintf( + newformat, len, "%s0x", entry->main->value_format)); __wt_free(session, entry->main->value_format); entry->main->value_format = newformat; } @@ -1310,11 +1309,10 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_RET_MSG(session, EINVAL, "unable to initialize a join cursor with existing owner"); - if (!WT_PREFIX_SKIP(uri, "join:")) - return (__wt_unexpected_object_type(session, uri, "join:")); tablename = uri; - if (!WT_PREFIX_SKIP(tablename, "table:")) - return (__wt_unexpected_object_type(session, uri, "table:")); + if (!WT_PREFIX_SKIP(tablename, "join:table:")) + return ( + __wt_unexpected_object_type(session, uri, "join:table:")); columns = strchr(tablename, '('); if (columns == NULL) @@ -1327,7 +1325,6 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, cursor = &cjoin->iface; *cursor = iface; cursor->session = &session->iface; - cursor->internal_uri = table->name; cursor->key_format = table->key_format; cursor->value_format = table->value_format; cjoin->table = table; @@ -1336,7 +1333,7 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { WT_ERR(__wt_struct_reformat(session, table, - columns, strlen(columns), NULL, 1, tmp)); + columns, strlen(columns), NULL, false, tmp)); WT_ERR(__wt_strndup( session, tmp->data, tmp->size, &cursor->value_format)); WT_ERR(__wt_strdup(session, columns, &cjoin->projection)); @@ -1535,8 +1532,8 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, len = strlen(cindex->iface.key_format) + 3; WT_RET(__wt_calloc(session, len, 1, &entry->repack_format)); - snprintf(entry->repack_format, len, "%s0x", - cindex->iface.key_format); + WT_RET(__wt_snprintf(entry->repack_format, + len, "%s0x", cindex->iface.key_format)); } } return (0); diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index 093ec3c59ac..e8ddb767863 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -8,8 +8,8 @@ #include "wt_internal.h" -static size_t __json_unpack_put(WT_SESSION_IMPL *, void *, u_char *, size_t, - WT_CONFIG_ITEM *); +static int __json_unpack_put( + WT_SESSION_IMPL *, void *, u_char *, size_t, WT_CONFIG_ITEM *, size_t *); static inline int __json_struct_size(WT_SESSION_IMPL *, const void *, size_t, const char *, WT_CONFIG_ITEM *, bool, size_t *); static inline int __json_struct_unpackv(WT_SESSION_IMPL *, const void *, size_t, @@ -23,20 +23,20 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, bool, const char *, size_t *); #define WT_PACK_JSON_GET(session, pv, jstr) do { \ - switch (pv.type) { \ + switch ((pv).type) { \ case 'x': \ break; \ case 's': \ case 'S': \ - WT_RET(json_string_arg(session, &jstr, &pv.u.item)); \ - pv.type = pv.type == 's' ? 'j' : 'J'; \ + WT_RET(json_string_arg(session, &(jstr), &(pv).u.item));\ + (pv).type = (pv).type == 's' ? 'j' : 'J'; \ break; \ case 'b': \ case 'h': \ case 'i': \ case 'l': \ case 'q': \ - WT_RET(json_int_arg(session, &jstr, &pv.u.i)); \ + WT_RET(json_int_arg(session, &(jstr), &(pv).u.i)); \ break; \ case 'B': \ case 'H': \ @@ -46,11 +46,11 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, case 'r': \ case 'R': \ case 't': \ - WT_RET(json_uint_arg(session, &jstr, &pv.u.u)); \ + WT_RET(json_uint_arg(session, &(jstr), &(pv).u.u)); \ break; \ case 'u': \ - WT_RET(json_string_arg(session, &jstr, &pv.u.item)); \ - pv.type = 'K'; \ + WT_RET(json_string_arg(session, &(jstr), &(pv).u.item));\ + (pv).type = 'K'; \ break; \ /* User format strings have already been validated. */ \ WT_ILLEGAL_VALUE(session); \ @@ -61,22 +61,22 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, * __json_unpack_put -- * Calculate the size of a packed byte string as formatted for JSON. */ -static size_t +static int __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, - u_char *buf, size_t bufsz, WT_CONFIG_ITEM *name) + u_char *buf, size_t bufsz, WT_CONFIG_ITEM *name, size_t *retsizep) { WT_PACK_VALUE *pv; const u_char *p, *end; size_t s, n; pv = (WT_PACK_VALUE *)voidpv; - s = (size_t)snprintf((char *)buf, bufsz, "\"%.*s\" : ", - (int)name->len, name->str); + + WT_RET(__wt_snprintf_len_set( + (char *)buf, bufsz, &s, "\"%.*s\" : ", (int)name->len, name->str)); if (s <= bufsz) { bufsz -= s; buf += s; - } - else + } else bufsz = 0; switch (pv->type) { @@ -118,7 +118,8 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, } if (bufsz > 0) *buf++ = '"'; - return (s); + *retsizep += s; + return (0); case 'U': case 'u': s += 2; @@ -140,14 +141,17 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, } if (bufsz > 0) *buf++ = '"'; - return (s); + *retsizep += s; + return (0); case 'b': case 'h': case 'i': case 'l': case 'q': - return (s + - (size_t)snprintf((char *)buf, bufsz, "%" PRId64, pv->u.i)); + WT_RET(__wt_snprintf_len_incr( + (char *)buf, bufsz, &s, "%" PRId64, pv->u.i)); + *retsizep += s; + return (0); case 'B': case 't': case 'H': @@ -156,11 +160,14 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, case 'Q': case 'r': case 'R': - return (s + - (size_t)snprintf((char *)buf, bufsz, "%" PRId64, pv->u.u)); + WT_RET(__wt_snprintf_len_incr( + (char *)buf, bufsz, &s, "%" PRId64, pv->u.u)); + *retsizep += s; + return (0); } - __wt_err(session, EINVAL, "unknown pack-value type: %c", (int)pv->type); - return ((size_t)-1); + + WT_RET_MSG(session, EINVAL, + "unknown pack-value type: %c", (int)pv->type); } /* @@ -194,7 +201,8 @@ __json_struct_size(WT_SESSION_IMPL *session, const void *buffer, needcr = true; WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_RET(__pack_name_next(&packname, &name)); - result += __json_unpack_put(session, &pv, NULL, 0, &name); + WT_RET( + __json_unpack_put(session, &pv, NULL, 0, &name, &result)); } if (ret == WT_NOTFOUND) ret = 0; @@ -243,8 +251,9 @@ __json_struct_unpackv(WT_SESSION_IMPL *session, needcr = true; WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_RET(__pack_name_next(&packname, &name)); - jsize = __json_unpack_put(session, - (u_char *)&pv, jbuf, jbufsize, &name); + jsize = 0; + WT_RET(__json_unpack_put(session, + (u_char *)&pv, jbuf, jbufsize, &name, &jsize)); WT_ASSERT(session, jsize <= jbufsize); jbuf += jsize; jbufsize -= jsize; @@ -270,7 +279,6 @@ __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, bool iskey, va_list ap) { WT_CONFIG_ITEM *names; - WT_DECL_RET; size_t needed; char **json_bufp; @@ -288,7 +296,7 @@ __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, WT_RET(__json_struct_unpackv(session, buffer, size, fmt, names, (u_char *)*json_bufp, needed + 1, iskey, ap)); - return (ret); + return (0); } /* @@ -305,7 +313,6 @@ __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) __wt_free(session, json->value_buf); __wt_free(session, json); } - return; } /* @@ -315,6 +322,7 @@ __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) */ size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { u_char abbrev; @@ -323,33 +331,32 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) if (bufsz >= 1) *buf = ch; return (1); - } else { - abbrev = '\0'; - switch (ch) { - case '\\': - case '"': - abbrev = ch; - break; - case '\f': - abbrev = 'f'; - break; - case '\n': - abbrev = 'n'; - break; - case '\r': - abbrev = 'r'; - break; - case '\t': - abbrev = 't'; - break; - } - if (abbrev != '\0') { - if (bufsz >= 2) { - *buf++ = '\\'; - *buf = abbrev; - } - return (2); + } + abbrev = '\0'; + switch (ch) { + case '\\': + case '"': + abbrev = ch; + break; + case '\f': + abbrev = 'f'; + break; + case '\n': + abbrev = 'n'; + break; + case '\r': + abbrev = 'r'; + break; + case '\t': + abbrev = 't'; + break; + } + if (abbrev != '\0') { + if (bufsz >= 2) { + *buf++ = '\\'; + *buf = abbrev; } + return (2); } } if (bufsz >= 6) { @@ -357,8 +364,8 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) *buf++ = 'u'; *buf++ = '0'; *buf++ = '0'; - *buf++ = __wt_hex[(ch & 0xf0) >> 4]; - *buf++ = __wt_hex[ch & 0x0f]; + *buf++ = __wt_hex((ch & 0xf0) >> 4); + *buf++ = __wt_hex(ch & 0x0f); } return (6); } @@ -369,11 +376,11 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) * of column names. */ void -__wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, +__wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) { WT_CURSOR_JSON *json; - const char *p, *end, *beginkey; + const char *beginkey, *end, *lparen, *p; uint32_t keycnt, nkeys; json = (WT_CURSOR_JSON *)cursor->json_private; @@ -400,8 +407,16 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, keycnt++; p++; } - json->value_names.str = p; - json->value_names.len = WT_PTRDIFF(end, p); + if ((lparen = strchr(uri, '(')) != NULL) { + /* This cursor is a projection. */ + json->value_names.str = lparen; + json->value_names.len = strlen(lparen) - 1; + WT_ASSERT((WT_SESSION_IMPL *)cursor->session, + json->value_names.str[json->value_names.len] == ')'); + } else { + json->value_names.str = p; + json->value_names.len = WT_PTRDIFF(end, p); + } if (idxconf == NULL) { if (p > beginkey) p--; @@ -413,15 +428,16 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, #define MATCH_KEYWORD(session, in, result, keyword, matchval) do { \ size_t _kwlen = strlen(keyword); \ if (strncmp(in, keyword, _kwlen) == 0 && \ - !__wt_isalnum((u_char)in[_kwlen])) { \ - in += _kwlen; \ - result = matchval; \ + !__wt_isalnum((u_char)(in)[_kwlen])) { \ + (in) += _kwlen; \ + (result) = matchval; \ } else { \ - const char *_bad = in; \ - while (__wt_isalnum((u_char)*in)) \ - in++; \ - __wt_errx(session, "unknown keyword \"%.*s\" in JSON", \ - (int)(in - _bad), _bad); \ + const char *_bad = (in); \ + while (__wt_isalnum((u_char)*(in))) \ + (in)++; \ + WT_RET_MSG(session, EINVAL, \ + "unknown keyword \"%.*s\" in JSON", \ + (int)((in) - _bad), _bad); \ } \ } while (0) @@ -451,6 +467,7 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; int result; @@ -501,9 +518,9 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, } src++; } - if (result != 's') - __wt_errx(session, "unterminated string in JSON"); - break; + if (result == 's') + break; + WT_RET_MSG(session, EINVAL, "unterminated string in JSON"); case '-': case '0': case '1': @@ -561,15 +578,14 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, if (isalph) while (*src != '\0' && __wt_isalnum((u_char)*src)) src++; - __wt_errx(session, "unknown token \"%.*s\" in JSON", - (int)(src - bad), bad); - break; + WT_RET_MSG(session, EINVAL, + "unknown token \"%.*s\" in JSON", (int)(src - bad), bad); + /* NOTREACHED */ } + WT_ASSERT(session, result != -1); + *toklen = (size_t)(src - *tokstart); *toktype = result; - - if (result < 0) - WT_RET_MSG(session, EINVAL, "illegal token in JSON"); return (0); } @@ -580,6 +596,7 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, */ const char * __wt_json_tokname(int toktype) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { switch (toktype) { case 0: return (""); @@ -682,12 +699,13 @@ json_uint_arg(WT_SESSION_IMPL *session, const char **jstr, uint64_t *up) #define JSON_EXPECT_TOKEN_GET(session, jstr, tokval, start, sz) do { \ int __tok; \ - WT_RET(__wt_json_token((WT_SESSION *)session, jstr, &__tok, &start, &sz));\ - if (__tok != tokval) \ + WT_RET(__wt_json_token( \ + (WT_SESSION *)(session), jstr, &__tok, &(start), &(sz))); \ + if (__tok != (tokval)) \ WT_RET_MSG(session, EINVAL, \ "expected JSON %s, got %s", \ __wt_json_tokname(tokval), __wt_json_tokname(__tok)); \ - jstr = start + sz; \ + (jstr) = (start) + (sz); \ } while (0) #define JSON_EXPECT_TOKEN(session, jstr, tokval) do { \ @@ -817,6 +835,7 @@ __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, */ ssize_t __wt_json_strlen(const char *src, size_t srclen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { const char *srcend; size_t dstlen; @@ -857,8 +876,9 @@ __wt_json_strlen(const char *src, size_t srclen) * bytes. If dstlen is greater than the needed size, the result if zero padded. */ int -__wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, - const char *src, size_t srclen) +__wt_json_strncpy(WT_SESSION *wt_session, + char **pdst, size_t dstlen, const char *src, size_t srclen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; char ch, *dst; diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 21a7f674c68..e5b56aa406f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -150,18 +150,22 @@ static int __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor) { WT_CURSOR_LOG *cl; - WT_ITEM item; - uint32_t fileid, key_count, opsize, optype; + WT_DECL_RET; + uint32_t fileid, key_count, opsize, optype, raw; cl = (WT_CURSOR_LOG *)cursor; + /* Temporarily turn off raw so we can do direct cursor operations. */ + raw = F_MASK(cursor, WT_CURSTD_RAW); + F_CLR(cursor, WT_CURSTD_RAW); + /* * If it is a commit and we have stepped over the header, peek to get * the size and optype and read out any key/value from this operation. */ if ((key_count = cl->step_count++) > 0) { - WT_RET(__wt_logop_read(session, + WT_ERR(__wt_logop_read(session, &cl->stepp, cl->stepp_end, &optype, &opsize)); - WT_RET(__curlog_op_read(session, cl, optype, opsize, &fileid)); + WT_ERR(__curlog_op_read(session, cl, optype, opsize, &fileid)); /* Position on the beginning of the next record part. */ cl->stepp += opsize; } else { @@ -181,39 +185,14 @@ __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor) * The log cursor sets the LSN and step count as the cursor key and * and log record related data in the value. The data in the value * contains any operation key/value that was in the log record. - * For the special case that the caller needs the result in raw form, - * we create packed versions of the key/value. */ - if (FLD_ISSET(cursor->flags, WT_CURSTD_RAW)) { - memset(&item, 0, sizeof(item)); - WT_RET(wiredtiger_struct_size((WT_SESSION *)session, - &item.size, WT_LOGC_KEY_FORMAT, cl->cur_lsn->l.file, - cl->cur_lsn->l.offset, key_count)); - WT_RET(__wt_realloc(session, NULL, item.size, &cl->packed_key)); - item.data = cl->packed_key; - WT_RET(wiredtiger_struct_pack((WT_SESSION *)session, - cl->packed_key, item.size, WT_LOGC_KEY_FORMAT, - cl->cur_lsn->l.file, cl->cur_lsn->l.offset, key_count)); - __wt_cursor_set_key(cursor, &item); - - WT_RET(wiredtiger_struct_size((WT_SESSION *)session, - &item.size, WT_LOGC_VALUE_FORMAT, cl->txnid, cl->rectype, - optype, fileid, cl->opkey, cl->opvalue)); - WT_RET(__wt_realloc(session, NULL, item.size, - &cl->packed_value)); - item.data = cl->packed_value; - WT_RET(wiredtiger_struct_pack((WT_SESSION *)session, - cl->packed_value, item.size, WT_LOGC_VALUE_FORMAT, - cl->txnid, cl->rectype, optype, fileid, cl->opkey, - cl->opvalue)); - __wt_cursor_set_value(cursor, &item); - } else { - __wt_cursor_set_key(cursor, cl->cur_lsn->l.file, - cl->cur_lsn->l.offset, key_count); - __wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype, - fileid, cl->opkey, cl->opvalue); - } - return (0); + __wt_cursor_set_key(cursor, cl->cur_lsn->l.file, cl->cur_lsn->l.offset, + key_count); + __wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype, fileid, + cl->opkey, cl->opvalue); + +err: F_SET(cursor, raw); + return (ret); } /* @@ -264,17 +243,19 @@ __curlog_search(WT_CURSOR *cursor) WT_DECL_RET; WT_LSN key; WT_SESSION_IMPL *session; - uint32_t counter, key_file, key_offset; + uint32_t counter, key_file, key_offset, raw; cl = (WT_CURSOR_LOG *)cursor; + /* Temporarily turn off raw so we can do direct cursor operations. */ + raw = F_MASK(cursor, WT_CURSTD_RAW); + F_CLR(cursor, WT_CURSTD_RAW); CURSOR_API_CALL(cursor, session, search, NULL); /* * !!! We are ignoring the counter and only searching based on the LSN. */ - WT_ERR(__wt_cursor_get_key((WT_CURSOR *)cl, - &key_file, &key_offset, &counter)); + WT_ERR(__wt_cursor_get_key(cursor, &key_file, &key_offset, &counter)); WT_SET_LSN(&key, key_file, key_offset); ret = __wt_log_scan(session, &key, WT_LOGSCAN_ONE, __curlog_logrec, cl); @@ -285,7 +266,8 @@ __curlog_search(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); -err: API_END_RET(session, ret); +err: F_SET(cursor, raw); + API_END_RET(session, ret); } /* @@ -323,7 +305,7 @@ __curlog_close(WT_CURSOR *cursor) WT_ASSERT(session, FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)); if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) - __wt_readunlock(session, conn->log->log_archive_lock); + __wt_readunlock(session, &conn->log->log_archive_lock); __wt_free(session, cl->cur_lsn); __wt_free(session, cl->next_lsn); @@ -401,7 +383,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_ERR(__wt_log_force_write(session, 1, NULL)); /* Log cursors block archiving. */ - __wt_readlock(session, log->log_archive_lock); + __wt_readlock(session, &log->log_archive_lock); F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); if (0) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index 10e2fdf28be..fbfc73956e2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -16,7 +16,7 @@ WT_CURSOR_NEEDKEY(cursor); \ WT_ERR(__wt_buf_set(session, \ &((WT_CURSOR_METADATA *)(cursor))->file_cursor->key, \ - cursor->key.data, cursor->key.size)); \ + (cursor)->key.data, (cursor)->key.size)); \ F_SET(((WT_CURSOR_METADATA *)(cursor))->file_cursor, \ WT_CURSTD_KEY_EXT); \ } while (0) @@ -25,7 +25,7 @@ WT_CURSOR_NEEDVALUE(cursor); \ WT_ERR(__wt_buf_set(session, \ &((WT_CURSOR_METADATA *)(cursor))->file_cursor->value, \ - cursor->value.data, cursor->value.size)); \ + (cursor)->value.data, (cursor)->value.size)); \ F_SET(((WT_CURSOR_METADATA *)(cursor))->file_cursor, \ WT_CURSTD_VALUE_EXT); \ } while (0) diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index 5fde64c74ca..0bff642370d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -163,7 +163,6 @@ static void __curstat_set_value(WT_CURSOR *cursor, ...) { WT_UNUSED(cursor); - return; } /* @@ -478,8 +477,8 @@ __curstat_join_desc(WT_CURSOR_STAT *cst, int slot, const char **resultp) len = strlen("join: ") + strlen(sgrp->desc_prefix) + strlen(static_desc) + 1; WT_RET(__wt_realloc(session, NULL, len, &cst->desc_buf)); - snprintf(cst->desc_buf, len, "join: %s%s", sgrp->desc_prefix, - static_desc); + WT_RET(__wt_snprintf( + cst->desc_buf, len, "join: %s%s", sgrp->desc_prefix, static_desc)); *resultp = cst->desc_buf; return (0); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 6264de89df9..99a9e373354 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -144,6 +144,7 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor) */ int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) + WT_GCC_FUNC_ATTRIBUTE((cold)) { WT_SESSION_IMPL *session; @@ -632,6 +633,7 @@ __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) { + WT_DECL_RET; WT_ITEM key; /* @@ -661,9 +663,11 @@ __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) * cursors cannot reference application memory after cursor operations * and that requirement will save the day. */ - WT_RET(cursor->search(cursor)); + F_SET(cursor, WT_CURSTD_RAW_SEARCH); + ret = cursor->search(cursor); + F_CLR(cursor, WT_CURSTD_RAW_SEARCH); - return (0); + return (ret); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 502d93639a8..f6855172e90 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -14,8 +14,8 @@ static int __curtable_update(WT_CURSOR *cursor); #define APPLY_CG(ctable, f) do { \ WT_CURSOR **__cp; \ u_int __i; \ - for (__i = 0, __cp = ctable->cg_cursors; \ - __i < WT_COLGROUPS(ctable->table); \ + for (__i = 0, __cp = (ctable)->cg_cursors; \ + __i < WT_COLGROUPS((ctable)->table); \ __i++, __cp++) \ WT_TRET((*__cp)->f(*__cp)); \ } while (0) @@ -511,9 +511,16 @@ __curtable_insert(WT_CURSOR *cursor) */ F_SET(primary, flag_orig | WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); - if (ret == WT_DUPLICATE_KEY && F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + if (ret == WT_DUPLICATE_KEY && F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { WT_ERR(__curtable_update(cursor)); - else { + + /* + * The cursor is no longer positioned. This isn't just cosmetic, + * without a reset, iteration on this cursor won't start at the + * beginning/end of the table. + */ + APPLY_CG(ctable, reset); + } else { WT_ERR(ret); for (i = 1; i < WT_COLGROUPS(ctable->table); i++, cp++) { @@ -601,22 +608,53 @@ err: CURSOR_UPDATE_API_END(session, ret); static int __curtable_remove(WT_CURSOR *cursor) { + WT_CURSOR *primary; WT_CURSOR_TABLE *ctable; WT_DECL_RET; WT_SESSION_IMPL *session; + bool positioned; ctable = (WT_CURSOR_TABLE *)cursor; JOINABLE_CURSOR_REMOVE_API_CALL(cursor, session, NULL); WT_ERR(__curtable_open_indices(ctable)); + /* Check if the cursor was positioned. */ + primary = *ctable->cg_cursors; + positioned = F_ISSET(primary, WT_CURSTD_KEY_INT); + /* Find the old record so it can be removed from indices */ if (ctable->table->nindices > 0) { APPLY_CG(ctable, search); + if (ret == WT_NOTFOUND) + goto notfound; WT_ERR(ret); WT_ERR(__apply_idx(ctable, offsetof(WT_CURSOR, remove), false)); } APPLY_CG(ctable, remove); + if (ret == WT_NOTFOUND) + goto notfound; + WT_ERR(ret); + +notfound: + /* + * If the cursor is configured to overwrite and the record is not found, + * that is exactly what we want. + */ + if (ret == WT_NOTFOUND && F_ISSET(primary, WT_CURSTD_OVERWRITE)) + ret = 0; + + /* + * If the cursor was positioned, it stays positioned with a key but no + * no value, otherwise, there's no position, key or value. This isn't + * just cosmetic, without a reset, iteration on this cursor won't start + * at the beginning/end of the table. + */ + F_CLR(primary, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (positioned) + F_SET(primary, WT_CURSTD_KEY_INT); + else + APPLY_CG(ctable, reset); err: CURSOR_UPDATE_API_END(session, ret); return (ret); @@ -763,16 +801,13 @@ err: API_END_RET(session, ret); static int __curtable_complete(WT_SESSION_IMPL *session, WT_TABLE *table) { - WT_DECL_RET; bool complete; if (table->cg_complete) return (0); /* If the table is incomplete, wait on the table lock and recheck. */ - complete = false; - WT_WITH_TABLE_LOCK(session, ret, complete = table->cg_complete); - WT_RET(ret); + WT_WITH_TABLE_READ_LOCK(session, complete = table->cg_complete); if (!complete) WT_RET_MSG(session, EINVAL, "'%s' not available until all column groups are created", @@ -908,6 +943,12 @@ __wt_curtable_open(WT_SESSION_IMPL *session, table->cgroups[0]->source, NULL, cfg, cursorp); __wt_schema_release_table(session, table); + if (ret == 0) { + /* Fix up the public URI to match what was passed in. */ + cursor = *cursorp; + __wt_free(session, cursor->uri); + WT_TRET(__wt_strdup(session, uri, &cursor->uri)); + } return (ret); } @@ -927,7 +968,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { WT_ERR(__wt_struct_reformat(session, table, - columns, strlen(columns), NULL, true, tmp)); + columns, strlen(columns), NULL, false, tmp)); WT_ERR(__wt_strndup( session, tmp->data, tmp->size, &cursor->value_format)); @@ -954,7 +995,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) __wt_json_column_init( - cursor, table->key_format, NULL, &table->colconf); + cursor, uri, table->key_format, NULL, &table->colconf); /* * Open the colgroup cursors immediately: we're going to need them for @@ -992,11 +1033,15 @@ __wt_curtable_open(WT_SESSION_IMPL *session, if (0) { err: if (*cursorp != NULL) { - if (*cursorp != cursor) - WT_TRET(__wt_cursor_close(*cursorp)); + /* + * When a dump cursor is opened, then *cursorp, not + * cursor, is the dump cursor. Close the dump cursor, + * and the table cursor will be closed as its child. + */ + cursor = *cursorp; *cursorp = NULL; } - WT_TRET(__curtable_close(cursor)); + WT_TRET(cursor->close(cursor)); } __wt_scr_free(session, &tmp); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 69e9716b425..3d8c46962f1 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -216,11 +216,19 @@ ALIASES = "notyet{1}=Note: "\1" not yet supported in Wired "hrow{3}=\1\2\3" \ "hrow{4}=\1\2\3\4" \ "hrow{5}=\1\2\3\4\5" \ + "hrow{6}=\1\2\3\4\5\6" \ + "hrow{7}=\1\2\3\4\5\6\7" \ + "hrow{8}=\1\2\3\4\5\6\7\8" \ + "hrow{9}=\1\2\3\4\5\6\7\8\9" \ "row{1}=\1" \ "row{2}=\1\2" \ "row{3}=\1\2\3" \ "row{4}=\1\2\3\4" \ "row{5}=\1\2\3\4\5" \ + "row{6}=\1\2\3\4\5\6" \ + "row{7}=\1\2\3\4\5\6\7" \ + "row{8}=\1\2\3\4\5\6\7\8" \ + "row{9}=\1\2\3\4\5\6\7\8\9" \ "configstart{2}=@param config\n Configuration string, see @ref config_strings. Permitted values:\n @hrow{Name,Effect,Values}" \ "config{3}= @row{\1,\2,\3}" \ "configend=
" \ diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 0f5c56d25ce..df52324f8f8 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -36,6 +36,39 @@ the user wants to force recovery on any command, use the \c -R option. In general, commands that modify the database or tables will run recovery by default and commands that only read data will not run recovery. +
+@section util_alter wt alter +Alter a table. + +@subsection util_alter_synopsis Synopsis +wt [-RVv] [-C config] [-E secretkey ] [-h directory] alter uri configuration ... + +The \c uri and \c configuration pairs may be specified to the +\c alter command. These configuration pairs can be used to modify the +configuration values from those passed to the WT_SESSION::create +call. + +The \c uri part of the configuration pair should match only one of the +objects being altered, but may be a prefix of the object being matched. +For example, the following two sets of configuration pairs are +equivalent in the case of altering a single table named \c xxx. + +@code +table access_pattern_hint=sequential +table:xxx access_pattern_hint=sequential +@endcode + +It's an error, however, to specify a matching prefix that matches more +than a single object being altered. + +Multiple \c configuration arguments may be specified. For example, the +following two sets of configuration pairs are equivalent: + +@code +table:xxx access_pattern_hint=random,cache_resident=false +table:xxx access_pattern_hint=random table:xxx cache_resident=false +@endcode +
@section util_backup wt backup Perform a backup of a database or set of data sources. @@ -336,6 +369,19 @@ The following are command-specific options for the \c stat command: Include only "fast" statistics in the output (equivalent to passing statistics=(fast)) to WT_SESSION::open_cursor. +
+@section util_truncate wt truncate +Truncate a table, removing all data. + +The \c truncate command truncates the specified \c uri. It is equivalent to a +call to WT_SESSION::truncate with no start or stop specified. + +@subsection util_truncate_synopsis Synopsis +wt [-RVv] [-C config] [-E secretkey ] [-h directory] truncate uri + +@subsection util_truncate_options Options +The \c truncate command has no command-specific options. +
@section util_upgrade wt upgrade Upgrade a table. diff --git a/src/third_party/wiredtiger/src/docs/cursor-ops.dox b/src/third_party/wiredtiger/src/docs/cursor-ops.dox index b743d81db57..e479ff29191 100644 --- a/src/third_party/wiredtiger/src/docs/cursor-ops.dox +++ b/src/third_party/wiredtiger/src/docs/cursor-ops.dox @@ -145,9 +145,5 @@ that may not be modified or freed by the application. If a longer scope is required, the application must make a copy of the memory before the cursor is re-used, closed or reset. -The comments in this example code explain when the application can safely -modify memory passed to WT_CURSOR::set_key or WT_CURSOR::set_value: - -@snippet ex_scope.c cursor scope operation @m_endif */ diff --git a/src/third_party/wiredtiger/src/docs/cursor-random.dox b/src/third_party/wiredtiger/src/docs/cursor-random.dox index a0a3212be6d..b6434e3d161 100644 --- a/src/third_party/wiredtiger/src/docs/cursor-random.dox +++ b/src/third_party/wiredtiger/src/docs/cursor-random.dox @@ -20,9 +20,4 @@ cursor configured using \c next_random_sample_size divides the object into \c next_random_sample_size pieces, and each subsequent retrieval returns a record from the next one of those pieces. -For example, setting \c next_random_sample_percent to \c 10 would cause -the cursor to sequentially return records from each tenth part of the -object. Setting \c next_random_sample_percent to \c 1000 would cause the -cursor to sequentially return records from each .1% of the object. - */ diff --git a/src/third_party/wiredtiger/src/docs/file-formats.dox b/src/third_party/wiredtiger/src/docs/file-formats.dox index d8990aca7a6..21dc4580bc2 100644 --- a/src/third_party/wiredtiger/src/docs/file-formats.dox +++ b/src/third_party/wiredtiger/src/docs/file-formats.dox @@ -110,7 +110,7 @@ considered. (See @subpage_single huffman for details.) compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @x_ref compression_formats for details.) +considered. (See @x_ref compression_considerations for details.) Block compression is disabled by default. @@ -146,7 +146,7 @@ Huffman encoding can be high, and should be considered. compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @x_ref compression_formats for details.) +considered. (See @x_ref compression_considerations for details.) Block compression is disabled by default. @@ -157,7 +157,7 @@ compression: block compression. compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @x_ref compression_formats for details.) +considered. (See @x_ref compression_considerations for details.) Block compression is disabled by default. diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index 81e612e8ee8..aa76bef4614 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -66,14 +66,13 @@ each of which is ordered by one or more columns. - @subpage_single wtstats

- @subpage_single tune_memory_allocator -- @subpage_single tune_page_sizes +- @subpage_single tune_page_size_and_comp - @subpage_single tune_cache - @subpage_single tune_bulk_load - @subpage_single tune_cursor_persist - @subpage_single tune_read_only - @subpage_single tune_durability - @subpage_single tune_checksum -- @subpage_single tune_compression - @subpage_single tune_file_alloc - @subpage_single tune_system_buffer_cache - @subpage_single tune_transparent_huge_pages diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 2413cbc93fb..bc2e16b1122 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -50,6 +50,7 @@ LDFLAGS LIBS LLVM LOGREC +LRU LRVv LSB LSM @@ -167,6 +168,7 @@ dNLen dNOff dT dataN +database's dataitem dataset datasets diff --git a/src/third_party/wiredtiger/src/docs/top/main.dox b/src/third_party/wiredtiger/src/docs/top/main.dox index 08f7536d540..84487c13174 100644 --- a/src/third_party/wiredtiger/src/docs/top/main.dox +++ b/src/third_party/wiredtiger/src/docs/top/main.dox @@ -6,12 +6,12 @@ WiredTiger is an high performance, scalable, production quality, NoSQL, @section releases Releases -@row{WiredTiger 2.8.0 (current), - [Release package], - [Documentation]} -@row{WiredTiger 2.7.0 (current), - [Release package], - [Documentation]} +@row{WiredTiger 2.9.1 (current), + [Release package], + [Documentation]} +@row{WiredTiger 2.9.0 (previous), + [Release package], + [Documentation]} @row{Development branch, [Source code], [Documentation]} diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index bbbd2d52296..3b438eda366 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -141,7 +141,7 @@ as if the transaction started at the time of the WT_SESSION::snapshot call that created the snapshot. Named snapshots keep data pinned in cache as if a real transaction were -running for the time that the named transaction is active. The resources +running for the time that the named snapshot is active. The resources associated with named snapshots should be released by calling WT_SESSION::snapshot with a configuration that includes "drop=". See WT_SESSION::snapshot documentation for details of diff --git a/src/third_party/wiredtiger/src/docs/tune-compression.dox b/src/third_party/wiredtiger/src/docs/tune-compression.dox deleted file mode 100644 index 8db2151aa76..00000000000 --- a/src/third_party/wiredtiger/src/docs/tune-compression.dox +++ /dev/null @@ -1,62 +0,0 @@ -/*! @page tune_compression Compression - -WiredTiger includes a number of optional compression techniques. Configuring -compression generally decreases on-disk and in-memory resource requirements -and the amount of I/O, and increases CPU cost when data are read and written. - -Configuring compression may change application throughput. For example, -in applications using solid-state drives (where I/O is less expensive), -turning off compression may increase application performance by reducing -CPU costs; in applications where I/O costs are more expensive, turning on -compression may increase application performance by reducing the overall -number of I/O operations. - -An example of turning on row-store key prefix compression: - -@snippet ex_all.c Configure key prefix compression on - -An example of turning on row-store or column-store dictionary compression: - -@snippet ex_all.c Configure dictionary compression on - -@section compression_formats Block Compression Formats -WiredTiger provides two methods of compressing your data when using block -compression: the raw and noraw methods. These methods change how WiredTiger -works to fit data into the blocks that are stored on disk. - -@subsection noraw_compression Noraw Compression -Noraw compression is the traditional compression model where a fixed -amount of data is given to the compression system, then turned into a -compressed block of data. The amount of data chosen to compress is the -data needed to fill the uncompressed block. Thus when compressed, the block will -be smaller than the normal data size and the sizes written to disk will often -vary depending on how compressible the data being stored is. Algorithms -using noraw compression include zlib-noraw, lz4-noraw and snappy. - -@subsection raw_compression Raw Compression -WiredTiger's raw compression takes advantage of compressors that provide a -streaming compression API. Using the streaming API WiredTiger will try to fit -as much data as possible into one block. This means that blocks created -with raw compression should be of similar size. Using a streaming compression -method should also make for less overhead in compression, as the setup and -initial work for compressing is done fewer times compared to the amount of -data stored. Algorithms using raw compression include zlib, lz4. - -@subsection to_raw_or_noraw Choosing between Raw and Noraw Compression -When looking at which compression method to use the biggest consideration is -that raw compression will normally provide higher compression levels while -using more CPU for compression. - -An additional consideration is that raw compression may provide a performance -advantage in workloads where data is accessed sequentially. That is because -more data is generally packed into each block on disk. Conversely, noraw -compression may perform better for workloads with random access patterns -because each block will tend to be smaller and require less work to read and -decompress. - -See @ref file_formats_compression for more information on available -compression techniques. - -See @ref compression for information on how to configure and enable compression. - - */ diff --git a/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox new file mode 100644 index 00000000000..96b0fda2333 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox @@ -0,0 +1,426 @@ +/*! @page tune_page_size_and_comp Tuning page size and compression + +This document aims to explain the role played by different page sizes in +WiredTiger. It also details motivation behind an application wanting to modify +these page sizes from their default values and the procedure to do so. +Applications commonly configure page sizes based on their workload's typical key +and value size. Once a page size has been chosen, appropriate defaults for the +other configuration values are derived by WiredTiger from the page sizes, and +relatively few applications will need to modify the other page and key/value +size configuration options. WiredTiger also offers several compression options +that have an impact on the size of the data both in-memory and on-disk. Hence +while selecting page sizes, an application must also look at its desired +compression needs. Since the data and workload for a table differs from one +table to another in the database, an application can choose to set page sizes +and compression options on a per-table basis. + +@section data_life_cycle Data life cycle +Before detailing each page size, here is a review of how data gets stored inside +WiredTiger: + - WiredTiger uses the physical disks to store data durably, creating on-disk +files for the tables in the database directory. It also caches the portion of +the table being currently accessed by the application for reading or writing in +main memory. + - WiredTiger maintains a table's data in memory using a data structure called a +B-Tree ( +B+ Tree to be specific), +referring to the nodes of a B-Tree as pages. Internal pages carry only keys. The +leaf pages store both keys and values. + - The format of the in-memory pages is not the same as the format of the +on-disk pages. Therefore, the in-memory pages regularly go through a process +called reconciliation to create data structures appropriate for storage on the +disk. These data structures are referred to as on-disk pages. An application can +set a maximum size separately for the internal and leaf on-disk pages otherwise +WiredTiger uses a default value. If reconciliation of an in-memory page is +leading to an on-disk page size greater than this maximum, WiredTiger creates +multiple smaller on-disk pages. + - A component of WiredTiger called the Block Manager divides the on-disk pages +into smaller chunks called blocks, which then get written to the disk. The size +of these blocks is defined by a parameter called allocation_size, which is the +underlying unit of allocation for the file the data gets stored in. An +application might choose to have data compressed before it gets stored to disk +by enabling block compression. + - A database's tables are usually much larger than the main memory available. +Not all of the data can be kept in memory at any given time. A process called +eviction takes care of making space for new data by freeing the memory of data +infrequently accessed. An eviction server regularly finds in-memory pages that +have not been accessed in a while (following an LRU algorithm). Several +background eviction threads continuously process these pages, reconcile them to +disk and remove them from the main memory. + - When an application does an insert or an update of a key/value pair, the +associated key is used to refer to an in-memory page. In the case of this page +not being in memory, appropriate on-disk page(s) are read and an in-memory page +constructed (the opposite of reconciliation). A data structure is maintained on +every in-memory page to store any insertions or modifications to the data done +on that page. As more and more data gets written to this page, the page's memory +footprint keeps growing. + - An application can choose to set the maximum size a page is allowed to grow +in-memory. A default size is set by WiredTiger if the application doesn't +specify one. To keep page management efficient, as a page grows larger in-memory +and approaches this maximum size, if possible, it is split into smaller +in-memory pages. + - When doing an insert or an update, if a page grows larger than the maximum, +the application thread is used to forcefully evict this page. This is done to +split the growing page into smaller in-memory pages and reconcile them into +on-disk pages. Once written to the disk they are removed from the main memory, +making space for more data to be written. When an application gets involved in +forced eviction, it might take longer than usual to do these inserts and +updates. It is not always possible to (force) evict a page from memory and this +page can temporarily grow larger in size than the configured maximum. This page +then remains marked to be evicted and reattempts are made as the application +puts more data in it. + +@section configurable_page_struct Configurable page structures in WiredTiger +There are three page sizes that the user can configure: + 1. The maximum page size of any type of in-memory page in the WiredTiger cache, +memory_page_max. + 2. The maximum size of the on-disk page for an internal page, internal_page_max. + 3. The maximum size of the on-disk leaf page, leaf_page_max. + +There are additional configuration settings that tune more esoteric and +specialized data. Those are included for completeness but are rarely changed. + +@subsection memory_page_max memory_page_max +The maximum size a table's page is allowed to grow to in memory before being +reconciled to disk. + - An integer, with acceptable values between 512B and 10TB + - Default size: 5 MB + - Additionally constrained by the condition: + leaf_page_max <= memory_page_max <= cache_size/10 + - Motivation to tune the value: +\n memory_page_max is significant for applications wanting to tune for +consistency in write intensive workloads. + - This is the parameter to start with for tuning and trying different values +to find the correct balance between overall throughput and individual operation +latency for each table. + - Splitting a growing in-memory page into smaller pages and reconciliation +both require exclusive access to the page which makes an application's write +operations wait. Having a large memory_page_max means that the pages will need +to be split and reconciled less often. But when that happens, the duration that +an exclusive access to the page is required is longer, increasing the latency of +an application's insert or update operations. Conversely, having a smaller +memory_page_max reduces the time taken for splitting and reconciling the pages, +but causes it to happen more frequently, forcing more frequent but shorter +exclusive accesses to the pages. + - Applications should choose the memory_page_max value considering the +trade-off between frequency of exclusive access to the pages (for reconciliation +or splitting pages into smaller pages) versus the duration that the exclusive +access is required. + - Configuration: +\n Specified as memory_page_max configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,memory_page_max=10MB"
+
+ +@subsection internal_page_max internal_page_max +The maximum page size for the reconciled on-disk internal pages of the B-Tree, +in bytes. When an internal page grows past this size, it splits into multiple +pages. + - An integer, with acceptable values between 512B and 512MB + - Default size: 4 KB (*appropriate for applications with relatively small keys) + - Additionally constrained by the condition: the size must be a multiple of the +allocation size + - Motivation to tune the value: +\n internal_page_max is significant for applications wanting to avoid excessive +L2 cache misses while searching the tree. + - Recall that only keys are stored on internal pages, so the type and size of +the key values for a table help drive the setting for this parameter. + - Should be sized to fit into on-chip caches. + - Applications doing full-table scans with out-of-memory workloads might +increase internal_page_max to transfer more data per I/O. + - Influences the shape of the B-Tree, i.e. depth and the number of children +each page in B-Tree has. To iterate to the desired key/value pair in the B-Tree, +WiredTiger has to binary search the key-range in a page to determine the child +page to proceed to and continue down the depth until it reaches the correct leaf +page. Having an unusually deep B-Tree, or having too many children per page can +negatively impact time taken to iterate the B-Tree, slowing down the application. +The number of children per page and, hence, the tree depth depends upon the +number of keys that can be stored in an internal page, which is +internal_page_max divided by key size. Applications should choose an appropriate +internal_page_max size that avoids the B-Tree from getting too deep. + - Configuration: +\n Specified as internal_page_max configuration option to WT_SESSION::create(). +An example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,internal_page_max=16KB,leaf_page_max=1MB"
+
+ +@subsection leaf_page_max leaf_page_max +The maximum page size for the reconciled on-disk leaf pages of the B-Tree, in +bytes. When a leaf page grows past this size, it splits into multiple pages. + - An integer, with acceptable values between 512B and 512MB + - Default size: 32 KB (*appropriate for applications with relatively small keys +and values) + - Additionally constrained by the condition: must be a multiple of the +allocation size + - Motivation to tune the value: +\n leaf_page_max is significant for applications wanting to maximize sequential +data transfer from a storage device. + - Should be sized to maximize I/O performance (when reading from disk, it is +usually desirable to read a large amount of data, assuming some locality of +reference in the application's access pattern). + - Applications doing full-table scans through out-of-cache workloads might +increase leaf_page_max to transfer more data per I/O. + - Applications focused on read/write amplification might decrease the page +size to better match the underlying storage block size. + - Configuration: +\n Specified as leaf_page_max configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,internal_page_max=16KB,leaf_page_max=1MB"
+
+ +The following configuration items following are rarely used. They are described +for completeness: + +@subsection allocation_size allocation_size +This is the underlying unit of allocation for the file. As the unit of file +allocation, it sets the minimum page size and how much space is wasted when +storing small amounts of data and overflow items. + - an integer between 512B and 128 MB + - must a power-of-two + - default : 4 KB + - Motivation to tune the value: +\n Most applications should not need to tune the allocation size. + - To be compatible with virtual memory page sizes and direct I/O requirements +on the platform (4KB for most common server platforms) + - Smaller values decrease the file space required by overflow items. + - For example, if the allocation size is set to 4KB, an overflow item of +18,000 bytes requires 5 allocation units and wastes about 2KB of space. If the +allocation size is 16KB, the same overflow item would waste more than 10KB. + - Configuration: +\n Specified as allocation_size configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,allocation_size=4KB"
+
+ +@subsection key_val_max internal/leaf key/value max + - Overflow items +\n Overflow items are keys and values too large to easily store on a page. Overflow +items are stored separately in the file from the page where the item logically +appears, and so reading or writing an overflow item is more expensive than an +on-page item, normally requiring additional I/O. Additionally, overflow values +are not cached in memory. This means overflow items won't affect the caching +behavior of the application. It also means that each time an overflow value is +read, it is re-read from disk. + - internal_key_max +\n The largest key stored in an internal page, in bytes. If set, keys larger than +the specified size are stored as overflow items. + - The default and the maximum allowed value are both one-tenth the size of a +newly split internal page. + - leaf_key_max +\n The largest key stored in a leaf page, in bytes. If set, keys larger than the +specified size are stored as overflow items. + - The default value is one-tenth the size of a newly split leaf page. + - leaf_value_max +\n The largest value stored in a leaf page, in bytes. If set, values larger than +the specified size are stored as overflow items + - The default is one-half the size of a newly split leaf page. + - If the size is larger than the maximum leaf page size, the page size is +temporarily ignored when large values are written. + - Motivation to tune the values: +\n Most applications should not need to tune the maximum key and value sizes. +Applications requiring a small page size, but also having latency concerns such +that the additional work to retrieve an overflow item may find modifying these +values useful. +\n Since overflow items are separately stored in the on-disk file, aren't cached +and require additional I/O to access (read or write), applications should avoid +creating overflow items. + - Since page sizes also determine the default size of overflow items, i.e., +keys and values too large to easily store on a page, they can be configured to +avoid performance penalties working with overflow items: + - Applications with large keys and values, and concerned with latency, +might increase the page size to avoid creating overflow items, in order to avoid +the additional cost of retrieving them. + - Applications with large keys and values, doing random searches, might +decrease the page size to avoid wasting cache space on overflow items that +aren't likely to be needed. + - Applications with large keys and values, doing table scans, might +increase the page size to avoid creating overflow items, as the overflow items +must be read into memory in all cases, anyway. + - internal_key_max, leaf_key_max and leaf_value_max configuration values +allow applications to change the size at which a key or value will be treated +as an overflow item. + - Most applications should not need to tune the maximum key and value +sizes. + - The value of internal_key_max is relative to the maximum internal page +size. Because the number of keys on an internal page determines the depth of the +tree, the internal_key_max value can only be adjusted within a certain range, +and the configured value will be automatically adjusted by WiredTiger, if +necessary, to ensure a reasonable number of keys fit on an internal page. + - The values of leaf_key_max and leaf_value_max are not relative to the +maximum leaf page size. If either is larger than the maximum page size, the page +size will be ignored when the larger keys and values are being written, and a +larger page will be created as necessary. + - Configuration: +\n Specified as internal_key_max, leaf_key_max and leaf_value_max configuration +options to WT_SESSION::create(). An example of configuration string for a large +leaf overflow value: + +
+     "key_format=S,value_format=S,leaf_page_max=16KB,leaf_value_max=256KB"
+
+ +@subsection split_pct split_pct (split percentage) +The size (specified as percentage of internal/leaf page_max) at which the +reconciled page must be split into multiple smaller pages before being sent for +compression and then be written to the disk. If the reconciled page can fit into +a single on-disk page without the page growing beyond it's set max size, +split_pct is ignored and the page isn't split. + - an integer between 25 and 100 + - default : 75 + - Motivation to tune the value: +\n Most applications should not need to tune the split percentage size. + - This value should be selected to avoid creating a large number of tiny +pages or repeatedly splitting whenever new entries are inserted. +\n For example, if the maximum page size is 1MB, a split_pct value of 10% +would potentially result in creating a large number of 100KB pages, which may +not be optimal for future I/O. Or, if the maximum page size is 1MB, a split_pct +value of 90% would potentially result in repeatedly splitting pages as the split +pages grow to 1MB over and over. The default value for split_pct is 75%, +intended to keep large pages relatively large, while still giving split pages +room to grow. + - Configuration: +\n Specified as split_pct configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,split_pct=60"
+
+ +@section compression_considerations Compression considerations +WiredTiger compresses data at several stages to preserve memory and disk space. +Applications can configure these different compression algorithms to tailor +their requirements between memory, disk and CPU consumption. Compression +algorithms other than block compression work by modifying how the keys and +values are represented, and hence reduce data size in-memory and on-disk. Block +compression on the other hand compress the data in its binary representation +while saving it on the disk. + +Configuring compression may change application throughput. For example, in +applications using solid-state drives (where I/O is less expensive), turning +off compression may increase application performance by reducing CPU costs; in +applications where I/O costs are more expensive, turning on compression may +increase application performance by reducing the overall number of I/O +operations. + +WiredTiger uses some internal algorithms to compress the amount of data stored +that are not configurable, but always on. For example, run-length reduces the +size requirement by storing sequential, duplicate values in the store only a +single time (with an associated count). + +Different compression options available with WiredTiger: + - Key-prefix + - Reduces the size requirement by storing any identical key prefix only once +per page. The cost is additional CPU and memory when operating on the in-memory +tree. Specifically, reverse sequential cursor movement (but not forward) through +a prefix-compressed page or the random lookup of a key/value pair will allocate +sufficient memory to hold some number of uncompressed keys. So, for example, if +key prefix compression only saves a small number of bytes per key, the +additional memory cost of instantiating the uncompressed key may mean prefix +compression is not worthwhile. Further, in cases where the on-disk cost is the +primary concern, block compression may mean prefix compression is less useful. + - Configuration: +\n Specified as prefix_compression configuration option to +WT_SESSION::create(). Applications may limit the use of prefix compression by +configuring the minimum number of bytes that must be gained before prefix +compression is used with prefix_compression_min configuration option. An example +of such a configuration string is as follows: + +
+          "key_format=S,value_format=S,prefix_compression=true,prefix_compression_min=7"
+
+ + - Dictionary + - Reduces the size requirement by storing any identical value only once per +page. + - Configuration: +\n Specified as dictionary configuration configuration option to +WT_SESSION::create(), which specifies the maximum number of unique values +remembered in the B-Tree row-store leaf page value dictionary. An example of +such a configuration string is as follows: + +
+          "key_format=S,value_format=S,dictionary=1000"
+
+ + - Huffman + - Reduces the size requirement by compressing individual key/value items, and +can be separately configured either or both keys and values. The additional CPU +cost of Huffman encoding can be high, and should be considered. (See Huffman +Encoding for details.) + - Configuration: +\n Specified as huffman_key and/or huffman_value configuration option to +WT_SESSION::create(). These options can take values of "english" (to use a +built-in English language frequency table), "utf8" or "utf16" (to +use a custom utf8 or utf16 symbol frequency table file). An example of such a +configuration string is as follows: + +
+          "key_format=S,value_format=S,huffman_key=english,huffman_value=english"
+
+ + - Block Compression + - Reduces the size requirement of on-disk objects by compressing blocks of +the backing object's file. The additional CPU cost of block compression can be +high, and should be considered. When block compression has been configured, +configured page sizes will not match the actual size of the page on disk. + - WiredTiger provides two methods of compressing your data when using block +compression: the raw and noraw methods. These methods change how WiredTiger +works to fit data into the blocks that are stored on disk. Applications needing +to write specific sized blocks may want to consider implementing a +WT_COMPRESSOR::compress_raw function. + - Noraw compression: +\n A fixed amount of data is given to the compression system, then turned into +a compressed block of data. The amount of data chosen to compress is the data +needed to fill the uncompressed block. Thus when compressed, the block will be +smaller than the normal data size and the sizes written to disk will often vary +depending on how compressible the data being stored is. Algorithms using noraw +compression include zlib-noraw, lz4-noraw and snappy. +Noraw compression is better suited for workloads with random access patterns +because each block will tend to be smaller and require less work to read and +decompress. + - Raw compression: +\n WiredTiger's raw compression takes advantage of compressors that provide a +streaming compression API. Using the streaming API WiredTiger will try to fit as +much data as possible into one block. This means that blocks created with raw +compression should be of similar size. Using a streaming compression method +should also make for less overhead in compression, as the setup and initial work +for compressing is done fewer times compared to the amount of data stored. +Algorithms using raw compression include zlib, lz4. +Compared to noraw, raw compression provides more compression while using more +CPU. Raw compression may provide a performance advantage in workloads where data +is accessed sequentially. That is because more data is generally packed into +each block on disk. + - Configuration: +\n Specified as the block_compressor configuration option to +WT_SESSION::create(). If WiredTiger has builtin support for "lz4", "snappy", +"zlib" or "zstd" compression, these names are available as the value to the +option. An example of such a configuration string is as follows: + +
+          "key_format=S,value_format=S,block_compressor=snappy"
+
+ +See @ref compression for further information on how to configure and enable +different compression options. + +@subsection table_compress Table summarizing compression in WiredTiger + +
+@hrow{Compression Type, Supported by row-store, Supported by variable col-store, + Supported by fixed col-store, Default config, Reduces in-mem size, + Reduces on-disk size, CPU and Memory cost} +@row{Key-prefix, yes, no, no, disabled, yes, yes, minor} +@row{Dictionary, yes, yes, no, disabled, yes, yes, minor} +@row{Huffman, yes, yes, no, disabled, yes, yes, can be high} +@row{Block, yes, yes, yes, disabled, no, yes, can be high} +
+ +*/ diff --git a/src/third_party/wiredtiger/src/docs/tune-page-sizes.dox b/src/third_party/wiredtiger/src/docs/tune-page-sizes.dox deleted file mode 100644 index 130e047a02d..00000000000 --- a/src/third_party/wiredtiger/src/docs/tune-page-sizes.dox +++ /dev/null @@ -1,142 +0,0 @@ -/*! @page tune_page_sizes Page and overflow key/value sizes - -There are seven page and key/value size configuration strings: - -- allocation size (\c allocation_size), -- page sizes (\c internal_page_max and \c leaf_page_max), -- key and value sizes (\c internal_key_max, \c leaf_key_max and \c leaf_value_max), and the -- page-split percentage (\c split_pct). - -All seven are specified to the WT_SESSION::create method, in other -words, they are configurable on a per-file basis. - -Applications commonly configure page sizes, based on their workload's -typical key and value size. Once the correct page size has been chosen, -appropriate defaults for the other configuration values are derived from -the page sizes, and relatively few applications will need to modify the -other page and key/value size configuration options. - -An example of configuring page and key/value sizes: - -@snippet ex_all.c Create a table and configure the page size - -@section tune_page_sizes_sizes Page, key and value sizes - -The \c internal_page_max and \c leaf_page_max configuration values -specify a maximum size for Btree internal and leaf pages. That is, when -an internal or leaf page grows past that size, it splits into multiple -pages. Generally, internal pages should be sized to fit into on-chip -caches in order to minimize cache misses when searching the tree, while -leaf pages should be sized to maximize I/O performance (if reading from -disk is necessary, it is usually desirable to read a large amount of -data, assuming some locality of reference in the application's access -pattern). - -The default page size configurations (2KB for \c internal_page_max, 32KB -for \c leaf_page_max), are appropriate for applications with relatively -small keys and values. - -- Applications doing full-table scans through out-of-memory workloads -might increase both internal and leaf page sizes to transfer more data -per I/O. -- Applications focused on read/write amplification might decrease the page -size to better match the underlying storage block size. - -When block compression has been configured, configured page sizes will -not match the actual size of the page on disk. Block compression in -WiredTiger happens within the I/O subsystem, and so a page might split -even if subsequent compression would result in a resulting page size -small enough to leave as a single page. In other words, page sizes are -based on in-memory sizes, not on-disk sizes. Applications needing to -write specific sized blocks may want to consider implementing a -WT_COMPRESSOR::compress_raw function. - -The page sizes also determine the default size of overflow items, that -is, keys and values too large to easily store on a page. Overflow items -are stored separately in the file from the page where the item logically -appears, and so reading or writing an overflow item is more expensive -than an on-page item, normally requiring additional I/O. Additionally, -overflow values are not cached in memory. This means overflow items -won't affect the caching behavior of the application, but it also means -that each time an overflow value is read, it is re-read from disk. - -For both of these reasons, applications should avoid creating large -numbers of commonly referenced overflow items. This is especially -important for keys, as keys on internal pages are referenced during -random searches, not just during data retrieval. Generally, -applications should make every attempt to avoid creating overflow keys. - -- Applications with large keys and values, and concerned with latency, -might increase the page size to avoid creating overflow items, in order -to avoid the additional cost of retrieving them. - -- Applications with large keys and values, doing random searches, might -decrease the page size to avoid wasting cache space on overflow items -that aren't likely to be needed. - -- Applications with large keys and values, doing table scans, might -increase the page size to avoid creating overflow items, as the overflow -items must be read into memory in all cases, anyway. - -The \c internal_key_max, \c leaf_key_max and \c leaf_value_max -configuration values allow applications to change the size at which a -key or value will be treated as an overflow item. - -The value of \c internal_key_max is relative to the maximum internal -page size. Because the number of keys on an internal page determines -the depth of the tree, the \c internal_key_max value can only be -adjusted within a certain range, and the configured value will be -automatically adjusted by WiredTiger, if necessary to ensure a -reasonable number of keys fit on an internal page. - -The values of \c leaf_key_max and \c leaf_value_max are not relative to -the maximum leaf page size. If either is larger than the maximum page -size, the page size will be ignored when the larger keys and values are -being written, and a larger page will be created as necessary. - -Most applications should not need to tune the maximum key and value -sizes. Applications requiring a small page size, but also having -latency concerns such that the additional work to retrieve an overflow -item is an issue, may find them useful. - -An example of configuring a large leaf overflow value: - -@snippet ex_all.c Create a table and configure a large leaf value max - -@section tune_page_sizes_split_percentage Split percentage - -The \c split_pct configuration string configures the size of a split -page. When a page grows sufficiently large that it must be written as -multiple disk blocks, the newly written block size is \c split_pct -percent of the maximum page size. This value should be selected to -avoid creating a large number of tiny pages or repeatedly splitting -whenever new entries are inserted. For example, if the maximum page -size is 1MB, a \c split_pct value of 10% would potentially result in -creating a large number of 100KB pages, which may not be optimal for -future I/O. Or, if the maximum page size is 1MB, a \c split_pct value -of 90% would potentially result in repeatedly splitting pages as the -split pages grow to 1MB over and over. The default value for \c -split_pct is 75%, intended to keep large pages relatively large, while -still giving split pages room to grow. - -Most applications should not need to tune the split percentage size. - -@section tune_page_sizes_allocation_size Allocation size - -The \c allocation_size configuration value is the underlying unit of -allocation for the file. As the unit of file allocation, it sets the -minimum page size and how much space is wasted when storing small -amounts of data and overflow items. For example, if the allocation size -is set to 4KB, an overflow item of 18,000 bytes requires 5 allocation -units and wastes about 2KB of space. If the allocation size is 16KB, -the same overflow item would waste more than 10KB. - -The default allocation size is 4KB, chosen for compatibility with -virtual memory page sizes and direct I/O requirements on common server -platforms. - -Most applications should not need to tune the allocation size; it is -primarily intended for applications coping with the specific -requirements some file systems make to support features like direct I/O. - -*/ diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index caa6b036892..e5fce3d0d5d 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,7 +1,75 @@ /*! @page upgrading Upgrading WiredTiger applications -@section version_281 Upgrading to Version 2.8.1 +@section version_292 Upgrading to Version 2.9.2

+ +
WiredTiger utility now supports truncate
+
+The WiredTiger utility \c wt can now \c truncate objects, removing all +contents from the specified object. +
+ +
Handle list lock statistics
+
+In the 2.9.1 release we added statistics tracking handle list lock timing, we +have switched that lock from a spin lock to a read-write lock, and consequently +changed the statistics tracking lock related wait time. +
+ +
Forced and named checkpoint error conditions changed
+
+There are new cases where checkpoints created with an explicit name or the +"force" configuration option can return an EBUSY error. This can happen if +the checkpoint overlaps with other schema operations, for example table create. +
+ +
WT_CURSOR::remove may not return a positioned cursor
+
+The WT_CURSOR::remove method was previously documented to always return a +positioned cursor on success, which is not possible when \c overwrite=true +and the record does not exist. + +The documentation has been updated, and the method has been changed to +never return a cursor position unless called with an existing cursor +position. In other words, if the cursor is positioned and the +WT_CURSOR::remove is called, the cursor will remain positioned; if the +cursor is not positioned and the WT_CURSOR::remove method is called, the +cursor will not be positioned on return. +
+ +
+@section version_291 Upgrading to Version 2.9.1 +
+ +
Changes to hazard pointer configuration
+
+The \c hazard_max parameter to ::wiredtiger_open is now ignored. Memory is +allocated for hazard pointers as required by each session. +
+ +
Change to the default fadvise behavior for data files
+
+The old default behavior was to advise the file system that access would be +random for data files, and there was no way to alter that. We no longer +call advise the file system of expected access patterns by default, and +have added a new \c access_pattern_hint configuration option available for +WT_SESSION::create that can be used to restore the old default by setting +the value to "random". +
+ +
+@section version_290 Upgrading to Version 2.9.0 +
+ +
Changes to cursor behavior after WT_CURSOR::insert
+
+After a successful call to WT_CURSOR::insert, unless a cursor has record +number keys and was configured with "append=true", the cursor's key is +cleared and a subsequent call to the WT_CURSOR::get_key method will fail. +In all cursors, the value is cleared and a subsequent call to +WT_CURSOR::get_value will fail. +
+
Cache management defaults
The default values for the \c eviction_dirty_target and \c @@ -17,6 +85,15 @@ how much work is done at the beginning of a checkpoint to make the critical section of checkpoints complete more quickly.
+
Change to default fadvise setting for data files
+
+The default behavior for data files was to advise the file system to optimize +for random access on POSIX and Windows platforms. The default is now to not +advise about access patterns. There is a new \c access_pattern_hint +configuration string available to WT_SESSION::create that can be used +to configure the old default behavior. +
+
Checkpoint server created checkpoint names
The ::wiredtiger_open checkpoint configuration no longer supports the @@ -34,13 +111,13 @@ of the path may no longer be specified. Applications depending on the ability to set statistics log file names will require modification.
-
Deprecated statistics field
+
Removed and renamed statistics fields
-The connection statistic \c WT_STAT_CONN_CACHE_BYTES_OVERFLOW has been -removed. Overflow information is now available in the -\c WT_STAT_CONN_CACHE_BYTES_OVERFLOW and \c WT_STAT_CONN_CACHE_OVERFLOW_VALUE. -Applications specifically looking for that statistic will require -modification. +WT_STAT_JOIN_ACTUAL_COUNT is renamed to WT_STAT_JOIN_ITERATED + +WT_STAT_CONN_CACHE_BYTES_OVERFLOW removed since overflow pages are not cached + +WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING removed since all eviction threads can now be the server
@@ -281,7 +358,7 @@ be updated. The WT_SESSION::create \c internal_item_max and \c leaf_item_max configuration strings are now deprecated in favor of the \c internal_key_max, \c leaf_key_max, and \c leaf_value_max -configuration strings. See @ref tune_page_sizes for more information. +configuration strings. See @ref tune_page_size_and_comp for more information.
diff --git a/src/third_party/wiredtiger/src/docs/wtperf.dox b/src/third_party/wiredtiger/src/docs/wtperf.dox index 83aadf8a776..6bdcf5f4f8d 100644 --- a/src/third_party/wiredtiger/src/docs/wtperf.dox +++ b/src/third_party/wiredtiger/src/docs/wtperf.dox @@ -167,6 +167,8 @@ do population phase; false to use existing database number of WiredTiger databases to use. Each database will execute the workload using a separate home directory and complete set of worker threads @par drop_tables (boolean, default=false) Whether to drop all tables at the end of the run, and report time taken to do the drop. +@par in_memory (boolean, default=false) +Whether to create the database in-memory. @par icount (unsigned int, default=5000) number of records to initially populate. If multiple tables are configured the count is spread evenly across all tables. @par idle_table_cycle (unsigned int, default=0) @@ -195,14 +197,14 @@ use pareto distribution for random numbers. Zero to disable, otherwise a percen number of operations to group into each transaction in the populate phase, zero for auto-commit @par populate_threads (unsigned int, default=1) number of populate threads, 1 for bulk load +@par pre_load_data (boolean, default=false) +Scan all data prior to starting the workload phase to warm the cache @par random_range (unsigned int, default=0) if non zero choose a value from within this range as the key for insert operations @par random_value (boolean, default=false) generate random content for the value @par range_partition (boolean, default=false) partition data by range (vs hash) -@par read_range (unsigned int, default=0) -scan a range of keys after each search @par readonly (boolean, default=false) reopen the connection between populate and workload phases in readonly mode. Requires reopen_connection turned on (default). Requires that read be the only workload specified @par reopen_connection (boolean, default=true) @@ -228,7 +230,7 @@ number of tables to run operations over. Keys are divided evenly over the table @par table_count_idle (unsigned int, default=0) number of tables to create, that won't be populated. Default 0. @par threads (string, default="") -workload configuration: each 'count' entry is the total number of threads, and the 'insert', 'read' and 'update' entries are the ratios of insert, read and update operations done by each worker thread; If a throttle value is provided each thread will do a maximum of that number of operations per second; multiple workload configurations may be specified per threads configuration; for example, a more complex threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed configuration values are 'count', 'throttle', 'update_delta', 'reads', 'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior modifiers, supported modifiers are 'ops_per_txn' +workload configuration: each 'count' entry is the total number of threads, and the 'insert', 'read' and 'update' entries are the ratios of insert, read and update operations done by each worker thread; If a throttle value is provided each thread will do a maximum of that number of operations per second; multiple workload configurations may be specified per threads configuration; for example, a more complex threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed configuration values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', 'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior modifiers, supported modifiers are 'ops_per_txn' @par transaction_config (string, default="") WT_SESSION.begin_transaction configuration string, applied during the populate phase when populate_ops_per_txn is nonzero @par table_name (string, default="test") diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index b0cd50cc655..3d8f4a61ca7 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -15,15 +15,27 @@ int __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) { + WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; WT_REF *next_ref, *ref; + btree = S2BT(session); + /* - * We need exclusive access to the file -- disable ordinary eviction - * and drain any blocks already queued. + * We need exclusive access to the file, we're about to discard the root + * page. Assert eviction has been locked out. */ - WT_RET(__wt_evict_file_exclusive_on(session)); + WT_ASSERT(session, + btree->evict_disabled > 0 || + !F_ISSET(session->dhandle, WT_DHANDLE_OPEN)); + + /* + * We do discard objects without pages in memory. If that's the case, + * we're done. + */ + if (btree->root.page == NULL) + return (0); /* Make sure the oldest transaction ID is up-to-date. */ WT_RET(__wt_txn_update_oldest( @@ -57,7 +69,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * error, retrying later. */ if (syncop == WT_SYNC_CLOSE && __wt_page_is_modified(page)) - WT_ERR(__wt_reconcile(session, ref, NULL, WT_EVICTING)); + WT_ERR(__wt_reconcile( + session, ref, NULL, WT_EVICTING, NULL)); /* * We can't evict the page just returned to us (it marks our @@ -101,7 +114,5 @@ err: /* On error, clear any left-over tree walk. */ session, next_ref, WT_READ_NO_EVICT)); } - __wt_evict_file_exclusive_off(session); - return (ret); } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 909fe813f98..26bbf9f679b 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -15,6 +15,7 @@ static int __evict_lru_walk(WT_SESSION_IMPL *); static int __evict_page(WT_SESSION_IMPL *, bool); static int __evict_pass(WT_SESSION_IMPL *); static int __evict_server(WT_SESSION_IMPL *, bool *); +static int __evict_tune_workers(WT_SESSION_IMPL *session); static int __evict_walk(WT_SESSION_IMPL *, WT_EVICT_QUEUE *); static int __evict_walk_file( WT_SESSION_IMPL *, WT_EVICT_QUEUE *, u_int, u_int *); @@ -22,6 +23,60 @@ static int __evict_walk_file( #define WT_EVICT_HAS_WORKERS(s) \ (S2C(s)->evict_threads.current_threads > 1) +/* + * __evict_lock_handle_list -- + * Try to get the handle list lock, with yield and sleep back off. + * Keep timing statistics overall. + */ +static int +__evict_lock_handle_list(WT_SESSION_IMPL *session) +{ + struct timespec enter, leave; + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_RWLOCK *dh_lock; + u_int spins; + bool dh_stats; + + conn = S2C(session); + cache = conn->cache; + dh_lock = &conn->dhandle_lock; + + /* + * Setup tracking of handle lock acquisition wait time if statistics + * are enabled. + */ + dh_stats = WT_STAT_ENABLED(session); + + if (dh_stats) + __wt_epoch(session, &enter); + + /* + * Use a custom lock acquisition back off loop so the eviction server + * notices any interrupt quickly. + */ + for (spins = 0; + (ret = __wt_try_readlock(session, dh_lock)) == EBUSY && + cache->pass_intr == 0; spins++) { + if (spins < WT_THOUSAND) + __wt_yield(); + else + __wt_sleep(0, WT_THOUSAND); + } + /* + * Only record statistics on success. + */ + WT_RET(ret); + if (dh_stats) { + __wt_epoch(session, &leave); + WT_STAT_CONN_INCRV( + session, lock_handle_list_wait_eviction, + (int64_t)WT_TIMEDIFF_US(leave, enter)); + } + return (0); +} + /* * __evict_entry_priority -- * Get the adjusted read generation for an eviction entry. @@ -143,8 +198,7 @@ __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref) } __wt_spin_unlock(session, &cache->evict_queues[q].evict_lock); } - WT_ASSERT(session, - !F_ISSET_ATOMIC(ref->page, WT_PAGE_EVICT_LRU)); + WT_ASSERT(session, !F_ISSET_ATOMIC(ref->page, WT_PAGE_EVICT_LRU)); __wt_spin_unlock(session, &cache->evict_queue_lock); } @@ -213,7 +267,7 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) } #endif - __wt_cond_auto_signal(session, cache->evict_cond); + __wt_cond_signal(session, cache->evict_cond); } /* @@ -226,12 +280,12 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - bool did_work; + bool did_work, was_intr; conn = S2C(session); cache = conn->cache; -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* * Ensure the cache stuck timer is initialized when starting eviction. */ @@ -254,12 +308,28 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) ret = __evict_server(session, &did_work); F_CLR(cache->walk_session, WT_SESSION_LOCKED_PASS); F_CLR(session, WT_SESSION_LOCKED_PASS); + was_intr = cache->pass_intr != 0; __wt_spin_unlock(session, &cache->evict_pass_lock); WT_ERR(ret); + + /* + * If the eviction server was interrupted, wait until + * requests have been processed: the system may + * otherwise be busy so don't go to sleep. + */ + if (was_intr) { + while (cache->pass_intr != 0 && + F_ISSET(conn, WT_CONN_EVICTION_RUN) && + F_ISSET(thread, WT_THREAD_RUN)) + __wt_yield(); + continue; + } + __wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping"); + /* Don't rely on signals: check periodically. */ __wt_cond_auto_wait( - session, cache->evict_cond, did_work); + session, cache->evict_cond, did_work, NULL); __wt_verbose(session, WT_VERB_EVICTSERVER, "waking"); } else WT_ERR(__evict_lru_pages(session, false)); @@ -271,7 +341,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) * can be closed. */ if (thread->id == 0) { - WT_WITH_PASS_LOCK(session, ret, + WT_WITH_PASS_LOCK(session, ret = __evict_clear_all_walks(session)); WT_ERR(ret); /* @@ -299,14 +369,13 @@ err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); static int __evict_server(WT_SESSION_IMPL *session, bool *did_work) { +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) + struct timespec now; +#endif WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; -#ifdef HAVE_DIAGNOSTIC - struct timespec now; -#endif uint64_t orig_pages_evicted; - u_int spins; conn = S2C(session); cache = conn->cache; @@ -317,7 +386,8 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) /* Evict pages from the cache as needed. */ WT_RET(__evict_pass(session)); - if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) + if (!F_ISSET(conn, WT_CONN_EVICTION_RUN) || + cache->pass_intr != 0) return (0); /* @@ -325,41 +395,55 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) * otherwise we can block applications evicting large pages. */ if (!__wt_cache_stuck(session)) { - for (spins = 0; (ret = __wt_spin_trylock( - session, &conn->dhandle_lock)) == EBUSY && - cache->pass_intr == 0; spins++) { - if (spins < WT_THOUSAND) - __wt_yield(); - else - __wt_sleep(0, WT_THOUSAND); - } /* - * If we gave up acquiring the lock, that indicates a - * session is waiting for us to clear walks. Do that - * as part of a normal pass (without the handle list + * Try to get the handle list lock: if we give up, that + * indicates a session is waiting for us to clear walks. Do + * that as part of a normal pass (without the handle list * lock) to avoid deadlock. */ - if (ret == EBUSY) + if ((ret = __evict_lock_handle_list(session)) == EBUSY) return (0); WT_RET(ret); ret = __evict_clear_all_walks(session); - __wt_spin_unlock(session, &conn->dhandle_lock); + __wt_readunlock(session, &conn->dhandle_lock); WT_RET(ret); cache->pages_evicted = 0; } else if (cache->pages_evicted != cache->pages_evict) { cache->pages_evicted = cache->pages_evict; -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) __wt_epoch(session, &cache->stuck_ts); - } else { - /* After being stuck for 5 minutes, give up. */ + } else if (!F_ISSET(conn, WT_CONN_IN_MEMORY)) { + /* + * If we're stuck for 5 minutes in diagnostic mode, or the + * verbose evict_stuck flag is configured, log the cache + * and transaction state. + * + * If we're stuck for 5 minutes in diagnostic mode, give up. + * + * We don't do this check for in-memory workloads because + * application threads are not blocked by the cache being full. + * If the cache becomes full of clean pages, we can be + * servicing reads while the cache appears stuck to eviction. + */ __wt_epoch(session, &now); if (WT_TIMEDIFF_SEC(now, cache->stuck_ts) > 300) { - ret = ETIMEDOUT; - __wt_err(session, ret, +#if defined(HAVE_DIAGNOSTIC) + __wt_err(session, ETIMEDOUT, "Cache stuck for too long, giving up"); - WT_TRET(__wt_cache_dump(session, NULL)); + ret = ETIMEDOUT; + WT_TRET(__wt_verbose_dump_txn(session)); + WT_TRET(__wt_verbose_dump_cache(session)); return (ret); +#elif defined(HAVE_VERBOSE) + if (WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) { + WT_RET(__wt_verbose_dump_txn(session)); + WT_RET(__wt_verbose_dump_cache(session)); + + /* Reset the timer. */ + __wt_epoch(session, &cache->stuck_ts); + } +#endif } #endif } @@ -382,11 +466,13 @@ __wt_evict_create(WT_SESSION_IMPL *session) /* Set first, the thread might run before we finish up. */ F_SET(conn, WT_CONN_EVICTION_RUN); - /* Create the eviction thread group */ + /* + * Create the eviction thread group. + * Set the group size to the maximum allowed sessions. + */ WT_RET(__wt_thread_group_create(session, &conn->evict_threads, - "eviction-server", conn->evict_threads_min, - conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL, - __wt_evict_thread_run)); + "eviction-server", conn->evict_threads_min, conn->evict_threads_max, + WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL, __wt_evict_thread_run)); /* * Allow queues to be populated now that the eviction threads @@ -413,7 +499,7 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) return (0); /* Wait for any eviction thread group changes to stabilize. */ - __wt_writelock(session, conn->evict_threads.lock); + __wt_writelock(session, &conn->evict_threads.lock); /* * Signal the threads to finish and stop populating the queue. @@ -541,6 +627,8 @@ __evict_pass(WT_SESSION_IMPL *session) if (loop == 0) prev = now; + if (conn->evict_threads.threads[0]->session == session) + WT_RET(__evict_tune_workers(session)); /* * Increment the shared read generation. Do this occasionally * even if eviction is not currently required, so that pages @@ -566,14 +654,6 @@ __evict_pass(WT_SESSION_IMPL *session) if (!__evict_update_work(session)) break; - /* - * Try to start a new thread if we have capacity and haven't - * reached the eviction targets. - */ - if (F_ISSET(cache, WT_CACHE_EVICT_ALL)) - WT_RET(__wt_thread_group_start_one( - session, &conn->evict_threads, false)); - __wt_verbose(session, WT_VERB_EVICTSERVER, "Eviction pass with: Max: %" PRIu64 " In use: %" PRIu64 " Dirty: %" PRIu64, @@ -648,8 +728,8 @@ __evict_pass(WT_SESSION_IMPL *session) */ WT_STAT_CONN_INCR(session, cache_eviction_server_slept); - __wt_cond_wait( - session, cache->evict_cond, WT_THOUSAND); + __wt_cond_wait(session, + cache->evict_cond, WT_THOUSAND, NULL); continue; } @@ -676,7 +756,7 @@ __evict_pass(WT_SESSION_IMPL *session) * Clear a single walk point. */ static int -__evict_clear_walk(WT_SESSION_IMPL *session, bool count_stat) +__evict_clear_walk(WT_SESSION_IMPL *session) { WT_BTREE *btree; WT_CACHE *cache; @@ -693,14 +773,14 @@ __evict_clear_walk(WT_SESSION_IMPL *session, bool count_stat) if ((ref = btree->evict_ref) == NULL) return (0); - if (count_stat) - WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned); + WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned); /* - * Clear evict_ref first, in case releasing it forces eviction (we - * assert we never try to evict the current eviction walk point). + * Clear evict_ref before releasing it in case that forces eviction (we + * assert that we never try to evict the current eviction walk point). */ btree->evict_ref = NULL; + WT_WITH_DHANDLE(cache->walk_session, session->dhandle, (ret = __wt_page_release(cache->walk_session, ref, WT_READ_NO_EVICT))); @@ -723,7 +803,7 @@ __evict_clear_all_walks(WT_SESSION_IMPL *session) TAILQ_FOREACH(dhandle, &conn->dhqh, q) if (WT_PREFIX_MATCH(dhandle->name, "file:")) WT_WITH_DHANDLE(session, dhandle, - WT_TRET(__evict_clear_walk(session, true))); + WT_TRET(__evict_clear_walk(session))); return (ret); } @@ -744,31 +824,19 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) btree = S2BT(session); cache = S2C(session)->cache; - /* - * Hold the walk lock to set the no-eviction flag. - * - * The no-eviction flag can be set permanently, in which case we never - * increment the no-eviction count. - */ + /* Hold the walk lock to turn off eviction. */ __wt_spin_lock(session, &cache->evict_walk_lock); - if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) { - if (btree->evict_disabled != 0) - ++btree->evict_disabled; + if (++btree->evict_disabled > 1) { __wt_spin_unlock(session, &cache->evict_walk_lock); return (0); } - ++btree->evict_disabled; /* * Ensure no new pages from the file will be queued for eviction after - * this point. + * this point, then clear any existing LRU eviction walk for the file. */ - F_SET(btree, WT_BTREE_NO_EVICTION); (void)__wt_atomic_addv32(&cache->pass_intr, 1); - - /* Clear any existing LRU eviction walk for the file. */ - WT_WITH_PASS_LOCK(session, ret, - ret = __evict_clear_walk(session, true)); + WT_WITH_PASS_LOCK(session, ret = __evict_clear_walk(session)); (void)__wt_atomic_subv32(&cache->pass_intr, 1); WT_ERR(ret); @@ -799,7 +867,6 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) if (0) { err: --btree->evict_disabled; - F_CLR(btree, WT_BTREE_NO_EVICTION); } __wt_spin_unlock(session, &cache->evict_walk_lock); return (ret); @@ -824,17 +891,233 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) */ WT_DIAGNOSTIC_YIELD; + /* Hold the walk lock to turn on eviction. */ + __wt_spin_lock(session, &cache->evict_walk_lock); WT_ASSERT(session, - btree->evict_ref == NULL && F_ISSET(btree, WT_BTREE_NO_EVICTION)); + btree->evict_ref == NULL && btree->evict_disabled > 0); + --btree->evict_disabled; + __wt_spin_unlock(session, &cache->evict_walk_lock); +} + +#define EVICT_TUNE_BATCH 1 /* Max workers to add each period */ +/* + * Data points needed before deciding if we should keep adding workers or settle + * on an earlier value. + */ +#define EVICT_TUNE_DATAPT_MIN 3 +#define EVICT_TUNE_PERIOD 1 /* Tune period in seconds */ + +/* + * We will do a fresh re-tune every that many seconds to adjust to + * significant phase changes. + */ +#define EVICT_FORCE_RETUNE 30 + +/* + * __evict_tune_workers -- + * Find the right number of eviction workers. Gradually ramp up the number of + * workers increasing the number in batches indicated by the setting above. + * Store the number of workers that gave us the best throughput so far and the + * number of data points we have tried. + * + * Every once in a while when we have the minimum number of data points we check + * whether the eviction throughput achieved with the current number of workers + * is the best we have seen so far. If so, we will keep increasing the number of + * workers. If not, we are past the infliction point on the eviction throughput + * curve. In that case, we will set the number of workers to the best observed + * so far and settle into a stable state. + */ +static int +__evict_tune_workers(WT_SESSION_IMPL *session) +{ + struct timespec current_time; + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + uint64_t delta_msec, delta_pages; + uint64_t pgs_evicted_cur, pgs_evicted_persec_cur, time_diff; + int32_t cur_threads, i, target_threads, thread_surplus; + + conn = S2C(session); + cache = conn->cache; + + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); + pgs_evicted_cur = pgs_evicted_persec_cur = 0; + + __wt_epoch(session, ¤t_time); + time_diff = WT_TIMEDIFF_SEC(current_time, conn->evict_tune_last_time); /* - * The no-eviction flag can be set permanently, in which case we never - * increment the no-eviction count. + * If we have reached the stable state and have not run long enough to + * surpass the forced re-tuning threshold, return. */ - __wt_spin_lock(session, &cache->evict_walk_lock); - if (btree->evict_disabled > 0 && --btree->evict_disabled == 0) - F_CLR(btree, WT_BTREE_NO_EVICTION); - __wt_spin_unlock(session, &cache->evict_walk_lock); + if (conn->evict_tune_stable) { + if (time_diff < EVICT_FORCE_RETUNE) + return (0); + + /* + * Stable state was reached a long time ago. Let's re-tune. + * Reset all the state. + */ + conn->evict_tune_stable = 0; + conn->evict_tune_last_action_time.tv_sec = 0; + conn->evict_tune_pgs_last = 0; + conn->evict_tune_num_points = 0; + conn->evict_tune_pg_sec_max = 0; + conn->evict_tune_workers_best = 0; + + /* Reduce the number of eviction workers to the minimum */ + thread_surplus = + (int32_t)conn->evict_threads.current_threads - + (int32_t)conn->evict_threads_min; + + for (i = 0; i < thread_surplus; i++) { + WT_ERR(__wt_thread_group_stop_one( + session, &conn->evict_threads, false)); + WT_STAT_CONN_INCR(session, + cache_eviction_worker_removed); + } + WT_STAT_CONN_INCR(session, cache_eviction_force_retune); + } else + if (time_diff < EVICT_TUNE_PERIOD) + /* + * If we have not reached stable state, don't do + * anything unless enough time has passed since the last + * time we have taken any action in this function. + */ + return (0); + + /* + * Measure the number of evicted pages so far. Eviction rate correlates + * to performance, so this is our metric of success. + */ + pgs_evicted_cur = cache->pages_evict; + + /* + * If we have recorded the number of pages evicted at the end of + * the previous measurement interval, we can compute the eviction + * rate in evicted pages per second achieved during the current + * measurement interval. + * Otherwise, we just record the number of evicted pages and return. + */ + if (conn->evict_tune_pgs_last == 0) + goto err; + + delta_msec = WT_TIMEDIFF_MS(current_time, conn->evict_tune_last_time); + delta_pages = pgs_evicted_cur - conn->evict_tune_pgs_last; + pgs_evicted_persec_cur = (delta_pages * WT_THOUSAND) / delta_msec; + conn->evict_tune_num_points++; + + /* + * Keep track of the maximum eviction throughput seen and the number + * of workers corresponding to that throughput. + */ + if (pgs_evicted_persec_cur > conn->evict_tune_pg_sec_max) { + conn->evict_tune_pg_sec_max = pgs_evicted_persec_cur; + conn->evict_tune_workers_best = + conn->evict_threads.current_threads; + } + + /* + * Compare the current number of data points with the number + * needed variable. If they are equal, we will check whether + * we are still going up on the performance curve, in which + * case we will continue increasing the number of workers, or + * we are past the inflection point on the curve, in which case + * we will go back to the best observed number of workers and + * settle into a stable state. + */ + if (conn->evict_tune_num_points >= conn->evict_tune_datapts_needed) { + if (conn->evict_tune_workers_best == + conn->evict_threads.current_threads && + conn->evict_threads.current_threads < + conn->evict_threads_max) { + /* + * Keep adding workers. We will check again + * at the next check point. + */ + conn->evict_tune_datapts_needed += WT_MIN( + EVICT_TUNE_DATAPT_MIN, + (conn->evict_threads_max - + conn->evict_threads.current_threads) / + EVICT_TUNE_BATCH); + } else { + /* + * We are past the inflection point. Choose the + * best number of eviction workers observed and + * settle into a stable state. + */ + thread_surplus = + (int32_t)conn->evict_threads.current_threads - + (int32_t)conn->evict_tune_workers_best; + + for (i = 0; i < thread_surplus; i++) { + /* + * If we get an error, it should be because we + * were unable to acquire the thread group lock. + * Break out of trying. + */ + WT_ERR(__wt_thread_group_stop_one( + session, &conn->evict_threads, false)); + WT_STAT_CONN_INCR(session, + cache_eviction_worker_removed); + } + WT_STAT_CONN_SET(session, + cache_eviction_stable_state_workers, + conn->evict_tune_workers_best); + conn->evict_tune_stable = true; + WT_STAT_CONN_SET(session, cache_eviction_active_workers, + conn->evict_threads.current_threads); + goto err; + } + } + + /* + * If we have not added any worker threads in the past, we set the + * number needed equal to the number of data points that we must + * accumulate before deciding if we should keep adding workers or settle + * on a previously tried value of workers. + */ + if (conn->evict_tune_last_action_time.tv_sec == 0) + conn->evict_tune_datapts_needed = WT_MIN(EVICT_TUNE_DATAPT_MIN, + (conn->evict_threads_max - + conn->evict_threads.current_threads) / EVICT_TUNE_BATCH); + + if (F_ISSET(cache, WT_CACHE_EVICT_ALL)) { + cur_threads = (int32_t)conn->evict_threads.current_threads; + target_threads = WT_MIN(cur_threads + EVICT_TUNE_BATCH, + (int32_t)conn->evict_threads_max); + /* + * Start the new threads. + */ + for (i = cur_threads; i < target_threads; ++i) { + /* + * If we get an error, it should be because we were + * unable to acquire the thread group lock. Break out + * of trying. + */ + WT_ERR(__wt_thread_group_start_one(session, + &conn->evict_threads, false)); + WT_STAT_CONN_INCR(session, + cache_eviction_worker_created); + __wt_verbose(session, WT_VERB_EVICTSERVER, + "added worker thread"); + } + conn->evict_tune_last_action_time = current_time; + } + + WT_STAT_CONN_SET(session, cache_eviction_active_workers, + conn->evict_threads.current_threads); + +err: conn->evict_tune_last_time = current_time; + conn->evict_tune_pgs_last = pgs_evicted_cur; + /* + * If we got an EBUSY trying to acquire the lock just return. + * We can try to tune the workers next time. + */ + if (ret == EBUSY) + ret = 0; + return (ret); } /* @@ -860,7 +1143,8 @@ __evict_lru_pages(WT_SESSION_IMPL *session, bool is_server) /* If a worker thread found the queue empty, pause. */ if (ret == WT_NOTFOUND && !is_server && F_ISSET(S2C(session), WT_CONN_EVICTION_RUN)) - __wt_cond_wait(session, conn->evict_threads.wait_cond, 10000); + __wt_cond_wait( + session, conn->evict_threads.wait_cond, 10000, NULL); return (ret == WT_NOTFOUND ? 0 : ret); } @@ -1039,7 +1323,7 @@ __evict_walk(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue) WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - u_int max_entries, retries, slot, spins, start_slot, total_candidates; + u_int max_entries, retries, slot, start_slot, total_candidates; bool dhandle_locked, incr; conn = S2C(session); @@ -1077,16 +1361,7 @@ retry: while (slot < max_entries) { * reference count to keep it alive while we sweep. */ if (!dhandle_locked) { - for (spins = 0; (ret = __wt_spin_trylock( - session, &conn->dhandle_lock)) == EBUSY && - cache->pass_intr == 0; - spins++) { - if (spins < WT_THOUSAND) - __wt_yield(); - else - __wt_sleep(0, WT_THOUSAND); - } - WT_ERR(ret); + WT_ERR(__evict_lock_handle_list(session)); dhandle_locked = true; } @@ -1122,7 +1397,7 @@ retry: while (slot < max_entries) { /* Skip files that don't allow eviction. */ btree = dhandle->handle; - if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) + if (btree->evict_disabled > 0) continue; /* @@ -1141,9 +1416,17 @@ retry: while (slot < max_entries) { !__wt_cache_aggressive(session)) continue; - /* Skip files if we have used all available hazard pointers. */ - if (btree->evict_ref == NULL && session->nhazard >= - conn->hazard_max - WT_MIN(conn->hazard_max / 2, 10)) + /* + * Skip files if we have too many active walks. + * + * This used to be limited by the configured maximum number of + * hazard pointers per session. Even though that ceiling has + * been removed, we need to test eviction with huge numbers of + * active trees before allowing larger numbers of hazard + * pointers in the walk session. + */ + if (btree->evict_ref == NULL && + session->nhazard > WT_EVICT_MAX_TREES) continue; /* @@ -1157,7 +1440,7 @@ retry: while (slot < max_entries) { (void)__wt_atomic_addi32(&dhandle->session_inuse, 1); incr = true; - __wt_spin_unlock(session, &conn->dhandle_lock); + __wt_readunlock(session, &conn->dhandle_lock); dhandle_locked = false; /* @@ -1170,13 +1453,23 @@ retry: while (slot < max_entries) { * the tree's current eviction point, and part of the process is * waiting on this thread to acknowledge that action. */ - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION) && + if (btree->evict_disabled == 0 && !__wt_spin_trylock(session, &cache->evict_walk_lock)) { - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { + if (btree->evict_disabled == 0) { + /* + * Assert the handle has a root page: eviction + * should have been locked out if the tree is + * being discarded or the root page is changing. + * As this has not always been the case, assert + * to debug that change. + */ + WT_ASSERT(session, btree->root.page != NULL); + cache->evict_file_next = dhandle; - WT_WITH_DHANDLE(session, dhandle, ret = - __evict_walk_file(session, queue, - max_entries, &slot)); + WT_WITH_DHANDLE(session, dhandle, + ret = __evict_walk_file( + session, queue, max_entries, &slot)); + WT_ASSERT(session, session->split_gen == 0); } __wt_spin_unlock(session, &cache->evict_walk_lock); @@ -1204,7 +1497,7 @@ retry: while (slot < max_entries) { } err: if (dhandle_locked) { - __wt_spin_unlock(session, &conn->dhandle_lock); + __wt_readunlock(session, &conn->dhandle_lock); dhandle_locked = false; } @@ -1267,8 +1560,8 @@ __evict_push_candidate(WT_SESSION_IMPL *session, * Get a few page eviction candidates from a single underlying file. */ static int -__evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, - u_int max_entries, u_int *slotp) +__evict_walk_file(WT_SESSION_IMPL *session, + WT_EVICT_QUEUE *queue, u_int max_entries, u_int *slotp) { WT_BTREE *btree; WT_CACHE *cache; @@ -1300,6 +1593,19 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, start = queue->evict_queue + *slotp; remaining_slots = max_entries - *slotp; total_slots = max_entries - queue->evict_entries; + btree_inuse = cache_inuse = 0; + target_pages_clean = target_pages_dirty = 0; + + /* + * The number of times we should fill the queue by the end of + * considering all trees. + */ +#define QUEUE_FILLS_PER_PASS 10 + + /* + * The minimum number of pages we should consider per tree. + */ +#define MIN_PAGES_PER_TREE 10 /* * The target number of pages for this tree is proportional to the @@ -1308,13 +1614,12 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, * cache (and only have to walk it once). */ if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { - btree_inuse = __wt_btree_bytes_inuse(session); + btree_inuse = __wt_btree_bytes_evictable(session); cache_inuse = __wt_cache_bytes_inuse(cache); bytes_per_slot = 1 + cache_inuse / total_slots; target_pages_clean = (uint32_t)( (btree_inuse + bytes_per_slot / 2) / bytes_per_slot); - } else - target_pages_clean = 0; + } if (F_ISSET(cache, WT_CACHE_EVICT_DIRTY)) { btree_inuse = __wt_btree_dirty_leaf_inuse(session); @@ -1322,35 +1627,58 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, bytes_per_slot = 1 + cache_inuse / total_slots; target_pages_dirty = (uint32_t)( (btree_inuse + bytes_per_slot / 2) / bytes_per_slot); - } else - target_pages_dirty = 0; + } - target_pages = WT_MAX(target_pages_clean, target_pages_dirty); + /* + * Weight the number of target pages by the number of times we want to + * fill the cache per pass through all the trees. Note that we don't + * build this into the calculation above because we don't want to favor + * small trees, so round to a whole number of slots (zero for small + * trees) before multiplying. + */ + target_pages = WT_MAX(target_pages_clean, target_pages_dirty) * + QUEUE_FILLS_PER_PASS; + /* + * Randomly walk trees with a small fraction of the cache in case there + * are so many trees that none of them use enough of the cache to be + * allocated slots. + * + * The chance of walking a tree is equal to the chance that a random + * byte in cache belongs to the tree, weighted by how many times we + * want to fill queues during a pass through all the trees in cache. + */ if (target_pages == 0) { - /* - * Randomly walk trees with a tiny fraction of the cache in - * case there are so many trees that none of them use enough of - * the cache to be allocated slots. Walk small trees 1% of the - * time. - */ - if (__wt_random(&session->rnd) > UINT32_MAX / 100) + if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { + btree_inuse = __wt_btree_bytes_evictable(session); + cache_inuse = __wt_cache_bytes_inuse(cache); + } else { + btree_inuse = __wt_btree_dirty_leaf_inuse(session); + cache_inuse = __wt_cache_dirty_leaf_inuse(cache); + } + if (btree_inuse == 0 || cache_inuse == 0) + return (0); + if (__wt_random64(&session->rnd) % cache_inuse > + btree_inuse * QUEUE_FILLS_PER_PASS) return (0); - target_pages = 10; } + /* + * There is some cost associated with walking a tree. If we're going + * to visit this tree, always look for a minimum number of pages. + */ + if (target_pages < MIN_PAGES_PER_TREE) + target_pages = MIN_PAGES_PER_TREE; + + /* + * If the tree is dead or we're near the end of the queue, fill the + * remaining slots. + */ if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || target_pages > remaining_slots) target_pages = remaining_slots; end = start + target_pages; - walk_flags = - WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT; - - /* Randomize the walk direction. */ - if (btree->evict_walk_reverse) - FLD_SET(walk_flags, WT_READ_PREV); - /* * Examine at least a reasonable number of pages before deciding * whether to give up. When we are only looking for dirty pages, @@ -1361,9 +1689,44 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) min_pages *= 10; + walk_flags = + WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT; + + /* + * Choose a random point in the tree if looking for candidates in a + * tree with no starting point set. This is mostly aimed at ensuring + * eviction fairly visits all pages in trees with a lot of in-cache + * content. + */ + switch ((WT_EVICT_WALK_START)btree->evict_start_type) { + case WT_EVICT_WALK_NEXT: + break; + case WT_EVICT_WALK_PREV: + FLD_SET(walk_flags, WT_READ_PREV); + break; + case WT_EVICT_WALK_RAND_PREV: + FLD_SET(walk_flags, WT_READ_PREV); + /* FALLTHROUGH */ + case WT_EVICT_WALK_RAND_NEXT: + if (btree->evict_ref == NULL) { + /* Ensure internal pages indexes remain valid */ + WT_WITH_PAGE_INDEX(session, ret = __wt_random_descent( + session, &btree->evict_ref, true)); + WT_RET_NOTFOUND_OK(ret); + } + break; + } + + /* + * Get some more eviction candidate pages, starting at the last saved + * point. Clear the saved point immediately, we assert when discarding + * pages we're not discarding an eviction point, so this clear must be + * complete before the page is released. + */ + ref = btree->evict_ref; + btree->evict_ref = NULL; + /* - * Get some more eviction candidate pages. - * * !!! Take care terminating this loop. * * Don't make an extra call to __wt_tree_walk after we hit the end of a @@ -1376,7 +1739,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, for (evict = start, pages_queued = pages_seen = refs_walked = 0; evict < end && (ret == 0 || ret == WT_NOTFOUND); ret = __wt_tree_walk_count( - session, &btree->evict_ref, &refs_walked, walk_flags)) { + session, &ref, &refs_walked, walk_flags)) { /* * Check whether we're finding a good ratio of candidates vs * pages seen. Some workloads create "deserts" in trees where @@ -1387,10 +1750,18 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, pages_seen > min_pages && (pages_queued == 0 || (pages_seen / pages_queued) > (min_pages / target_pages)); - if (give_up) + if (give_up) { + /* + * Try a different walk start point next time if a + * walk gave up. + */ + btree->evict_start_type = + (btree->evict_start_type + 1) % + WT_EVICT_WALK_START_NUM; break; + } - if ((ref = btree->evict_ref) == NULL) { + if (ref == NULL) { if (++restarts == 2) break; WT_STAT_CONN_INCR( @@ -1424,7 +1795,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, if (page->read_gen == WT_READGEN_NOTSET) __wt_cache_read_gen_new(session, page); - /* Pages we no longer need (clean or dirty), are found money. */ + /* Pages being forcibly evicted go on the urgent queue. */ if (page->read_gen == WT_READGEN_OLDEST || page->memory_footprint >= btree->splitmempage) { WT_STAT_CONN_INCR( @@ -1434,7 +1805,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, continue; } - /* Pages that are empty or from dead trees are also good. */ + /* Pages that are empty or from dead trees are fast-tracked. */ if (__wt_page_is_empty(page) || F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) goto fast; @@ -1449,7 +1820,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, /* Limit internal pages to 50% of the total. */ if (WT_PAGE_IS_INTERNAL(page) && - internal_pages >= (int)(evict - start) / 2) + internal_pages > (int)(evict - start) / 2) continue; /* If eviction gets aggressive, anything else is fair game. */ @@ -1480,7 +1851,7 @@ fast: /* If the page can't be evicted, give up. */ ++pages_queued; if (WT_PAGE_IS_INTERNAL(page)) - ++internal_pages; + ++internal_pages; __wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" WT_SIZET_FMT, @@ -1493,18 +1864,20 @@ fast: /* If the page can't be evicted, give up. */ session, cache_eviction_pages_queued, (u_int)(evict - start)); /* - * If we didn't find any candidates in the file, reverse the direction - * of the walk and skip it next time. + * If we couldn't find the number of pages we were looking for, skip + * the tree next time. */ - if (give_up) - btree->evict_walk_reverse = !btree->evict_walk_reverse; - if (give_up && !urgent_queued) + if (pages_queued < target_pages / 2 && !urgent_queued) btree->evict_walk_period = WT_MIN( WT_MAX(1, 2 * btree->evict_walk_period), 100); else if (pages_queued == target_pages) btree->evict_walk_period = 0; + else if (btree->evict_walk_period > 0) + btree->evict_walk_period /= 2; /* + * Give up the walk occasionally. + * * If we happen to end up on the root page or a page requiring urgent * eviction, clear it. We have to track hazard pointers, and the root * page complicates that calculation. @@ -1516,16 +1889,20 @@ fast: /* If the page can't be evicted, give up. */ * If we land on a page requiring forced eviction, move on to the next * page: we want this page evicted as quickly as possible. */ - if ((ref = btree->evict_ref) != NULL) { - /* Give up the walk occasionally. */ + if (ref != NULL) { if (__wt_ref_is_root(ref) || evict == start || give_up || ref->page->read_gen == WT_READGEN_OLDEST || - ref->page->memory_footprint >= btree->splitmempage) - WT_RET(__evict_clear_walk(session, restarts == 0)); - else if (ref->page->read_gen == WT_READGEN_OLDEST) + ref->page->memory_footprint >= btree->splitmempage) { + if (restarts == 0) + WT_STAT_CONN_INCR( + session, cache_eviction_walks_abandoned); + WT_RET(__wt_page_release(cache->walk_session, + ref, WT_READ_NO_EVICT)); + ref = NULL; + } else if (ref->page->read_gen == WT_READGEN_OLDEST) WT_RET_NOTFOUND_OK(__wt_tree_walk_count( - session, &btree->evict_ref, - &refs_walked, walk_flags)); + session, &ref, &refs_walked, walk_flags)); + btree->evict_ref = ref; } WT_STAT_CONN_INCRV(session, cache_eviction_walk, refs_walked); @@ -1782,6 +2159,7 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *txn_state; uint64_t init_evict_count, max_pages_evicted; + bool timer; conn = S2C(session); cache = conn->cache; @@ -1802,7 +2180,9 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) __wt_evict_server_wake(session); /* Track how long application threads spend doing eviction. */ - if (WT_STAT_ENABLED(session) && !F_ISSET(session, WT_SESSION_INTERNAL)) + timer = + WT_STAT_ENABLED(session) && !F_ISSET(session, WT_SESSION_INTERNAL); + if (timer) __wt_epoch(session, &enter); for (init_evict_count = cache->pages_evict;; ret = 0) { @@ -1859,8 +2239,8 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) break; case WT_NOTFOUND: /* Allow the queue to re-populate before retrying. */ - __wt_cond_wait( - session, conn->evict_threads.wait_cond, 10000); + __wt_cond_wait(session, + conn->evict_threads.wait_cond, 10000, NULL); cache->app_waits++; break; default: @@ -1868,8 +2248,7 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) } } -err: if (WT_STAT_ENABLED(session) && - !F_ISSET(session, WT_SESSION_INTERNAL)) { +err: if (timer) { __wt_epoch(session, &leave); WT_STAT_CONN_INCRV(session, application_cache_time, WT_TIMEDIFF_US(leave, enter)); @@ -1897,7 +2276,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) page = ref->page; if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU) || - F_ISSET(S2BT(session), WT_BTREE_NO_EVICTION)) + S2BT(session)->evict_disabled > 0) return (false); /* Append to the urgent queue if we can. */ @@ -1907,7 +2286,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) __wt_spin_lock(session, &cache->evict_queue_lock); if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU) || - F_ISSET(S2BT(session), WT_BTREE_NO_EVICTION)) + S2BT(session)->evict_disabled > 0) goto done; __wt_spin_lock(session, &urgent_queue->evict_lock); @@ -1956,125 +2335,140 @@ __wt_evict_priority_clear(WT_SESSION_IMPL *session) S2BT(session)->evict_priority = 0; } -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* - * __wt_cache_dump -- - * Dump debugging information to a file (default stderr) about the size of - * the files in the cache. + * __verbose_dump_cache_single -- + * Output diagnostic information about a single file in the cache. */ -int -__wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) +static int +__verbose_dump_cache_single(WT_SESSION_IMPL *session, + uint64_t *total_bytesp, uint64_t *total_dirty_bytesp) { - FILE *fp; - WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle, *saved_dhandle; + WT_DATA_HANDLE *dhandle; WT_PAGE *page; WT_REF *next_walk; + size_t size; uint64_t intl_bytes, intl_bytes_max, intl_dirty_bytes; uint64_t intl_dirty_bytes_max, intl_dirty_pages, intl_pages; uint64_t leaf_bytes, leaf_bytes_max, leaf_dirty_bytes; uint64_t leaf_dirty_bytes_max, leaf_dirty_pages, leaf_pages; + + intl_bytes = intl_bytes_max = intl_dirty_bytes = 0; + intl_dirty_bytes_max = intl_dirty_pages = intl_pages = 0; + leaf_bytes = leaf_bytes_max = leaf_dirty_bytes = 0; + leaf_dirty_bytes_max = leaf_dirty_pages = leaf_pages = 0; + + next_walk = NULL; + while (__wt_tree_walk(session, &next_walk, + WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && + next_walk != NULL) { + page = next_walk->page; + size = page->memory_footprint; + + if (WT_PAGE_IS_INTERNAL(page)) { + ++intl_pages; + intl_bytes += size; + intl_bytes_max = WT_MAX(intl_bytes_max, size); + if (__wt_page_is_modified(page)) { + ++intl_dirty_pages; + intl_dirty_bytes += size; + intl_dirty_bytes_max = + WT_MAX(intl_dirty_bytes_max, size); + } + } else { + ++leaf_pages; + leaf_bytes += size; + leaf_bytes_max = WT_MAX(leaf_bytes_max, size); + if (__wt_page_is_modified(page)) { + ++leaf_dirty_pages; + leaf_dirty_bytes += size; + leaf_dirty_bytes_max = + WT_MAX(leaf_dirty_bytes_max, size); + } + } + } + + dhandle = session->dhandle; + if (dhandle->checkpoint == NULL) + WT_RET(__wt_msg(session, "%s():", dhandle->name)); + else + WT_RET(__wt_msg(session, "%s(checkpoint=%s):", + dhandle->name, dhandle->checkpoint)); + if (intl_pages != 0) + WT_RET(__wt_msg(session, + "internal: " + "%" PRIu64 " pages, " + "%" PRIu64 "MB, " + "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " + "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " + "%" PRIu64 "MB max page, " + "%" PRIu64 "MB max dirty page", + intl_pages, + intl_bytes / WT_MEGABYTE, + intl_pages - intl_dirty_pages, + intl_dirty_pages, + (intl_bytes - intl_dirty_bytes) / WT_MEGABYTE, + intl_dirty_bytes / WT_MEGABYTE, + intl_bytes_max / WT_MEGABYTE, + intl_dirty_bytes_max / WT_MEGABYTE)); + if (leaf_pages != 0) + WT_RET(__wt_msg(session, + "leaf: " + "%" PRIu64 " pages, " + "%" PRIu64 "MB, " + "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " + "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " + "%" PRIu64 "MB max page, " + "%" PRIu64 "MB max dirty page", + leaf_pages, + leaf_bytes / WT_MEGABYTE, + leaf_pages - leaf_dirty_pages, + leaf_dirty_pages, + (leaf_bytes - leaf_dirty_bytes) / WT_MEGABYTE, + leaf_dirty_bytes / WT_MEGABYTE, + leaf_bytes_max / WT_MEGABYTE, + leaf_dirty_bytes_max / WT_MEGABYTE)); + + *total_bytesp += intl_bytes + leaf_bytes; + *total_dirty_bytesp += intl_dirty_bytes + leaf_dirty_bytes; + + return (0); +} + +/* + * __wt_verbose_dump_cache -- + * Output diagnostic information about the cache. + */ +int +__wt_verbose_dump_cache(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; uint64_t total_bytes, total_dirty_bytes; - size_t size; conn = S2C(session); total_bytes = total_dirty_bytes = 0; - if (ofile == NULL) - fp = stderr; - else if ((fp = fopen(ofile, "w")) == NULL) - return (EIO); + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "cache dump")); - /* Note: odd string concatenation avoids spelling errors. */ - (void)fprintf(fp, "==========\n" "cache dump\n"); - - saved_dhandle = session->dhandle; - TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); + if (dhandle == NULL) + break; if (!WT_PREFIX_MATCH(dhandle->name, "file:") || !F_ISSET(dhandle, WT_DHANDLE_OPEN)) continue; - intl_bytes = intl_bytes_max = intl_dirty_bytes = 0; - intl_dirty_bytes_max = intl_dirty_pages = intl_pages = 0; - leaf_bytes = leaf_bytes_max = leaf_dirty_bytes = 0; - leaf_dirty_bytes_max = leaf_dirty_pages = leaf_pages = 0; - - next_walk = NULL; - session->dhandle = dhandle; - while (__wt_tree_walk(session, &next_walk, - WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && - next_walk != NULL) { - page = next_walk->page; - size = page->memory_footprint; - - if (WT_PAGE_IS_INTERNAL(page)) { - ++intl_pages; - intl_bytes += size; - intl_bytes_max = WT_MAX(intl_bytes_max, size); - if (__wt_page_is_modified(page)) { - ++intl_dirty_pages; - intl_dirty_bytes += size; - intl_dirty_bytes_max = - WT_MAX(intl_dirty_bytes_max, size); - } - } else { - ++leaf_pages; - leaf_bytes += size; - leaf_bytes_max = WT_MAX(leaf_bytes_max, size); - if (__wt_page_is_modified(page)) { - ++leaf_dirty_pages; - leaf_dirty_bytes += size; - leaf_dirty_bytes_max = - WT_MAX(leaf_dirty_bytes_max, size); - } - } - } - session->dhandle = NULL; - - if (dhandle->checkpoint == NULL) - (void)fprintf(fp, "%s(): \n", dhandle->name); - else - (void)fprintf(fp, "%s(checkpoint=%s): \n", - dhandle->name, dhandle->checkpoint); - if (intl_pages != 0) - (void)fprintf(fp, - "\t" "internal: " - "%" PRIu64 " pages, " - "%" PRIu64 "MB, " - "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " - "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " - "%" PRIu64 "MB max page, " - "%" PRIu64 "MB max dirty page\n", - intl_pages, - intl_bytes >> 20, - intl_pages - intl_dirty_pages, - intl_dirty_pages, - (intl_bytes - intl_dirty_bytes) >> 20, - intl_dirty_bytes >> 20, - intl_bytes_max >> 20, - intl_dirty_bytes_max >> 20); - if (leaf_pages != 0) - (void)fprintf(fp, - "\t" "leaf: " - "%" PRIu64 " pages, " - "%" PRIu64 "MB, " - "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " - "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " - "%" PRIu64 "MB max page, " - "%" PRIu64 "MB max dirty page\n", - leaf_pages, - leaf_bytes >> 20, - leaf_pages - leaf_dirty_pages, - leaf_dirty_pages, - (leaf_bytes - leaf_dirty_bytes) >> 20, - leaf_dirty_bytes >> 20, - leaf_bytes_max >> 20, - leaf_dirty_bytes_max >> 20); - - total_bytes += intl_bytes + leaf_bytes; - total_dirty_bytes += intl_dirty_bytes + leaf_dirty_bytes; + WT_WITH_DHANDLE(session, dhandle, + ret = __verbose_dump_cache_single( + session, &total_bytes, &total_dirty_bytes)); + if (ret != 0) + break; } - session->dhandle = saved_dhandle; + WT_RET(ret); /* * Apply the overhead percentage so our total bytes are comparable with @@ -2082,16 +2476,16 @@ __wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) */ total_bytes = __wt_cache_bytes_plus_overhead(conn->cache, total_bytes); - (void)fprintf(fp, + WT_RET(__wt_msg(session, "cache dump: " - "total found = %" PRIu64 "MB vs tracked inuse %" PRIu64 "MB\n" - "total dirty bytes = %" PRIu64 "MB\n", - total_bytes >> 20, __wt_cache_bytes_inuse(conn->cache) >> 20, - total_dirty_bytes >> 20); - (void)fprintf(fp, "==========\n"); - - if (ofile != NULL && fclose(fp) != 0) - return (EIO); + "total found: %" PRIu64 "MB vs tracked inuse %" PRIu64 "MB", + total_bytes / WT_MEGABYTE, + __wt_cache_bytes_inuse(conn->cache) / WT_MEGABYTE)); + WT_RET(__wt_msg(session, + "total dirty bytes: %" PRIu64 "MB", + total_dirty_bytes / WT_MEGABYTE)); + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + return (0); } #endif diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 3d1557e027e..85689efd0b1 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -37,7 +37,7 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) * Check for a hazard pointer indicating another thread is using the * page, meaning the page cannot be evicted. */ - if (__wt_page_hazard_check(session, ref->page) == NULL) + if (__wt_hazard_check(session, ref) == NULL) return (0); WT_STAT_DATA_INCR(session, cache_eviction_hazard); @@ -66,7 +66,7 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) * between. */ locked = __wt_atomic_casv32(&ref->state, WT_REF_MEM, WT_REF_LOCKED); - if ((ret = __wt_hazard_clear(session, page)) != 0 || !locked) { + if ((ret = __wt_hazard_clear(session, ref)) != 0 || !locked) { if (locked) ref->state = WT_REF_MEM; return (ret == 0 ? EBUSY : ret); @@ -396,7 +396,7 @@ __evict_review( WT_DECL_RET; WT_PAGE *page; uint32_t flags; - bool modified; + bool lookaside_retry, modified; flags = WT_EVICTING; *flagsp = flags; @@ -480,8 +480,8 @@ __evict_review( if (LF_ISSET(WT_EVICT_INMEM_SPLIT)) return (__wt_split_insert(session, ref)); - /* We are done if reconciliation is disabled. */ - if (F_ISSET(S2BT(session), WT_BTREE_NO_RECONCILE)) + /* If splits are the only permitted operation, we're done. */ + if (F_ISSET(S2BT(session), WT_BTREE_ALLOW_SPLITS)) return (EBUSY); } @@ -495,27 +495,29 @@ __evict_review( * If we have an exclusive lock (we're discarding the tree), assert * there are no updates we cannot read. * - * Don't set any other flags for internal pages: they don't have update - * lists to be saved and restored, nor can we re-create them in memory. + * Don't set any other flags for internal pages: there are no update + * lists to be saved and restored, changes can't be written into the + * lookaside table, nor can we re-create internal pages in memory. * * For leaf pages: * - * If an in-memory configuration or the page is being forcibly evicted, - * set the update-restore flag, so reconciliation will write blocks it + * In-memory pages are a known configuration. + * + * Set the update/restore flag, so reconciliation will write blocks it * can write and create a list of skipped updates for blocks it cannot - * write, along with disk images. This is how eviction of active, huge + * write, along with disk images. This is how eviction of active, huge * pages works: we take a big page and reconcile it into blocks, some of * which we write and discard, the rest of which we re-create as smaller * in-memory pages, (restoring the updates that stopped us from writing - * the block), and inserting the whole mess into the page's parent. - * - * Otherwise, if eviction is getting pressed, configure reconciliation - * to write not-yet-globally-visible updates to the lookaside table, - * allowing the eviction of pages we'd otherwise have to retain in cache - * to support older readers. + * the block), and inserting the whole mess into the page's parent. Set + * the flag in all cases because the incremental cost of update/restore + * in reconciliation is minimal, eviction shouldn't have picked a page + * where update/restore is necessary, absent some cache pressure. It's + * possible updates occurred after we selected this page for eviction, + * but it's unlikely and we don't try and manage that risk. * - * Finally, if we don't need to do eviction at the moment, create disk - * images of split pages in order to re-instantiate them. + * Additionally, if we aren't trying to free space in the cache, scrub + * the page and keep it in memory. */ cache = S2C(session)->cache; if (closing) @@ -524,25 +526,33 @@ __evict_review( if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); - else if (__wt_cache_stuck(session)) - LF_SET(WT_EVICT_LOOKASIDE); - else if (!__wt_txn_visible_all( - session, page->modify->update_txn) || - page->read_gen == WT_READGEN_OLDEST || - page->memory_footprint >= S2BT(session)->splitmempage) + else { LF_SET(WT_EVICT_UPDATE_RESTORE); - /* - * If we aren't trying to free space in the cache, scrub the - * page and keep it around. - */ - if (!LF_ISSET(WT_EVICT_LOOKASIDE) && - F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) - LF_SET(WT_EVICT_SCRUB); + if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) + LF_SET(WT_EVICT_SCRUB); + } } - *flagsp = flags; - WT_RET(__wt_reconcile(session, ref, NULL, flags)); + /* Reconcile the page. */ + ret = __wt_reconcile(session, ref, NULL, flags, &lookaside_retry); + + /* + * If reconciliation fails, eviction is stuck and reconciliation reports + * it might succeed if we use the lookaside table (the page didn't have + * uncommitted updates, it was not-yet-globally visible updates causing + * the problem), configure reconciliation to write those updates to the + * lookaside table, allowing the eviction of pages we'd otherwise have + * to retain in cache to support older readers. + */ + if (ret == EBUSY && __wt_cache_stuck(session) && lookaside_retry) { + LF_CLR(WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); + LF_SET(WT_EVICT_LOOKASIDE); + ret = __wt_reconcile(session, ref, NULL, flags, NULL); + } + + *flagsp = flags; + WT_RET(ret); /* * Success: assert the page is clean or reconciliation was configured diff --git a/src/third_party/wiredtiger/src/evict/evict_stat.c b/src/third_party/wiredtiger/src/evict/evict_stat.c index 2dd3b1e83a0..7c2d5722a63 100644 --- a/src/third_party/wiredtiger/src/evict/evict_stat.c +++ b/src/third_party/wiredtiger/src/evict/evict_stat.c @@ -134,5 +134,5 @@ __wt_curstat_cache_walk(WT_SESSION_IMPL *session) WT_STAT_DATA_SET(session, cache_state_root_size, btree->root.page->memory_footprint); - WT_WITH_HANDLE_LIST_LOCK(session, __evict_stat_walk(session)); + __evict_stat_walk(session); } diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 2783d17f825..a3636eb8040 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -7,22 +7,21 @@ */ /* Standard entry points to the API: declares/initializes local variables. */ -#define API_SESSION_INIT(s, h, n, cur, dh) \ +#define API_SESSION_INIT(s, h, n, dh) \ WT_DATA_HANDLE *__olddh = (s)->dhandle; \ const char *__oldname = (s)->name; \ - (s)->cursor = (cur); \ (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n; \ -#define API_CALL_NOCONF(s, h, n, cur, dh) do { \ - API_SESSION_INIT(s, h, n, cur, dh); \ +#define API_CALL_NOCONF(s, h, n, dh) do { \ + API_SESSION_INIT(s, h, n, dh); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ __wt_verbose((s), WT_VERB_API, "CALL: " #h ":" #n) -#define API_CALL(s, h, n, cur, dh, config, cfg) do { \ - const char *cfg[] = \ +#define API_CALL(s, h, n, dh, config, cfg) do { \ + const char *(cfg)[] = \ { WT_CONFIG_BASE(s, h##_##n), config, NULL }; \ - API_SESSION_INIT(s, h, n, cur, dh); \ + API_SESSION_INIT(s, h, n, dh); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ if ((config) != NULL) \ WT_ERR(__wt_config_check((s), \ @@ -42,17 +41,17 @@ } while (0) /* An API call wrapped in a transaction if necessary. */ -#define TXN_API_CALL(s, h, n, cur, bt, config, cfg) do { \ +#define TXN_API_CALL(s, h, n, bt, config, cfg) do { \ bool __autotxn = false; \ - API_CALL(s, h, n, bt, cur, config, cfg); \ + API_CALL(s, h, n, bt, config, cfg); \ __autotxn = !F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING);\ if (__autotxn) \ F_SET(&(s)->txn, WT_TXN_AUTOCOMMIT) /* An API call wrapped in a transaction if necessary. */ -#define TXN_API_CALL_NOCONF(s, h, n, cur, bt) do { \ +#define TXN_API_CALL_NOCONF(s, h, n, bt) do { \ bool __autotxn = false; \ - API_CALL_NOCONF(s, h, n, cur, bt); \ + API_CALL_NOCONF(s, h, n, bt); \ __autotxn = !F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING);\ if (__autotxn) \ F_SET(&(s)->txn, WT_TXN_AUTOCOMMIT) @@ -63,15 +62,16 @@ if (__autotxn) { \ if (F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT)) \ F_CLR(&(s)->txn, WT_TXN_AUTOCOMMIT); \ - else if (ret == 0 && !F_ISSET(&(s)->txn, WT_TXN_ERROR)) \ - ret = __wt_txn_commit((s), NULL); \ + else if ((ret) == 0 && \ + !F_ISSET(&(s)->txn, WT_TXN_ERROR)) \ + (ret) = __wt_txn_commit((s), NULL); \ else { \ if (retry) \ WT_TRET(__wt_session_copy_values(s)); \ WT_TRET(__wt_txn_rollback((s), NULL)); \ - if ((ret == 0 || ret == WT_ROLLBACK) && \ + if (((ret) == 0 || (ret) == WT_ROLLBACK) && \ (retry)) { \ - ret = 0; \ + (ret) = 0; \ continue; \ } \ WT_TRET(__wt_session_reset_cursors(s, false)); \ @@ -98,24 +98,24 @@ #define CONNECTION_API_CALL(conn, s, n, config, cfg) \ s = (conn)->default_session; \ - API_CALL(s, WT_CONNECTION, n, NULL, NULL, config, cfg) + API_CALL(s, WT_CONNECTION, n, NULL, config, cfg) #define CONNECTION_API_CALL_NOCONF(conn, s, n) \ s = (conn)->default_session; \ - API_CALL_NOCONF(s, WT_CONNECTION, n, NULL, NULL) + API_CALL_NOCONF(s, WT_CONNECTION, n, NULL) #define SESSION_API_CALL(s, n, config, cfg) \ - API_CALL(s, WT_SESSION, n, NULL, NULL, config, cfg) + API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define SESSION_API_CALL_NOCONF(s, n) \ - API_CALL_NOCONF(s, WT_SESSION, n, NULL, NULL) + API_CALL_NOCONF(s, WT_SESSION, n, NULL) #define SESSION_TXN_API_CALL(s, n, config, cfg) \ - TXN_API_CALL(s, WT_SESSION, n, NULL, NULL, config, cfg) + TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define CURSOR_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - API_CALL_NOCONF(s, WT_CURSOR, n, cur, \ + API_CALL_NOCONF(s, WT_CURSOR, n, \ ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle) #define JOINABLE_CURSOR_CALL_CHECK(cur) \ @@ -128,7 +128,7 @@ #define CURSOR_REMOVE_API_CALL(cur, s, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, cur, \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, \ ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); #define JOINABLE_CURSOR_REMOVE_API_CALL(cur, s, bt) \ @@ -137,7 +137,7 @@ #define CURSOR_UPDATE_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, n, cur, \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, n, \ ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && \ !F_ISSET((WT_BTREE *)(bt), WT_BTREE_IGNORE_CACHE) && \ @@ -153,4 +153,4 @@ #define ASYNCOP_API_CALL(conn, s, n) \ s = (conn)->default_session; \ - API_CALL_NOCONF(s, asyncop, n, NULL, NULL) + API_CALL_NOCONF(s, asyncop, n, NULL) diff --git a/src/third_party/wiredtiger/src/include/bitstring.i b/src/third_party/wiredtiger/src/include/bitstring.i index 08746beb9b9..118dc0bba01 100644 --- a/src/third_party/wiredtiger/src/include/bitstring.i +++ b/src/third_party/wiredtiger/src/include/bitstring.i @@ -230,7 +230,7 @@ __bit_getv(uint8_t *bitf, uint64_t entry, uint8_t width) #define __BIT_GET(len, mask) \ case len: \ if (__bit_test(bitf, bit)) \ - value |= mask; \ + value |= (mask); \ ++bit \ /* FALLTHROUGH */ diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 84c91097a99..d0b21b17965 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -414,18 +414,20 @@ struct __wt_page_modify { size_t discard_allocated; } *ovfl_track; +#define WT_PAGE_LOCK(s, p) \ + __wt_spin_lock((s), &(p)->modify->page_lock) +#define WT_PAGE_TRYLOCK(s, p) \ + __wt_spin_trylock((s), &(p)->modify->page_lock) +#define WT_PAGE_UNLOCK(s, p) \ + __wt_spin_unlock((s), &(p)->modify->page_lock) + WT_SPINLOCK page_lock; /* Page's spinlock */ + /* * The write generation is incremented when a page is modified, a page * is clean if the write generation is 0. */ uint32_t write_gen; -#define WT_PAGE_LOCK(s, p) \ - __wt_spin_lock((s), &S2C(s)->page_lock[(p)->modify->page_lock]) -#define WT_PAGE_UNLOCK(s, p) \ - __wt_spin_unlock((s), &S2C(s)->page_lock[(p)->modify->page_lock]) - uint8_t page_lock; /* Page's spinlock */ - #define WT_PM_REC_EMPTY 1 /* Reconciliation: no replacement */ #define WT_PM_REC_MULTIBLOCK 2 /* Reconciliation: multiple blocks */ #define WT_PM_REC_REPLACE 3 /* Reconciliation: single block */ @@ -434,6 +436,19 @@ struct __wt_page_modify { uint8_t update_restored; /* Page created by restoring updates */ }; +/* + * WT_COL_RLE -- + * Variable-length column-store pages have an array of page entries with RLE + * counts greater than 1 when reading the page, so it's not necessary to walk + * the page counting records to find a specific entry. We can do a binary search + * in this array, then an offset calculation to find the cell. + */ +WT_PACKED_STRUCT_BEGIN(__wt_col_rle) + uint64_t recno; /* Record number of first repeat. */ + uint64_t rle; /* Repeat count. */ + uint32_t indx; /* Slot of entry in col_var. */ +WT_PACKED_STRUCT_END + /* * WT_PAGE -- * The WT_PAGE structure describes the in-memory page information. @@ -444,9 +459,6 @@ struct __wt_page { /* * Internal pages (both column- and row-store). * - * The page record number is only used by column-store, but it's - * simpler having only one kind of internal page. - * * In-memory internal pages have an array of pointers to child * structures, maintained in collated order. When a page is * read into memory, the initial list of children is stored in @@ -473,6 +485,7 @@ struct __wt_page { */ struct { WT_REF *parent_ref; /* Parent reference */ + uint64_t split_gen; /* Generation of last split */ struct __wt_page_index { uint32_t entries; @@ -482,6 +495,8 @@ struct __wt_page { } intl; #undef pg_intl_parent_ref #define pg_intl_parent_ref u.intl.parent_ref +#undef pg_intl_split_gen +#define pg_intl_split_gen u.intl.split_gen /* * Macros to copy/set the index because the name is obscured to ensure @@ -494,7 +509,7 @@ struct __wt_page { #define WT_INTL_INDEX_GET_SAFE(page) \ ((page)->u.intl.__index) #define WT_INTL_INDEX_GET(session, page, pindex) do { \ - WT_ASSERT(session, session->split_gen != 0); \ + WT_ASSERT(session, (session)->split_gen != 0); \ (pindex) = WT_INTL_INDEX_GET_SAFE(page); \ } while (0) #define WT_INTL_INDEX_SET(page, v) do { \ @@ -518,53 +533,54 @@ struct __wt_page { } while (0) /* Row-store leaf page. */ - struct { - WT_ROW *d; /* Key/value pairs */ - uint32_t entries; /* Entries */ - } row; -#undef pg_row_d -#define pg_row_d u.row.d -#undef pg_row_entries -#define pg_row_entries u.row.entries + WT_ROW *row; /* Key/value pairs */ +#undef pg_row +#define pg_row u.row /* Fixed-length column-store leaf page. */ - struct { - uint8_t *bitf; /* Values */ - uint32_t entries; /* Entries */ - } col_fix; + uint8_t *fix_bitf; /* Values */ #undef pg_fix_bitf -#define pg_fix_bitf u.col_fix.bitf -#undef pg_fix_entries -#define pg_fix_entries u.col_fix.entries +#define pg_fix_bitf u.fix_bitf /* Variable-length column-store leaf page. */ struct { - WT_COL *d; /* Values */ + WT_COL *col_var; /* Values */ /* - * Variable-length column-store files maintain a list of - * RLE entries on the page so it's unnecessary to walk - * the page counting records to find a specific entry. + * Variable-length column-store pages have an array + * of page entries with RLE counts greater than 1 when + * reading the page, so it's not necessary to walk the + * page counting records to find a specific entry. We + * can do a binary search in this array, then an offset + * calculation to find the cell. + * + * It's a separate structure to keep the page structure + * as small as possible. */ - WT_COL_RLE *repeats; /* RLE array for lookups */ - uint32_t nrepeats; /* Number of repeat slots */ - - uint32_t entries; /* Entries */ + struct __wt_col_var_repeat { + uint32_t nrepeats; /* repeat slots */ + WT_COL_RLE repeats[0]; /* lookup RLE array */ + } *repeats; +#define WT_COL_VAR_REPEAT_SET(page) \ + ((page)->u.col_var.repeats != NULL) } col_var; -#undef pg_var_d -#define pg_var_d u.col_var.d +#undef pg_var +#define pg_var u.col_var.col_var #undef pg_var_repeats -#define pg_var_repeats u.col_var.repeats +#define pg_var_repeats u.col_var.repeats->repeats #undef pg_var_nrepeats -#define pg_var_nrepeats u.col_var.nrepeats -#undef pg_var_entries -#define pg_var_entries u.col_var.entries +#define pg_var_nrepeats u.col_var.repeats->nrepeats } u; /* - * The page's type and flags are positioned at the end of the WT_PAGE - * union, it reduces cache misses in the row-store search function. + * Page entries, type and flags are positioned at the end of the WT_PAGE + * union to reduce cache misses in the row-store search function. + * + * The entries field only applies to leaf pages, internal pages use the + * page-index entries instead. */ + uint32_t entries; /* Leaf page entries */ + #define WT_PAGE_IS_INTERNAL(page) \ ((page)->type == WT_PAGE_COL_INT || (page)->type == WT_PAGE_ROW_INT) #define WT_PAGE_INVALID 0 /* Invalid page */ @@ -582,20 +598,12 @@ struct __wt_page { #define WT_PAGE_DISK_MAPPED 0x04 /* Disk image in mapped memory */ #define WT_PAGE_EVICT_LRU 0x08 /* Page is on the LRU queue */ #define WT_PAGE_OVERFLOW_KEYS 0x10 /* Page has overflow keys */ -#define WT_PAGE_SPLIT_BLOCK 0x20 /* Split blocking eviction and splits */ -#define WT_PAGE_SPLIT_INSERT 0x40 /* A leaf page was split for append */ -#define WT_PAGE_UPDATE_IGNORE 0x80 /* Ignore updates on page discard */ +#define WT_PAGE_SPLIT_INSERT 0x20 /* A leaf page was split for append */ +#define WT_PAGE_UPDATE_IGNORE 0x40 /* Ignore updates on page discard */ uint8_t flags_atomic; /* Atomic flags, use F_*_ATOMIC */ uint8_t unused[2]; /* Unused padding */ - /* - * Used to protect and co-ordinate splits for internal pages and - * reconciliation for all pages. Only used to co-ordinate among the - * uncommon cases that require exclusive access to a page. - */ - WT_RWLOCK page_lock; - /* * The page's read generation acts as an LRU value for each page in the * tree; it is used by the eviction server thread to select pages to be @@ -621,8 +629,6 @@ struct __wt_page { #define WT_READGEN_START_VALUE 100 #define WT_READGEN_STEP 100 uint64_t read_gen; - /* The evict pass generation for the page */ - uint64_t evict_pass_gen; size_t memory_footprint; /* Memory attached to the page */ @@ -631,6 +637,10 @@ struct __wt_page { /* If/when the page is modified, we need lots more information. */ WT_PAGE_MODIFY *modify; + + /* This is the 64 byte boundary, try to keep hot fields above here. */ + + uint64_t evict_pass_gen; /* Eviction pass generation */ }; /* @@ -723,7 +733,7 @@ struct __wt_ref { * up our slot in the page's index structure. */ WT_PAGE * volatile home; /* Reference page */ - uint32_t pindex_hint; /* Reference page index hint */ + volatile uint32_t pindex_hint; /* Reference page index hint */ #define WT_REF_DISK 0 /* Page is on disk */ #define WT_REF_DELETED 1 /* Page is on disk, but deleted */ @@ -795,11 +805,11 @@ struct __wt_row { /* On-page key, on-page cell, or off-page WT_IKEY */ * Walk the entries of an in-memory row-store leaf page. */ #define WT_ROW_FOREACH(page, rip, i) \ - for ((i) = (page)->pg_row_entries, \ - (rip) = (page)->pg_row_d; (i) > 0; ++(rip), --(i)) + for ((i) = (page)->entries, \ + (rip) = (page)->pg_row; (i) > 0; ++(rip), --(i)) #define WT_ROW_FOREACH_REVERSE(page, rip, i) \ - for ((i) = (page)->pg_row_entries, \ - (rip) = (page)->pg_row_d + ((page)->pg_row_entries - 1); \ + for ((i) = (page)->entries, \ + (rip) = (page)->pg_row + ((page)->entries - 1); \ (i) > 0; --(rip), --(i)) /* @@ -807,7 +817,7 @@ struct __wt_row { /* On-page key, on-page cell, or off-page WT_IKEY */ * Return the 0-based array offset based on a WT_ROW reference. */ #define WT_ROW_SLOT(page, rip) \ - ((uint32_t)(((WT_ROW *)(rip)) - (page)->pg_row_d)) + ((uint32_t)(((WT_ROW *)(rip)) - (page)->pg_row)) /* * WT_COL -- @@ -831,18 +841,6 @@ struct __wt_col { uint32_t __col_value; }; -/* - * WT_COL_RLE -- - * In variable-length column store leaf pages, we build an array of entries - * with RLE counts greater than 1 when reading the page. We can do a binary - * search in this array, then an offset calculation to find the cell. - */ -WT_PACKED_STRUCT_BEGIN(__wt_col_rle) - uint64_t recno; /* Record number of first repeat. */ - uint64_t rle; /* Repeat count. */ - uint32_t indx; /* Slot of entry in col_var.d */ -WT_PACKED_STRUCT_END - /* * WT_COL_PTR, WT_COL_PTR_SET -- * Return/Set a pointer corresponding to the data offset. (If the item does @@ -859,15 +857,15 @@ WT_PACKED_STRUCT_END * Walk the entries of variable-length column-store leaf page. */ #define WT_COL_FOREACH(page, cip, i) \ - for ((i) = (page)->pg_var_entries, \ - (cip) = (page)->pg_var_d; (i) > 0; ++(cip), --(i)) + for ((i) = (page)->entries, \ + (cip) = (page)->pg_var; (i) > 0; ++(cip), --(i)) /* * WT_COL_SLOT -- * Return the 0-based array offset based on a WT_COL reference. */ #define WT_COL_SLOT(page, cip) \ - ((uint32_t)(((WT_COL *)cip) - (page)->pg_var_d)) + ((uint32_t)(((WT_COL *)(cip)) - (page)->pg_var)) /* * WT_IKEY -- @@ -932,7 +930,7 @@ WT_PACKED_STRUCT_BEGIN(__wt_update) #define WT_UPDATE_MEMSIZE(upd) \ WT_ALIGN(sizeof(WT_UPDATE) + \ (WT_UPDATE_DELETED_ISSET(upd) ? 0 : (upd)->size), 32) -}; +WT_PACKED_STRUCT_END /* * WT_INSERT -- @@ -976,10 +974,10 @@ struct __wt_insert { } key; } u; -#define WT_INSERT_KEY_SIZE(ins) (((WT_INSERT *)ins)->u.key.size) +#define WT_INSERT_KEY_SIZE(ins) (((WT_INSERT *)(ins))->u.key.size) #define WT_INSERT_KEY(ins) \ - ((void *)((uint8_t *)(ins) + ((WT_INSERT *)ins)->u.key.offset)) -#define WT_INSERT_RECNO(ins) (((WT_INSERT *)ins)->u.recno) + ((void *)((uint8_t *)(ins) + ((WT_INSERT *)(ins))->u.key.offset)) +#define WT_INSERT_RECNO(ins) (((WT_INSERT *)(ins))->u.recno) WT_INSERT *next[0]; /* forward-linked skip list */ }; @@ -988,9 +986,9 @@ struct __wt_insert { * Skiplist helper macros. */ #define WT_SKIP_FIRST(ins_head) \ - (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)ins_head)->head[0]) + (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)(ins_head))->head[0]) #define WT_SKIP_LAST(ins_head) \ - (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)ins_head)->tail[0]) + (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)(ins_head))->tail[0]) #define WT_SKIP_NEXT(ins) ((ins)->next[0]) #define WT_SKIP_FOREACH(ins, ins_head) \ for ((ins) = WT_SKIP_FIRST(ins_head); \ @@ -1003,7 +1001,7 @@ struct __wt_insert { #define WT_PAGE_ALLOC_AND_SWAP(s, page, dest, v, count) do { \ if (((v) = (dest)) == NULL) { \ WT_ERR(__wt_calloc_def(s, count, &(v))); \ - if (__wt_atomic_cas_ptr(&dest, NULL, v)) \ + if (__wt_atomic_cas_ptr(&(dest), NULL, v)) \ __wt_cache_page_inmem_incr( \ s, page, (count) * sizeof(*(v))); \ else \ @@ -1044,7 +1042,7 @@ struct __wt_insert_head { #define WT_ROW_INSERT_SMALLEST(page) \ ((page)->modify == NULL || \ (page)->modify->mod_row_insert == NULL ? \ - NULL : (page)->modify->mod_row_insert[(page)->pg_row_entries]) + NULL : (page)->modify->mod_row_insert[(page)->entries]) /* * The column-store leaf page update lists are arrays of pointers to structures, diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 713d46ae85f..28fe1b94b23 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -44,7 +44,7 @@ * Record numbers are stored in 64-bit unsigned integers, meaning the largest * record number is "really, really big". */ -#define WT_BTREE_MAX_OBJECT_SIZE (UINT32_MAX - 1024) +#define WT_BTREE_MAX_OBJECT_SIZE ((uint32_t)(UINT32_MAX - 1024)) /* * A location in a file is a variable-length cookie, but it has a maximum size @@ -57,6 +57,12 @@ /* Evict pages if we see this many consecutive deleted records. */ #define WT_BTREE_DELETE_THRESHOLD 1000 +/* + * Minimum size of the chunks (in percentage of the page size) a page gets split + * into during reconciliation. + */ +#define WT_BTREE_MIN_SPLIT_PCT 50 + /* * WT_BTREE -- * A btree handle. @@ -114,23 +120,26 @@ struct __wt_btree { int split_pct; /* Split page percent */ WT_COMPRESSOR *compressor; /* Page compressor */ WT_KEYED_ENCRYPTOR *kencryptor; /* Page encryptor */ - WT_RWLOCK *ovfl_lock; /* Overflow lock */ + WT_RWLOCK ovfl_lock; /* Overflow lock */ uint64_t last_recno; /* Column-store last record number */ - WT_REF root; /* Root page reference */ - bool modified; /* If the tree ever modified */ - bool bulk_load_ok; /* Bulk-load is a possibility */ + WT_REF root; /* Root page reference */ + bool modified; /* If the tree ever modified */ + uint8_t original; /* Newly created: bulk-load possible + (want a bool but needs atomic cas) */ + + bool lsm_primary; /* Handle is/was the LSM primary */ WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ uint64_t checkpoint_gen; /* Checkpoint generation */ - bool include_checkpoint_txn;/* ID checks include checkpoint */ uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ uint64_t bytes_inmem; /* Cache bytes in memory. */ + uint64_t bytes_dirty_intl; /* Bytes in dirty internal pages. */ uint64_t bytes_dirty_leaf; /* Bytes in dirty leaf pages. */ WT_REF *evict_ref; /* Eviction thread's location */ @@ -138,10 +147,10 @@ struct __wt_btree { u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_saved; /* Saved walk skips for checkpoints */ u_int evict_walk_skips; /* Number of walks skipped */ - u_int evict_disabled; /* Eviction disabled count */ + int evict_disabled; /* Eviction disabled count */ volatile uint32_t evict_busy; /* Count of threads in eviction */ - bool evict_walk_reverse; /* Walk direction */ - + int evict_start_type; /* Start position for eviction walk + (see WT_EVICT_WALK_START). */ enum { WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING } checkpointing; /* Checkpoint in progress */ @@ -154,19 +163,19 @@ struct __wt_btree { WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ /* Flags values up to 0xff are reserved for WT_DHANDLE_* */ -#define WT_BTREE_BULK 0x000100 /* Bulk-load handle */ -#define WT_BTREE_IGNORE_CACHE 0x000200 /* Cache-resident object */ -#define WT_BTREE_IN_MEMORY 0x000400 /* Cache-resident object */ -#define WT_BTREE_LOOKASIDE 0x000800 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x001000 /* Disable checkpoints */ -#define WT_BTREE_NO_EVICTION 0x002000 /* Disable eviction */ -#define WT_BTREE_NO_LOGGING 0x004000 /* Disable logging */ -#define WT_BTREE_NO_RECONCILE 0x008000 /* Allow splits, even with no evict */ -#define WT_BTREE_REBALANCE 0x010000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x020000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x040000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x080000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x100000 /* Handle is for verify */ +#define WT_BTREE_ALLOW_SPLITS 0x000100 /* Allow splits, even with no evict */ +#define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ +#define WT_BTREE_CLOSED 0x000400 /* Handle closed */ +#define WT_BTREE_IGNORE_CACHE 0x000800 /* Cache-resident object */ +#define WT_BTREE_IN_MEMORY 0x001000 /* Cache-resident object */ +#define WT_BTREE_LOOKASIDE 0x002000 /* Look-aside table */ +#define WT_BTREE_NO_CHECKPOINT 0x004000 /* Disable checkpoints */ +#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ +#define WT_BTREE_REBALANCE 0x020000 /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x040000 /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x080000 /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x100000 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x200000 /* Handle is for verify */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index e48189d50ea..1d6fcd6272c 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -70,6 +70,47 @@ __wt_btree_bytes_inuse(WT_SESSION_IMPL *session) return (__wt_cache_bytes_plus_overhead(cache, btree->bytes_inmem)); } +/* + * __wt_btree_bytes_evictable -- + * Return the number of bytes that can be evicted (i.e. bytes apart from + * the pinned root page). + */ +static inline uint64_t +__wt_btree_bytes_evictable(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_CACHE *cache; + WT_PAGE *root_page; + uint64_t bytes_inmem, bytes_root; + + btree = S2BT(session); + cache = S2C(session)->cache; + root_page = btree->root.page; + + bytes_inmem = btree->bytes_inmem; + bytes_root = root_page == NULL ? 0 : root_page->memory_footprint; + + return (bytes_inmem <= bytes_root ? 0 : + __wt_cache_bytes_plus_overhead(cache, bytes_inmem - bytes_root)); +} + +/* + * __wt_btree_dirty_inuse -- + * Return the number of dirty bytes in use. + */ +static inline uint64_t +__wt_btree_dirty_inuse(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_CACHE *cache; + + btree = S2BT(session); + cache = S2C(session)->cache; + + return (__wt_cache_bytes_plus_overhead(cache, + btree->bytes_dirty_intl + btree->bytes_dirty_leaf)); +} + /* * __wt_btree_dirty_leaf_inuse -- * Return the number of bytes in use by dirty leaf pages. @@ -105,11 +146,12 @@ __wt_cache_page_inmem_incr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) (void)__wt_atomic_addsize(&page->memory_footprint, size); if (__wt_page_is_modified(page)) { (void)__wt_atomic_addsize(&page->modify->bytes_dirty, size); - if (WT_PAGE_IS_INTERNAL(page)) + if (WT_PAGE_IS_INTERNAL(page)) { + (void)__wt_atomic_add64(&btree->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->bytes_dirty_intl, size); - else { - (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + } else if (!btree->lsm_primary) { (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); + (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); } } /* Track internal size in cache. */ @@ -238,10 +280,12 @@ __wt_cache_page_byte_dirty_decr( if (i == 5) return; - if (WT_PAGE_IS_INTERNAL(page)) + if (WT_PAGE_IS_INTERNAL(page)) { + __wt_cache_decr_check_uint64(session, &btree->bytes_dirty_intl, + decr, "WT_BTREE.bytes_dirty_intl"); __wt_cache_decr_check_uint64(session, &cache->bytes_dirty_intl, decr, "WT_CACHE.bytes_dirty_intl"); - else { + } else if (!btree->lsm_primary) { __wt_cache_decr_check_uint64(session, &btree->bytes_dirty_leaf, decr, "WT_BTREE.bytes_dirty_leaf"); __wt_cache_decr_check_uint64(session, &cache->bytes_dirty_leaf, @@ -297,11 +341,14 @@ __wt_cache_dirty_incr(WT_SESSION_IMPL *session, WT_PAGE *page) */ size = page->memory_footprint; if (WT_PAGE_IS_INTERNAL(page)) { + (void)__wt_atomic_add64(&btree->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->pages_dirty_intl, 1); } else { - (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); - (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + if (!btree->lsm_primary) { + (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); + (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + } (void)__wt_atomic_add64(&cache->pages_dirty_leaf, 1); } (void)__wt_atomic_addsize(&page->modify->bytes_dirty, size); @@ -366,7 +413,7 @@ __wt_cache_page_image_incr(WT_SESSION_IMPL *session, uint32_t size) * Evict pages from the cache. */ static inline void -__wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) +__wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page, bool rewrite) { WT_BTREE *btree; WT_CACHE *cache; @@ -390,23 +437,34 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) /* Update the cache's dirty-byte count. */ if (modify != NULL && modify->bytes_dirty != 0) { - if (WT_PAGE_IS_INTERNAL(page)) + if (WT_PAGE_IS_INTERNAL(page)) { + __wt_cache_decr_zero_uint64(session, + &btree->bytes_dirty_intl, + modify->bytes_dirty, "WT_BTREE.bytes_dirty_intl"); __wt_cache_decr_zero_uint64(session, &cache->bytes_dirty_intl, modify->bytes_dirty, "WT_CACHE.bytes_dirty_intl"); - else { - __wt_cache_decr_zero_uint64(session, - &cache->bytes_dirty_leaf, - modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf"); + } else if (!btree->lsm_primary) { __wt_cache_decr_zero_uint64(session, &btree->bytes_dirty_leaf, modify->bytes_dirty, "WT_BTREE.bytes_dirty_leaf"); + __wt_cache_decr_zero_uint64(session, + &cache->bytes_dirty_leaf, + modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf"); } } /* Update pages and bytes evicted. */ (void)__wt_atomic_add64(&cache->bytes_evict, page->memory_footprint); - (void)__wt_atomic_addv64(&cache->pages_evict, 1); + + /* + * Don't count rewrites as eviction: there's no guarantee we are making + * real progress. + */ + if (rewrite) + (void)__wt_atomic_subv64(&cache->pages_inmem, 1); + else + (void)__wt_atomic_addv64(&cache->pages_evict, 1); } /* @@ -982,7 +1040,7 @@ __wt_cursor_row_leaf_key(WT_CURSOR_BTREE *cbt, WT_ITEM *key) if (cbt->ins == NULL) { session = (WT_SESSION_IMPL *)cbt->iface.session; page = cbt->ref->page; - rip = &page->u.row.d[cbt->slot]; + rip = &page->pg_row[cbt->slot]; WT_RET(__wt_row_leaf_key(session, page, rip, key, false)); } else { key->data = WT_INSERT_KEY(cbt->ins); @@ -1179,11 +1237,10 @@ __wt_leaf_page_can_split(WT_SESSION_IMPL *session, WT_PAGE *page) * data in the last skiplist on the page. Split if there are enough * items and the skiplist does not fit within a single disk page. */ - ins_head = page->type == WT_PAGE_ROW_LEAF ? - (page->pg_row_entries == 0 ? + (page->entries == 0 ? WT_ROW_INSERT_SMALLEST(page) : - WT_ROW_INSERT_SLOT(page, page->pg_row_entries - 1)) : + WT_ROW_INSERT_SLOT(page, page->entries - 1)) : WT_COL_APPEND(page); if (ins_head == NULL) return (false); @@ -1297,9 +1354,14 @@ __wt_page_can_evict( * the original parent page's index, because evicting an internal page * discards its WT_REF array, and a thread traversing the original * parent page index might see a freed WT_REF. + * + * One special case where we know this is safe is if the handle is + * locked exclusive (e.g., when the whole tree is being evicted). In + * that case, no readers can be looking at an old index. */ - if (WT_PAGE_IS_INTERNAL(page) && - F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_BLOCK)) + if (!F_ISSET(session->dhandle, WT_DHANDLE_EXCLUSIVE) && + WT_PAGE_IS_INTERNAL(page) && !__wt_split_obsolete( + session, page->pg_intl_split_gen)) return (false); /* @@ -1351,9 +1413,9 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) if (page->read_gen != WT_READGEN_OLDEST || LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(session, WT_SESSION_NO_EVICTION) || - F_ISSET(btree, WT_BTREE_NO_EVICTION) || + btree->evict_disabled > 0 || !__wt_page_can_evict(session, ref, NULL)) - return (__wt_hazard_clear(session, page)); + return (__wt_hazard_clear(session, ref)); WT_RET_BUSY_OK(__wt_page_release_evict(session, ref)); return (0); @@ -1434,53 +1496,6 @@ __wt_page_swap_func( return (ret); } -/* - * __wt_page_hazard_check -- - * Return if there's a hazard pointer to the page in the system. - */ -static inline WT_HAZARD * -__wt_page_hazard_check(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_CONNECTION_IMPL *conn; - WT_HAZARD *hp; - WT_SESSION_IMPL *s; - uint32_t i, j, hazard_size, max, session_cnt; - - conn = S2C(session); - - /* - * No lock is required because the session array is fixed size, but it - * may contain inactive entries. We must review any active session - * that might contain a hazard pointer, so insert a barrier before - * reading the active session count. That way, no matter what sessions - * come or go, we'll check the slots for all of the sessions that could - * have been active when we started our check. - */ - WT_STAT_CONN_INCR(session, cache_hazard_checks); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (s = conn->sessions, i = 0, j = 0, max = 0; - i < session_cnt; ++s, ++i) { - if (!s->active) - continue; - WT_ORDERED_READ(hazard_size, s->hazard_size); - if (s->hazard_size > max) { - max = s->hazard_size; - WT_STAT_CONN_SET(session, - cache_hazard_max, max); - } - for (hp = s->hazard; hp < s->hazard + hazard_size; ++hp) { - ++j; - if (hp->page == page) { - WT_STAT_CONN_INCRV(session, - cache_hazard_walks, j); - return (hp); - } - } - } - WT_STAT_CONN_INCRV(session, cache_hazard_walks, j); - return (NULL); -} - /* * __wt_skip_choose_depth -- * Randomly choose a depth for a skiplist insert. @@ -1518,7 +1533,7 @@ __wt_btree_lsm_over_size(WT_SESSION_IMPL *session, uint64_t maxsize) return (false); /* A tree that can be evicted always requires a switch. */ - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) + if (btree->evict_disabled == 0) return (true); /* Check for a tree with a single leaf page. */ diff --git a/src/third_party/wiredtiger/src/include/buf.i b/src/third_party/wiredtiger/src/include/buf.i index ebbee6b4633..d192e292dcf 100644 --- a/src/third_party/wiredtiger/src/include/buf.i +++ b/src/third_party/wiredtiger/src/include/buf.i @@ -37,28 +37,30 @@ __wt_buf_extend(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) /* * __wt_buf_init -- - * Initialize a buffer at a specific size. + * Create an empty buffer at a specific size. */ static inline int __wt_buf_init(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) { + /* + * The buffer grow function does what we need, but anticipates data + * referenced by the buffer. Avoid any data copy by setting data to + * reference the buffer's allocated memory, and clearing it. + */ buf->data = buf->mem; - buf->size = 0; /* Clear existing data length */ - WT_RET(__wt_buf_grow(session, buf, size)); - - return (0); + buf->size = 0; + return (__wt_buf_grow(session, buf, size)); } /* * __wt_buf_initsize -- - * Initialize a buffer at a specific size, and set the data length. + * Create an empty buffer at a specific size, and set the data length. */ static inline int __wt_buf_initsize(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) { - buf->data = buf->mem; - buf->size = 0; /* Clear existing data length */ - WT_RET(__wt_buf_grow(session, buf, size)); + WT_RET(__wt_buf_init(session, buf, size)); + buf->size = size; /* Set the data length. */ return (0); @@ -72,14 +74,15 @@ static inline int __wt_buf_set( WT_SESSION_IMPL *session, WT_ITEM *buf, const void *data, size_t size) { - /* Ensure the buffer is large enough. */ - WT_RET(__wt_buf_initsize(session, buf, size)); - - /* Copy the data, allowing for overlapping strings. */ - if (size != 0) - memmove(buf->mem, data, size); - - return (0); + /* + * The buffer grow function does what we need, but expects the data to + * be referenced by the buffer. If we're copying data from outside the + * buffer, set it up so it makes sense to the buffer grow function. (No + * test needed, this works if WT_ITEM.data is already set to "data".) + */ + buf->data = data; + buf->size = size; + return (__wt_buf_grow(session, buf, size)); } /* diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 9a2b83b5b57..04920c3585a 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -16,6 +16,17 @@ #define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ #define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ +#define WT_EVICT_MAX_TREES 1000 /* Maximum walk points */ + +/* Ways to position when starting an eviction walk. */ +typedef enum { + WT_EVICT_WALK_NEXT, + WT_EVICT_WALK_PREV, + WT_EVICT_WALK_RAND_NEXT, + WT_EVICT_WALK_RAND_PREV +} WT_EVICT_WALK_START; +#define WT_EVICT_WALK_START_NUM (WT_EVICT_WALK_RAND_PREV + 1) + /* * WT_EVICT_ENTRY -- * Encapsulation of an eviction candidate. @@ -81,7 +92,7 @@ struct __wt_cache { uint64_t worker_evicts; /* Pages evicted by worker threads */ uint64_t evict_max_page_size; /* Largest page seen at eviction */ -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) struct timespec stuck_ts; /* Stuck timestamp */ #endif @@ -185,9 +196,9 @@ struct __wt_cache { uint32_t flags; }; -#define WT_WITH_PASS_LOCK(session, ret, op) do { \ +#define WT_WITH_PASS_LOCK(session, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_PASS)); \ - WT_WITH_LOCK(session, ret, \ + WT_WITH_LOCK_WAIT(session, \ &cache->evict_pass_lock, WT_SESSION_LOCKED_PASS, op); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index 17ab39e97d2..90dd1bcdda8 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -360,11 +360,13 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool *didworkp) /* * LSM sets the no-cache-check flag when holding the LSM tree lock, in - * that case, or when holding the schema or handle list locks (which - * block eviction), we don't want to highjack the thread for eviction. + * that case, or when holding the handle list, schema or table locks + * (which can block checkpoints and eviction), don't block the thread + * for eviction. */ if (F_ISSET(session, WT_SESSION_NO_EVICTION | - WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_LOCKED_SCHEMA)) + WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_LOCKED_SCHEMA | + WT_SESSION_LOCKED_TABLE)) return (0); /* In memory configurations don't block when the cache is full. */ @@ -372,11 +374,14 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool *didworkp) return (0); /* - * Threads operating on cache-resident trees are ignored because they're - * not contributing to the problem. + * Threads operating on cache-resident trees are ignored because + * they're not contributing to the problem. We also don't block while + * reading metadata because we're likely to be holding some other + * resources that could block checkpoints or eviction. */ btree = S2BT_SAFE(session); - if (btree != NULL && F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (btree != NULL && (F_ISSET(btree, WT_BTREE_IN_MEMORY) || + WT_IS_METADATA(session->dhandle))) return (0); /* Check if eviction is needed. */ diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index c130768e595..71c2515daf0 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -361,14 +361,12 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) cell->__chunk[0] = (uint8_t) ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT); return (1); - } else { - byte = (uint8_t)size; /* Type + length */ - cell->__chunk[0] = (uint8_t) - ((byte << WT_CELL_SHORT_SHIFT) | - WT_CELL_KEY_SHORT_PFX); - cell->__chunk[1] = prefix; /* Prefix */ - return (2); } + byte = (uint8_t)size; /* Type + length */ + cell->__chunk[0] = (uint8_t) + ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT_PFX); + cell->__chunk[1] = prefix; /* Prefix */ + return (2); } if (prefix == 0) { @@ -569,8 +567,8 @@ __wt_cell_unpack_safe( */ #define WT_CELL_LEN_CHK(t, len) do { \ if (start != NULL && \ - ((uint8_t *)t < (uint8_t *)start || \ - (((uint8_t *)t) + (len)) > (uint8_t *)end)) \ + ((uint8_t *)(t) < (uint8_t *)start || \ + (((uint8_t *)(t)) + (len)) > (uint8_t *)end)) \ return (WT_ERROR); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/column.i b/src/third_party/wiredtiger/src/include/column.i index d15f874b281..07b627315e6 100644 --- a/src/third_party/wiredtiger/src/include/column.i +++ b/src/third_party/wiredtiger/src/include/column.i @@ -108,7 +108,7 @@ __col_insert_search_match(WT_INSERT_HEAD *ins_head, uint64_t recno) /* Fast path the check for values at the end of the skiplist. */ if (recno > WT_INSERT_RECNO(ret_ins)) return (NULL); - else if (recno == WT_INSERT_RECNO(ret_ins)) + if (recno == WT_INSERT_RECNO(ret_ins)) return (ret_ins); /* @@ -127,7 +127,7 @@ __col_insert_search_match(WT_INSERT_HEAD *ins_head, uint64_t recno) if (cmp == 0) /* Exact match: return */ return (*insp); - else if (cmp > 0) /* Keep going at this level */ + if (cmp > 0) /* Keep going at this level */ insp = &(*insp)->next[i]; else { /* Drop down a level */ --i; @@ -221,13 +221,13 @@ __col_var_last_recno(WT_REF *ref) * This function ignores those records, our callers must handle that * explicitly, if they care. */ - if (page->pg_var_nrepeats == 0) - return (page->pg_var_entries == 0 ? 0 : - ref->ref_recno + (page->pg_var_entries - 1)); + if (!WT_COL_VAR_REPEAT_SET(page)) + return (page->entries == 0 ? 0 : + ref->ref_recno + (page->entries - 1)); repeat = &page->pg_var_repeats[page->pg_var_nrepeats - 1]; return ((repeat->recno + repeat->rle) - 1 + - (page->pg_var_entries - (repeat->indx + 1))); + (page->entries - (repeat->indx + 1))); } /* @@ -246,8 +246,7 @@ __col_fix_last_recno(WT_REF *ref) * This function ignores those records, our callers must handle that * explicitly, if they care. */ - return (page->pg_fix_entries == 0 ? - 0 : ref->ref_recno + (page->pg_fix_entries - 1)); + return (page->entries == 0 ? 0 : ref->ref_recno + (page->entries - 1)); } /* @@ -273,7 +272,9 @@ __col_var_search(WT_REF *ref, uint64_t recno, uint64_t *start_recnop) * slot for this record number, because we know any intervening records * have repeat counts of 1. */ - for (base = 0, limit = page->pg_var_nrepeats; limit != 0; limit >>= 1) { + for (base = 0, + limit = WT_COL_VAR_REPEAT_SET(page) ? page->pg_var_nrepeats : 0; + limit != 0; limit >>= 1) { indx = base + (limit >> 1); repeat = page->pg_var_repeats + indx; @@ -281,7 +282,7 @@ __col_var_search(WT_REF *ref, uint64_t recno, uint64_t *start_recnop) recno < repeat->recno + repeat->rle) { if (start_recnop != NULL) *start_recnop = repeat->recno; - return (page->pg_var_d + repeat->indx); + return (page->pg_var + repeat->indx); } if (recno < repeat->recno) continue; @@ -306,14 +307,14 @@ __col_var_search(WT_REF *ref, uint64_t recno, uint64_t *start_recnop) * !!! * The test could be written more simply as: * - * (recno >= start_recno + (page->pg_var_entries - start_indx)) + * (recno >= start_recno + (page->entries - start_indx)) * * It's split into two parts because the simpler test will overflow if * searching for large record numbers. */ if (recno >= start_recno && - recno - start_recno >= page->pg_var_entries - start_indx) + recno - start_recno >= page->entries - start_indx) return (NULL); - return (page->pg_var_d + start_indx + (uint32_t)(recno - start_recno)); + return (page->pg_var + start_indx + (uint32_t)(recno - start_recno)); } diff --git a/src/third_party/wiredtiger/src/include/compact.h b/src/third_party/wiredtiger/src/include/compact.h index 2bba52e7173..96797f6b275 100644 --- a/src/third_party/wiredtiger/src/include/compact.h +++ b/src/third_party/wiredtiger/src/include/compact.h @@ -6,8 +6,10 @@ * See the file LICENSE for redistribution information. */ -struct __wt_compact { +struct __wt_compact_state { uint32_t lsm_count; /* Number of LSM trees seen */ uint32_t file_count; /* Number of files seen */ uint64_t max_time; /* Configured timeout */ + + struct timespec begin; /* Starting time */ }; diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 486aa50e86c..f2746fc76d9 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -62,39 +62,40 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 10 #define WT_CONFIG_ENTRY_WT_CURSOR_close 11 #define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 12 -#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 13 -#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 14 -#define WT_CONFIG_ENTRY_WT_SESSION_close 15 -#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 16 -#define WT_CONFIG_ENTRY_WT_SESSION_compact 17 -#define WT_CONFIG_ENTRY_WT_SESSION_create 18 -#define WT_CONFIG_ENTRY_WT_SESSION_drop 19 -#define WT_CONFIG_ENTRY_WT_SESSION_join 20 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 21 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 22 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 23 -#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 24 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 25 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 26 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 27 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 28 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 29 -#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 30 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 31 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 32 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 33 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 34 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 35 -#define WT_CONFIG_ENTRY_colgroup_meta 36 -#define WT_CONFIG_ENTRY_file_config 37 -#define WT_CONFIG_ENTRY_file_meta 38 -#define WT_CONFIG_ENTRY_index_meta 39 -#define WT_CONFIG_ENTRY_lsm_meta 40 -#define WT_CONFIG_ENTRY_table_meta 41 -#define WT_CONFIG_ENTRY_wiredtiger_open 42 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 43 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 44 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 45 +#define WT_CONFIG_ENTRY_WT_SESSION_alter 13 +#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 14 +#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 15 +#define WT_CONFIG_ENTRY_WT_SESSION_close 16 +#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 17 +#define WT_CONFIG_ENTRY_WT_SESSION_compact 18 +#define WT_CONFIG_ENTRY_WT_SESSION_create 19 +#define WT_CONFIG_ENTRY_WT_SESSION_drop 20 +#define WT_CONFIG_ENTRY_WT_SESSION_join 21 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 22 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 23 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 24 +#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 25 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 26 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 27 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 28 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 29 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 30 +#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 31 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 32 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 33 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 34 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 35 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 36 +#define WT_CONFIG_ENTRY_colgroup_meta 37 +#define WT_CONFIG_ENTRY_file_config 38 +#define WT_CONFIG_ENTRY_file_meta 39 +#define WT_CONFIG_ENTRY_index_meta 40 +#define WT_CONFIG_ENTRY_lsm_meta 41 +#define WT_CONFIG_ENTRY_table_meta 42 +#define WT_CONFIG_ENTRY_wiredtiger_open 43 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 44 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 45 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 46 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 7d3d07a6abc..f74732684f5 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -107,7 +107,7 @@ struct __wt_named_extractor { * Allocate some additional slots for internal sessions so the user cannot * configure too few sessions for us to run. */ -#define WT_EXTRA_INTERNAL_SESSIONS 10 +#define WT_EXTRA_INTERNAL_SESSIONS 20 /* * WT_CONN_CHECK_PANIC -- @@ -123,15 +123,19 @@ struct __wt_named_extractor { * main queue and the hashed queue. */ #define WT_CONN_DHANDLE_INSERT(conn, dhandle, bucket) do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); \ TAILQ_INSERT_HEAD(&(conn)->dhqh, dhandle, q); \ TAILQ_INSERT_HEAD(&(conn)->dhhash[bucket], dhandle, hashq); \ - ++conn->dhandle_count; \ + ++(conn)->dhandle_count; \ } while (0) #define WT_CONN_DHANDLE_REMOVE(conn, dhandle, bucket) do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); \ TAILQ_REMOVE(&(conn)->dhqh, dhandle, q); \ TAILQ_REMOVE(&(conn)->dhhash[bucket], dhandle, hashq); \ - --conn->dhandle_count; \ + --(conn)->dhandle_count; \ } while (0) /* @@ -163,28 +167,13 @@ struct __wt_connection_impl { WT_SPINLOCK api_lock; /* Connection API spinlock */ WT_SPINLOCK checkpoint_lock; /* Checkpoint spinlock */ - WT_SPINLOCK dhandle_lock; /* Data handle list spinlock */ WT_SPINLOCK fh_lock; /* File handle queue spinlock */ WT_SPINLOCK metadata_lock; /* Metadata update spinlock */ WT_SPINLOCK reconfig_lock; /* Single thread reconfigure */ WT_SPINLOCK schema_lock; /* Schema operation spinlock */ - WT_SPINLOCK table_lock; /* Table creation spinlock */ + WT_RWLOCK table_lock; /* Table list lock */ WT_SPINLOCK turtle_lock; /* Turtle file spinlock */ - - /* - * We distribute the btree page locks across a set of spin locks. Don't - * use too many: they are only held for very short operations, each one - * is 64 bytes, so 256 will fill the L1 cache on most CPUs. - * - * Use a prime number of buckets rather than assuming a good hash - * (Reference Sedgewick, Algorithms in C, "Hash Functions"). - * - * Note: this can't be an array, we impose cache-line alignment and gcc - * doesn't support that for arrays smaller than the alignment. - */ -#define WT_PAGE_LOCKS 17 - WT_SPINLOCK *page_lock; /* Btree page spinlocks */ - u_int page_lock_cnt; /* Next spinlock to use */ + WT_RWLOCK dhandle_lock; /* Data handle list lock */ /* Connection queue */ TAILQ_ENTRY(__wt_connection_impl) q; @@ -255,12 +244,6 @@ struct __wt_connection_impl { size_t session_scratch_max; /* Max scratch memory per session */ - /* - * WiredTiger allocates space for a fixed number of hazard pointers - * in each thread of control. - */ - uint32_t hazard_max; /* Hazard array size */ - WT_CACHE *cache; /* Page cache */ volatile uint64_t cache_size; /* Cache size (either statically configured or the current size @@ -268,7 +251,7 @@ struct __wt_connection_impl { WT_TXN_GLOBAL txn_global; /* Global transaction state */ - WT_RWLOCK *hot_backup_lock; /* Hot backup serialization */ + WT_RWLOCK hot_backup_lock; /* Hot backup serialization */ bool hot_backup; /* Hot backup in progress */ char **hot_backup_list; /* Hot backup file list */ @@ -307,6 +290,15 @@ struct __wt_connection_impl { uint32_t evict_threads_max;/* Max eviction threads */ uint32_t evict_threads_min;/* Min eviction threads */ + uint32_t evict_tune_datapts_needed;/* Data needed to tune */ + struct timespec evict_tune_last_action_time;/* Time of last action */ + struct timespec evict_tune_last_time; /* Time of last check */ + uint32_t evict_tune_num_points; /* Number of values tried */ + uint64_t evict_tune_pgs_last; /* Number of pages evicted */ + uint64_t evict_tune_pg_sec_max; /* Max throughput encountered */ + bool evict_tune_stable; /* Are we stable? */ + uint32_t evict_tune_workers_best;/* Best performing value */ + #define WT_STATLOG_FILENAME "WiredTigerStat.%d.%H" WT_SESSION_IMPL *stat_session; /* Statistics log session */ wt_thread_t stat_tid; /* Statistics log thread */ @@ -332,11 +324,11 @@ struct __wt_connection_impl { bool log_tid_set; /* Log server thread set */ WT_CONDVAR *log_file_cond; /* Log file thread wait mutex */ WT_SESSION_IMPL *log_file_session;/* Log file thread session */ - wt_thread_t log_file_tid; /* Log file thread thread */ + wt_thread_t log_file_tid; /* Log file thread */ bool log_file_tid_set;/* Log file thread set */ WT_CONDVAR *log_wrlsn_cond;/* Log write lsn thread wait mutex */ WT_SESSION_IMPL *log_wrlsn_session;/* Log write lsn thread session */ - wt_thread_t log_wrlsn_tid; /* Log write lsn thread thread */ + wt_thread_t log_wrlsn_tid; /* Log write lsn thread */ bool log_wrlsn_tid_set;/* Log write lsn thread set */ WT_LOG *log; /* Logging structure */ WT_COMPRESSOR *log_compressor;/* Logging compressor */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index d522abc2a56..f32b4250d30 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -52,8 +52,8 @@ { 0 }, /* recno raw buffer */ \ NULL, /* json_private */ \ NULL, /* lang_private */ \ - { NULL, 0, 0, NULL, 0 }, /* WT_ITEM key */ \ - { NULL, 0, 0, NULL, 0 }, /* WT_ITEM value */ \ + { NULL, 0, NULL, 0, 0 }, /* WT_ITEM key */ \ + { NULL, 0, NULL, 0, 0 }, /* WT_ITEM value */ \ 0, /* int saved_err */ \ NULL, /* internal_uri */ \ 0 /* uint32_t flags */ \ @@ -73,7 +73,7 @@ struct __wt_cursor_backup { #define WT_CURBACKUP_LOCKER 0x01 /* Hot-backup started */ uint8_t flags; }; -#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)cursor)->maxid) +#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) struct __wt_cursor_btree { WT_CURSOR iface; @@ -474,7 +474,7 @@ struct __wt_cursor_stat { * Return a reference to a statistic cursor's stats structures. */ #define WT_CURSOR_STATS(cursor) \ - (((WT_CURSOR_STAT *)cursor)->stats) + (((WT_CURSOR_STAT *)(cursor))->stats) struct __wt_cursor_table { WT_CURSOR iface; @@ -493,7 +493,7 @@ struct __wt_cursor_table { }; #define WT_CURSOR_PRIMARY(cursor) \ - (((WT_CURSOR_TABLE *)cursor)->cg_cursors[0]) + (((WT_CURSOR_TABLE *)(cursor))->cg_cursors[0]) #define WT_CURSOR_RECNO(cursor) WT_STREQ((cursor)->key_format, "r") @@ -550,4 +550,4 @@ struct __wt_cursor_table { } while (0) #define WT_CURSOR_RAW_OK \ - WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW + (WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW) diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index e142441e0a6..12044e0e228 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -76,34 +76,19 @@ __cursor_leave(WT_SESSION_IMPL *session) } /* - * __curfile_enter -- - * Activate a file cursor. - */ -static inline int -__curfile_enter(WT_CURSOR_BTREE *cbt) -{ - WT_SESSION_IMPL *session; - - session = (WT_SESSION_IMPL *)cbt->iface.session; - - if (!F_ISSET(cbt, WT_CBT_NO_TXN)) - WT_RET(__cursor_enter(session)); - F_SET(cbt, WT_CBT_ACTIVE); - return (0); -} - -/* - * __curfile_leave -- - * Clear a file cursor's position. + * __cursor_reset -- + * Reset the cursor, it no longer holds any position. */ static inline int -__curfile_leave(WT_CURSOR_BTREE *cbt) +__cursor_reset(WT_CURSOR_BTREE *cbt) { WT_DECL_RET; WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)cbt->iface.session; + __cursor_pos_clear(cbt); + /* If the cursor was active, deactivate it. */ if (F_ISSET(cbt, WT_CBT_ACTIVE)) { if (!F_ISSET(cbt, WT_CBT_NO_TXN)) @@ -111,12 +96,15 @@ __curfile_leave(WT_CURSOR_BTREE *cbt) F_CLR(cbt, WT_CBT_ACTIVE); } + /* If we're not holding a cursor reference, we're done. */ + if (cbt->ref == NULL) + return (0); + /* * If we were scanning and saw a lot of deleted records on this page, * try to evict the page when we release it. */ - if (cbt->ref != NULL && - cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD) + if (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD) __wt_page_evict_soon(session, cbt->ref); cbt->page_deleted_count = 0; @@ -247,7 +235,7 @@ __cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) #ifdef HAVE_DIAGNOSTIC __wt_cursor_key_order_reset(cbt); #endif - WT_RET(__curfile_leave(cbt)); + WT_RET(__cursor_reset(cbt)); } /* @@ -259,36 +247,22 @@ __cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) /* If the transaction is idle, check that the cache isn't full. */ WT_RET(__wt_txn_idle_cache_check(session)); - if (!F_ISSET(cbt, WT_CBT_ACTIVE)) - WT_RET(__curfile_enter(cbt)); + /* Activate the file cursor. */ + if (!F_ISSET(cbt, WT_CBT_ACTIVE)) { + if (!F_ISSET(cbt, WT_CBT_NO_TXN)) + WT_RET(__cursor_enter(session)); + F_SET(cbt, WT_CBT_ACTIVE); + } /* * If this is an ordinary transactional cursor, make sure we are set up * to read. */ if (!F_ISSET(cbt, WT_CBT_NO_TXN)) - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); return (0); } -/* - * __cursor_reset -- - * Reset the cursor. - */ -static inline int -__cursor_reset(WT_CURSOR_BTREE *cbt) -{ - WT_DECL_RET; - - /* - * The cursor is leaving the API, and no longer holds any position, - * generally called to clean up the cursor after an error. - */ - ret = __curfile_leave(cbt); - __cursor_pos_clear(cbt); - return (ret); -} - /* * __cursor_row_slot_return -- * Return a row-store leaf page slot's K/V pair. diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 001eb268f86..8861e96112b 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -37,12 +37,30 @@ #define WT_SESSION_META_DHANDLE(s) \ (((WT_CURSOR_BTREE *)((s)->meta_cursor))->btree->dhandle) +#define WT_DHANDLE_ACQUIRE(dhandle) \ + (void)__wt_atomic_add32(&(dhandle)->session_ref, 1) + +#define WT_DHANDLE_RELEASE(dhandle) \ + (void)__wt_atomic_sub32(&(dhandle)->session_ref, 1) + +#define WT_DHANDLE_NEXT(session, dhandle, head, field) do { \ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST));\ + if ((dhandle) == NULL) \ + (dhandle) = TAILQ_FIRST(head); \ + else { \ + WT_DHANDLE_RELEASE(dhandle); \ + (dhandle) = TAILQ_NEXT(dhandle, field); \ + } \ + if ((dhandle) != NULL) \ + WT_DHANDLE_ACQUIRE(dhandle); \ +} while (0) + /* * WT_DATA_HANDLE -- * A handle for a generic named data source. */ struct __wt_data_handle { - WT_RWLOCK *rwlock; /* Lock for shared/exclusive ops */ + WT_RWLOCK rwlock; /* Lock for shared/exclusive ops */ TAILQ_ENTRY(__wt_data_handle) q; TAILQ_ENTRY(__wt_data_handle) hashq; @@ -62,6 +80,8 @@ struct __wt_data_handle { const char *checkpoint; /* Checkpoint name (or NULL) */ const char **cfg; /* Configuration information */ + bool compact_skip; /* If the handle failed to compact */ + WT_DATA_SOURCE *dsrc; /* Data source for this handle */ void *handle; /* Generic handle */ diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index bbb7f989332..c338acb370f 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -67,14 +67,16 @@ int __ret; \ if ((__ret = (a)) != 0 && \ (__ret == WT_PANIC || \ - ret == 0 || ret == WT_DUPLICATE_KEY || ret == WT_NOTFOUND)) \ + ret == 0 || ret == WT_DUPLICATE_KEY || \ + ret == WT_NOTFOUND || ret == WT_RESTART)) \ ret = __ret; \ } while (0) #define WT_TRET_ERROR_OK(a, e) do { \ int __ret; \ if ((__ret = (a)) != 0 && __ret != (e) && \ (__ret == WT_PANIC || \ - ret == 0 || ret == WT_DUPLICATE_KEY || ret == WT_NOTFOUND)) \ + ret == 0 || ret == WT_DUPLICATE_KEY || \ + ret == WT_NOTFOUND || ret == WT_RESTART)) \ ret = __ret; \ } while (0) #define WT_TRET_NOTFOUND_OK(a) WT_TRET_ERROR_OK(a, WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 6234f2f6bc5..bf3279d0f94 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -14,7 +14,7 @@ extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_init( WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -79,18 +79,18 @@ extern int __wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key); -extern int __wt_bloom_finalize(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_bloom_finalize(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_bloom_hash(WT_BLOOM *bloom, WT_ITEM *key, WT_BLOOM_HASH *bhash); extern int __wt_bloom_hash_get(WT_BLOOM *bloom, WT_BLOOM_HASH *bhash) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_inmem_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_close(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_close(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_compact(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_key_order_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_key_order_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -98,14 +98,14 @@ extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt); extern void __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt); extern int __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp); extern int __wt_btcur_reset(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_btcur_update_check(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -126,6 +126,7 @@ extern int __wt_delete_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) extern void __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref); extern bool __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all); extern int __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ref_out_int(WT_SESSION_IMPL *session, WT_REF *ref, bool rewrite); extern void __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref); extern void __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep); extern void __wt_free_ref( WT_SESSION_IMPL *session, WT_REF *ref, int page_type, bool free_pages); @@ -133,15 +134,15 @@ extern void __wt_free_ref_index(WT_SESSION_IMPL *session, WT_PAGE *page, WT_PAGE extern void __wt_free_update_list(WT_SESSION_IMPL *session, WT_UPDATE *upd); extern int __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_root_ref_init(WT_REF *root_ref, WT_PAGE *root, bool is_recno); extern int __wt_btree_tree_open( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_btree_evictable(WT_SESSION_IMPL *session, bool on); extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern const char *__wt_page_type_string(u_int type); +extern const char *__wt_page_type_string(u_int type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern const char *__wt_cell_type_string(uint8_t type); extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf); @@ -150,6 +151,9 @@ extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint32_t alloc_entries, bool alloc_refs, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, bool eviction) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_remove_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint32_t btree_id, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags @@ -158,8 +162,10 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #endif ); extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_split_obsolete(WT_SESSION_IMPL *session, uint64_t split_gen); extern void __wt_split_stash_discard(WT_SESSION_IMPL *session); extern void __wt_split_stash_discard_all( WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session); extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -176,7 +182,7 @@ extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *tag, WT_ITEM *b extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tree_walk_count(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tree_walk_skip(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_col_search(WT_SESSION_IMPL *session, uint64_t search_recno, WT_REF *leaf, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_leaf_key_copy( WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -185,15 +191,13 @@ extern int __wt_row_ikey_alloc(WT_SESSION_IMPL *session, uint32_t cell_offset, c extern int __wt_row_ikey_incr(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_ikey(WT_SESSION_IMPL *session, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_update_alloc( WT_SESSION_IMPL *session, WT_ITEM *value, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_UPDATE *__wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); extern void __wt_update_obsolete_free( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *leaf, WT_CURSOR_BTREE *cbt, bool insert) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_las_stats_update(WT_SESSION_IMPL *session); extern int __wt_las_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -203,7 +207,7 @@ extern int __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) W extern void __wt_las_cursor( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t *session_flags); extern int __wt_las_cursor_close( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t session_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_sweep(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_checksum_init(void); extern void __wt_config_initn( WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len); extern void __wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str); @@ -218,16 +222,19 @@ extern int __wt_config_getones_none(WT_SESSION_IMPL *session, const char *config extern int __wt_config_gets_def(WT_SESSION_IMPL *session, const char **cfg, const char *key, int def, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p); extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern const WT_CONFIG_ENTRY *__wt_conn_config_match(const char *method); -extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_get_string(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *config, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_wiredtiger_error(int error); extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -251,6 +258,7 @@ extern WT_THREAD_RET __wt_cache_pool_server(void *arg); extern int __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); +extern int __wt_conn_dhandle_alloc( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -259,7 +267,7 @@ extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *ur extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_connection_destroy(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_truncate_files( WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_wrlsn(WT_SESSION_IMPL *session, int *yield); @@ -282,7 +290,7 @@ extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curfile_next_random(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_curfile_update_check(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curfile_insert_check(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curjoin_joined(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -290,13 +298,13 @@ extern int __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSO extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx, WT_CURSOR *ref_cursor, uint8_t flags, uint8_t range, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); -extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode); -extern void __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); -extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern const char *__wt_json_tokname(int toktype); +extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); +extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_json_tokname(int toktype) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern ssize_t __wt_json_strlen(const char *src, size_t srclen); -extern int __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern ssize_t __wt_json_strlen(const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curlog_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); @@ -312,7 +320,7 @@ extern int __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *e extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); -extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); extern int __wt_cursor_get_raw_key(WT_CURSOR *cursor, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -349,7 +357,7 @@ extern int __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int extern bool __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref); extern void __wt_evict_priority_set(WT_SESSION_IMPL *session, uint64_t v); extern void __wt_evict_priority_clear(WT_SESSION_IMPL *session); -extern int __wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_curstat_cache_walk(WT_SESSION_IMPL *session); @@ -396,11 +404,10 @@ extern int __wt_logop_row_truncate_unpack( WT_SESSION_IMPL *session, const uint8 extern int __wt_logop_row_truncate_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_op_printlog(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); -extern int __wt_log_slot_switch( WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_slot_new(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_slot_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot); +extern int __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int64_t __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -420,7 +427,7 @@ extern int __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, uint32_t type, extern int __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *newconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curstat_lsm_init( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_bloom_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -434,6 +441,7 @@ extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tre extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only); extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -449,20 +457,21 @@ extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_ckptlist_get( WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT *ckptbase); +extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep); extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt); extern int __wt_ext_metadata_insert(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_remove( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase); +extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_metadata_cursor_open( WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -489,15 +498,15 @@ extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const c extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg); +extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_errno(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); extern int __wt_ext_map_windows_error( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint32_t windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -508,8 +517,8 @@ extern int __wt_close_connection_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DEC extern int __wt_os_inmemory(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_os_stdio(WT_SESSION_IMPL *session); -extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base); +extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_ext_struct_pack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_struct_size(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t *sizep, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_struct_unpack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -540,7 +549,7 @@ extern int __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uin extern void __wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint32_t __wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize); extern int __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -548,6 +557,7 @@ extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -594,11 +604,11 @@ extern int __wt_session_copy_values(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_A extern int __wt_session_release_resources(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_open_cursor(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_session_strerror(WT_SESSION *wt_session, int error); extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -608,17 +618,15 @@ extern void __wt_session_close_cache(WT_SESSION_IMPL *session); extern int __wt_session_get_btree(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cond_auto_alloc( WT_SESSION_IMPL *session, const char *name, bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_cond_auto_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern void __wt_cond_auto_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled); -extern void __wt_cond_auto_wait( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress); -extern int __wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); +extern void __wt_cond_auto_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)); extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); extern int __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, const char *file_name, int line_number, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); +extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -633,9 +641,9 @@ __wt_assert(WT_SESSION_IMPL *session, #ifdef HAVE_DIAGNOSTIC WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) #endif -; -extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -650,9 +658,10 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp , const char *file, int line #endif ); -extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_hazard_close(WT_SESSION_IMPL *session); -extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_PAGE *page); +extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref); +extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref); extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp); extern int __wt_raw_to_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_raw_to_esc_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -665,24 +674,24 @@ extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg); extern void __wt_print_huffman_code(void *huffman_arg, uint16_t symbol); extern int __wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_rwlock_alloc( WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp); -extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern int __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l); extern uint32_t __wt_nlpo2_round(uint32_t v); extern uint32_t __wt_nlpo2(uint32_t v); extern uint32_t __wt_log2_int(uint32_t n); extern bool __wt_ispo2(uint32_t v); extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2); -extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state); -extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state); -extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state); +extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern uint64_t __wt_random64(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -722,8 +731,9 @@ extern int __wt_thread_group_resize( WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, const char *name, uint32_t min, uint32_t max, uint32_t flags, int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_group_stop_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); -extern int __wt_txn_get_snapshot(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session); extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_release(WT_SESSION_IMPL *session); @@ -734,8 +744,9 @@ extern void __wt_txn_stats_update(WT_SESSION_IMPL *session); extern void __wt_txn_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_global_destroy(WT_SESSION_IMPL *session); +extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -751,7 +762,7 @@ extern int __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, const uint8_t * extern int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_truncate_log( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_truncate_end(WT_SESSION_IMPL *session); -extern int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_named_snapshot_drop(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h index fd94ef0ddf2..c0ed056c7b6 100644 --- a/src/third_party/wiredtiger/src/include/extern_posix.h +++ b/src/third_party/wiredtiger/src/include/extern_posix.h @@ -7,25 +7,26 @@ extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, b extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_file_extend( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_os_posix(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_map_preload(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, const void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_map_discard(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_unmap(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_region, size_t len, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_absolute_path(const char *path); extern const char *__wt_path_separator(void); extern bool __wt_has_priv(void); -extern void __wt_stream_set_line_buffer(FILE *fp); -extern void __wt_stream_set_no_buffer(FILE *fp); -extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_thread_id(char *buf, size_t buflen); -extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); -extern void __wt_yield(void); +extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/extern_win.h b/src/third_party/wiredtiger/src/include/extern_win.h index f06ee881ece..d548ee0b2ec 100644 --- a/src/third_party/wiredtiger/src/include/extern_win.h +++ b/src/third_party/wiredtiger/src/include/extern_win.h @@ -10,10 +10,10 @@ extern int __wt_os_win(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((war extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_absolute_path(const char *path); @@ -22,9 +22,10 @@ extern bool __wt_has_priv(void); extern void __wt_stream_set_line_buffer(FILE *fp); extern void __wt_stream_set_no_buffer(FILE *fp); extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_thread_id(char *buf, size_t buflen); +extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index b0d167525b2..f26a45c68f5 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -6,19 +6,19 @@ #define WT_CONN_CACHE_POOL 0x00000001 #define WT_CONN_CKPT_SYNC 0x00000002 #define WT_CONN_CLOSING 0x00000004 -#define WT_CONN_EVICTION_RUN 0x00000008 -#define WT_CONN_IN_MEMORY 0x00000010 -#define WT_CONN_LAS_OPEN 0x00000020 -#define WT_CONN_LEAK_MEMORY 0x00000040 -#define WT_CONN_LOG_SERVER_RUN 0x00000080 +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000008 +#define WT_CONN_EVICTION_RUN 0x00000010 +#define WT_CONN_IN_MEMORY 0x00000020 +#define WT_CONN_LAS_OPEN 0x00000040 +#define WT_CONN_LEAK_MEMORY 0x00000080 #define WT_CONN_LSM_MERGE 0x00000100 #define WT_CONN_PANIC 0x00000200 #define WT_CONN_READONLY 0x00000400 #define WT_CONN_RECOVERING 0x00000800 #define WT_CONN_SERVER_ASYNC 0x00001000 #define WT_CONN_SERVER_CHECKPOINT 0x00002000 -#define WT_CONN_SERVER_LSM 0x00004000 -#define WT_CONN_SERVER_RUN 0x00008000 +#define WT_CONN_SERVER_LOG 0x00004000 +#define WT_CONN_SERVER_LSM 0x00008000 #define WT_CONN_SERVER_STATISTICS 0x00010000 #define WT_CONN_SERVER_SWEEP 0x00020000 #define WT_CONN_WAS_BACKUP 0x00040000 @@ -53,23 +53,24 @@ #define WT_SESSION_CAN_WAIT 0x00000001 #define WT_SESSION_INTERNAL 0x00000002 #define WT_SESSION_LOCKED_CHECKPOINT 0x00000004 -#define WT_SESSION_LOCKED_HANDLE_LIST 0x00000008 -#define WT_SESSION_LOCKED_METADATA 0x00000010 -#define WT_SESSION_LOCKED_PASS 0x00000020 -#define WT_SESSION_LOCKED_SCHEMA 0x00000040 -#define WT_SESSION_LOCKED_SLOT 0x00000080 -#define WT_SESSION_LOCKED_TABLE 0x00000100 -#define WT_SESSION_LOCKED_TURTLE 0x00000200 -#define WT_SESSION_LOCK_NO_WAIT 0x00000400 -#define WT_SESSION_LOGGING_INMEM 0x00000800 -#define WT_SESSION_LOOKASIDE_CURSOR 0x00001000 -#define WT_SESSION_NO_CACHE 0x00002000 -#define WT_SESSION_NO_DATA_HANDLES 0x00004000 -#define WT_SESSION_NO_EVICTION 0x00008000 -#define WT_SESSION_NO_LOGGING 0x00010000 -#define WT_SESSION_NO_SCHEMA_LOCK 0x00020000 -#define WT_SESSION_QUIET_CORRUPT_FILE 0x00040000 -#define WT_SESSION_SERVER_ASYNC 0x00080000 +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000008 +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000010 +#define WT_SESSION_LOCKED_METADATA 0x00000020 +#define WT_SESSION_LOCKED_PASS 0x00000040 +#define WT_SESSION_LOCKED_SCHEMA 0x00000080 +#define WT_SESSION_LOCKED_SLOT 0x00000100 +#define WT_SESSION_LOCKED_TABLE_READ 0x00000200 +#define WT_SESSION_LOCKED_TABLE_WRITE 0x00000400 +#define WT_SESSION_LOCKED_TURTLE 0x00000800 +#define WT_SESSION_LOGGING_INMEM 0x00001000 +#define WT_SESSION_LOOKASIDE_CURSOR 0x00002000 +#define WT_SESSION_NO_CACHE 0x00004000 +#define WT_SESSION_NO_DATA_HANDLES 0x00008000 +#define WT_SESSION_NO_EVICTION 0x00010000 +#define WT_SESSION_NO_LOGGING 0x00020000 +#define WT_SESSION_NO_SCHEMA_LOCK 0x00040000 +#define WT_SESSION_QUIET_CORRUPT_FILE 0x00080000 +#define WT_SESSION_SERVER_ASYNC 0x00100000 #define WT_STAT_CLEAR 0x00000001 #define WT_STAT_JSON 0x00000002 #define WT_STAT_ON_CLOSE 0x00000004 @@ -91,27 +92,29 @@ #define WT_VERB_COMPACT 0x00000008 #define WT_VERB_EVICT 0x00000010 #define WT_VERB_EVICTSERVER 0x00000020 -#define WT_VERB_FILEOPS 0x00000040 -#define WT_VERB_HANDLEOPS 0x00000080 -#define WT_VERB_LOG 0x00000100 -#define WT_VERB_LSM 0x00000200 -#define WT_VERB_LSM_MANAGER 0x00000400 -#define WT_VERB_METADATA 0x00000800 -#define WT_VERB_MUTEX 0x00001000 -#define WT_VERB_OVERFLOW 0x00002000 -#define WT_VERB_READ 0x00004000 -#define WT_VERB_REBALANCE 0x00008000 -#define WT_VERB_RECONCILE 0x00010000 -#define WT_VERB_RECOVERY 0x00020000 -#define WT_VERB_SALVAGE 0x00040000 -#define WT_VERB_SHARED_CACHE 0x00080000 -#define WT_VERB_SPLIT 0x00100000 -#define WT_VERB_TEMPORARY 0x00200000 -#define WT_VERB_THREAD_GROUP 0x00400000 -#define WT_VERB_TRANSACTION 0x00800000 -#define WT_VERB_VERIFY 0x01000000 -#define WT_VERB_VERSION 0x02000000 -#define WT_VERB_WRITE 0x04000000 +#define WT_VERB_EVICT_STUCK 0x00000040 +#define WT_VERB_FILEOPS 0x00000080 +#define WT_VERB_HANDLEOPS 0x00000100 +#define WT_VERB_LOG 0x00000200 +#define WT_VERB_LSM 0x00000400 +#define WT_VERB_LSM_MANAGER 0x00000800 +#define WT_VERB_METADATA 0x00001000 +#define WT_VERB_MUTEX 0x00002000 +#define WT_VERB_OVERFLOW 0x00004000 +#define WT_VERB_READ 0x00008000 +#define WT_VERB_REBALANCE 0x00010000 +#define WT_VERB_RECONCILE 0x00020000 +#define WT_VERB_RECOVERY 0x00040000 +#define WT_VERB_RECOVERY_PROGRESS 0x00080000 +#define WT_VERB_SALVAGE 0x00100000 +#define WT_VERB_SHARED_CACHE 0x00200000 +#define WT_VERB_SPLIT 0x00400000 +#define WT_VERB_TEMPORARY 0x00800000 +#define WT_VERB_THREAD_GROUP 0x01000000 +#define WT_VERB_TRANSACTION 0x02000000 +#define WT_VERB_VERIFY 0x04000000 +#define WT_VERB_VERSION 0x08000000 +#define WT_VERB_WRITE 0x10000000 #define WT_VISIBILITY_ERR 0x00000080 /* * flags section: END diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index ce6afdd6e9c..22d78fc165a 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -10,8 +10,6 @@ #define WT_SIZET_FMT "zu" /* size_t format string */ /* Add GCC-specific attributes to types and function declarations. */ -#define WT_COMPILER_TYPE_ALIGN(x) __attribute__((aligned(x))) - #define WT_PACKED_STRUCT_BEGIN(name) \ struct __attribute__ ((__packed__)) name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/hardware.h b/src/third_party/wiredtiger/src/include/hardware.h index 0e52818ae05..2530659db21 100644 --- a/src/third_party/wiredtiger/src/include/hardware.h +++ b/src/third_party/wiredtiger/src/include/hardware.h @@ -55,7 +55,19 @@ #else #define WT_CACHE_LINE_ALIGNMENT 64 #endif -#define WT_CACHE_LINE_ALIGNMENT_VERIFY(session, a) \ - WT_ASSERT(session, \ - WT_PTRDIFF(&(a)[1], &(a)[0]) >= WT_CACHE_LINE_ALIGNMENT && \ - WT_PTRDIFF(&(a)[1], &(a)[0]) % WT_CACHE_LINE_ALIGNMENT == 0) + +/* + * Pad a structure so an array of structures get separate cache lines. + * + * Note that we avoid compiler structure alignment because that requires + * allocating aligned blocks of memory, and alignment pollutes any other type + * that contains an aligned field. It is possible that a hot field positioned + * before this one will be on the same cache line, but not if it is also + * padded. + * + * This alignment has a small impact on portability as well, as we are using an + * anonymous union here which is supported under C11, earlier versions of + * the GNU standard, and MSVC versions as early as 2003. + */ +#define WT_CACHE_LINE_PAD_BEGIN union { struct { +#define WT_CACHE_LINE_PAD_END }; char __padding[WT_CACHE_LINE_ALIGNMENT]; }; diff --git a/src/third_party/wiredtiger/src/include/intpack.i b/src/third_party/wiredtiger/src/include/intpack.i index e8bea58cede..a534de9d9a8 100644 --- a/src/third_party/wiredtiger/src/include/intpack.i +++ b/src/third_party/wiredtiger/src/include/intpack.i @@ -59,21 +59,21 @@ /* Count the leading zero bytes. */ #if defined(__GNUC__) #define WT_LEADING_ZEROS(x, i) \ - (i = (x == 0) ? (int)sizeof(x) : __builtin_clzll(x) >> 3) + ((i) = ((x) == 0) ? (int)sizeof(x) : __builtin_clzll(x) >> 3) #elif defined(_MSC_VER) #define WT_LEADING_ZEROS(x, i) do { \ - if (x == 0) i = (int)sizeof(x); \ + if ((x) == 0) (i) = (int)sizeof(x); \ else { \ unsigned long __index; \ _BitScanReverse64(&__index, x); \ __index = 63 ^ __index; \ - i = (int)(__index >> 3); } \ + (i) = (int)(__index >> 3); } \ } while (0) #else #define WT_LEADING_ZEROS(x, i) do { \ uint64_t __x = (x); \ uint64_t __m = (uint64_t)0xff << 56; \ - for (i = 0; !(__x & __m) && i != 8; i++) \ + for ((i) = 0; !(__x & __m) && (i) != 8; (i)++) \ __m >>= 8; \ } while (0) #endif @@ -231,7 +231,8 @@ __wt_vpack_int(uint8_t **pp, size_t maxlen, int64_t x) if (x < NEG_2BYTE_MIN) { *p = NEG_MULTI_MARKER; return (__wt_vpack_negint(pp, maxlen, (uint64_t)x)); - } else if (x < NEG_1BYTE_MIN) { + } + if (x < NEG_1BYTE_MIN) { WT_SIZE_CHECK_PACK(2, maxlen); x -= NEG_2BYTE_MIN; *p++ = NEG_2BYTE_MARKER | GET_BITS(x, 13, 8); @@ -358,12 +359,10 @@ __wt_vsize_uint(uint64_t x) { if (x <= POS_1BYTE_MAX) return (1); - else if (x <= POS_2BYTE_MAX + 1) { + if (x <= POS_2BYTE_MAX + 1) return (2); - } else { - x -= POS_2BYTE_MAX + 1; - return (__wt_vsize_posint(x)); - } + x -= POS_2BYTE_MAX + 1; + return (__wt_vsize_posint(x)); } /* @@ -373,13 +372,12 @@ __wt_vsize_uint(uint64_t x) static inline size_t __wt_vsize_int(int64_t x) { - if (x < NEG_2BYTE_MIN) { + if (x < NEG_2BYTE_MIN) return (__wt_vsize_negint((uint64_t)x)); - } else if (x < NEG_1BYTE_MIN) { + if (x < NEG_1BYTE_MIN) return (2); - } else if (x < 0) { + if (x < 0) return (1); - } else - /* For non-negative values, use the unsigned code above. */ - return (__wt_vsize_uint((uint64_t)x)); + /* For non-negative values, use the unsigned code above. */ + return (__wt_vsize_uint((uint64_t)x)); } diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 1b64186cbab..2d0f47988b7 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -9,8 +9,6 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ -#define WT_COMPILER_TYPE_ALIGN(x) - #define WT_PACKED_STRUCT_BEGIN(name) \ struct name { #define WT_PACKED_STRUCT_END \ @@ -31,9 +29,9 @@ __wt_atomic_fetch_add##name(type *vp, type v) \ { \ type orig; \ \ - old = *vp; \ + orig = *vp; \ *vp += v; \ - return (old); \ + return (orig); \ } \ static inline ret \ __wt_atomic_store##name(type *vp, type v) \ @@ -42,7 +40,7 @@ __wt_atomic_store##name(type *vp, type v) \ \ orig = *vp; \ *vp = v; \ - return (old); \ + return (orig); \ } \ static inline ret \ __wt_atomic_sub##name(type *vp, type v) \ @@ -51,9 +49,9 @@ __wt_atomic_sub##name(type *vp, type v) \ return (*vp); \ } \ static inline bool \ -__wt_atomic_cas##name(type *vp, type old, type new) \ +__wt_atomic_cas##name(type *vp, type orig, type new) \ { \ - if (*vp == old) { \ + if (*vp == orig) { \ *vp = new; \ return (true); \ } \ @@ -77,8 +75,8 @@ WT_ATOMIC_FUNC(size, size_t, size_t) * Pointer compare and swap. */ static inline bool -__wt_atomic_cas_ptr(void *vp, void *old, void *new) { - if (*(void **)vp == old) { +__wt_atomic_cas_ptr(void *vp, void *orig, void *new) { + if (*(void **)vp == orig) { *(void **)vp = new; return (true); } diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index b91c7dffe60..fb3c961417f 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -86,8 +86,8 @@ union __wt_lsn { * The high bit is reserved for the special states. If the high bit is * set (WT_LOG_SLOT_RESERVED) then we are guaranteed to be in a special state. */ -#define WT_LOG_SLOT_FREE -1 /* Not in use */ -#define WT_LOG_SLOT_WRITTEN -2 /* Slot data written, not processed */ +#define WT_LOG_SLOT_FREE (-1) /* Not in use */ +#define WT_LOG_SLOT_WRITTEN (-2) /* Slot data written, not processed */ /* * We allocate the buffer size, but trigger a slot switch when we cross @@ -144,8 +144,8 @@ union __wt_lsn { /* Slot is in use, but closed to new joins */ #define WT_LOG_SLOT_CLOSED(state) \ (WT_LOG_SLOT_ACTIVE(state) && \ - (FLD64_ISSET((uint64_t)state, WT_LOG_SLOT_CLOSE) && \ - !FLD64_ISSET((uint64_t)state, WT_LOG_SLOT_RESERVED))) + (FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ + !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_RESERVED))) /* Slot is in use, all data copied into buffer */ #define WT_LOG_SLOT_INPROGRESS(state) \ (WT_LOG_SLOT_RELEASED(state) != WT_LOG_SLOT_JOINED(state)) @@ -159,10 +159,11 @@ union __wt_lsn { !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ WT_LOG_SLOT_JOINED(state) < WT_LOG_SLOT_BUF_MAX) -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot { +struct __wt_logslot { + WT_CACHE_LINE_PAD_BEGIN volatile int64_t slot_state; /* Slot state */ int64_t slot_unbuffered; /* Unbuffered data in this slot */ - int32_t slot_error; /* Error value */ + int slot_error; /* Error value */ wt_off_t slot_start_offset; /* Starting file offset */ wt_off_t slot_last_offset; /* Last record offset */ WT_LSN slot_release_lsn; /* Slot release LSN */ @@ -176,14 +177,15 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot { #define WT_SLOT_SYNC 0x04 /* Needs sync on release */ #define WT_SLOT_SYNC_DIR 0x08 /* Directory sync on release */ uint32_t flags; /* Flags */ + WT_CACHE_LINE_PAD_END }; #define WT_SLOT_INIT_FLAGS 0 -#define WT_WITH_SLOT_LOCK(session, log, ret, op) do { \ +#define WT_WITH_SLOT_LOCK(session, log, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SLOT)); \ - WT_WITH_LOCK(session, ret, \ - &log->log_slot_lock, WT_SESSION_LOCKED_SLOT, op); \ + WT_WITH_LOCK_WAIT(session, \ + &(log)->log_slot_lock, WT_SESSION_LOCKED_SLOT, op); \ } while (0) struct __wt_myslot { @@ -191,7 +193,8 @@ struct __wt_myslot { wt_off_t end_offset; /* My end offset in buffer */ wt_off_t offset; /* Slot buffer offset */ #define WT_MYSLOT_CLOSE 0x01 /* This thread is closing the slot */ -#define WT_MYSLOT_UNBUFFERED 0x02 /* Write directly */ +#define WT_MYSLOT_NEEDS_RELEASE 0x02 /* This thread is releasing the slot */ +#define WT_MYSLOT_UNBUFFERED 0x04 /* Write directly */ uint32_t flags; /* Flags */ }; @@ -233,7 +236,7 @@ struct __wt_log { WT_SPINLOCK log_sync_lock; /* Locked: Single-thread fsync */ WT_SPINLOCK log_writelsn_lock; /* Locked: write LSN */ - WT_RWLOCK *log_archive_lock; /* Archive and log cursors */ + WT_RWLOCK log_archive_lock;/* Archive and log cursors */ /* Notify any waiting threads when sync_lsn is updated. */ WT_CONDVAR *log_sync_cond; @@ -252,6 +255,7 @@ struct __wt_log { #define WT_SLOT_POOL 128 WT_LOGSLOT *active_slot; /* Active slot */ WT_LOGSLOT slot_pool[WT_SLOT_POOL]; /* Pool of all slots */ + int32_t pool_index; /* Index into slot pool */ size_t slot_buf_size; /* Buffer size for slots */ #ifdef HAVE_DIAGNOSTIC uint64_t write_calls; /* Calls to log_write */ diff --git a/src/third_party/wiredtiger/src/include/lsm.h b/src/third_party/wiredtiger/src/include/lsm.h index b433e4c3c44..e3f6897ef9d 100644 --- a/src/third_party/wiredtiger/src/include/lsm.h +++ b/src/third_party/wiredtiger/src/include/lsm.h @@ -23,11 +23,14 @@ struct __wt_lsm_worker_cookie { struct __wt_lsm_worker_args { WT_SESSION_IMPL *session; /* Session */ WT_CONDVAR *work_cond; /* Owned by the manager */ + wt_thread_t tid; /* Thread id */ + bool tid_set; /* Thread id set */ + u_int id; /* My manager slot id */ uint32_t type; /* Types of operations handled */ -#define WT_LSM_WORKER_RUN 0x01 - uint32_t flags; /* Worker flags */ + + volatile bool running; /* Worker is running */ }; /* @@ -162,6 +165,9 @@ struct __wt_lsm_manager { #define WT_LSM_MAX_WORKERS 20 #define WT_LSM_MIN_WORKERS 3 WT_LSM_WORKER_ARGS lsm_worker_cookies[WT_LSM_MAX_WORKERS]; + +#define WT_LSM_MANAGER_SHUTDOWN 0x01 /* Manager has shut down */ + uint32_t flags; }; /* @@ -189,7 +195,7 @@ struct __wt_lsm_tree { #define LSM_TREE_MAX_QUEUE 100 uint32_t queue_ref; - WT_RWLOCK *rwlock; + WT_RWLOCK rwlock; TAILQ_ENTRY(__wt_lsm_tree) q; uint64_t dsk_gen; @@ -249,17 +255,21 @@ struct __wt_lsm_tree { int64_t lsm_merge_throttle; /* - * The tree is open for business. This used to be a flag, but it is - * susceptible to races. + * Following fields used to be flags but are susceptible to races. + * Don't merge them with flags. + */ + bool active; /* The tree is open for business */ + bool aggressive_timer_enabled; /* Timer for merge aggression enabled */ + bool need_switch; /* New chunk needs creating */ + + /* + * flags here are not protected for concurrent access, don't put + * anything here that is susceptible to races. */ - bool active; - -#define WT_LSM_TREE_AGGRESSIVE_TIMER 0x01 /* Timer for merge aggression */ -#define WT_LSM_TREE_COMPACTING 0x02 /* Tree being compacted */ -#define WT_LSM_TREE_MERGES 0x04 /* Tree should run merges */ -#define WT_LSM_TREE_NEED_SWITCH 0x08 /* New chunk needs creating */ -#define WT_LSM_TREE_OPEN 0x10 /* The tree is open */ -#define WT_LSM_TREE_THROTTLE 0x20 /* Throttle updates */ +#define WT_LSM_TREE_COMPACTING 0x01 /* Tree being compacted */ +#define WT_LSM_TREE_MERGES 0x02 /* Tree should run merges */ +#define WT_LSM_TREE_OPEN 0x04 /* The tree is open */ +#define WT_LSM_TREE_THROTTLE 0x08 /* Throttle updates */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 6d4a167a8e5..68ac2e339d0 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -32,8 +32,7 @@ * Optimize comparisons against the metafile URI, flag handles that reference * the metadata file. */ -#define WT_IS_METADATA(session, dh) \ - F_ISSET((dh), WT_DHANDLE_IS_METADATA) +#define WT_IS_METADATA(dh) F_ISSET((dh), WT_DHANDLE_IS_METADATA) #define WT_METAFILE_ID 0 /* Metadata file ID */ #define WT_METADATA_VERSION "WiredTiger version" /* Version keys */ @@ -43,9 +42,9 @@ * WT_WITH_TURTLE_LOCK -- * Acquire the turtle file lock, perform an operation, drop the lock. */ -#define WT_WITH_TURTLE_LOCK(session, ret, op) do { \ +#define WT_WITH_TURTLE_LOCK(session, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_TURTLE));\ - WT_WITH_LOCK(session, ret, \ + WT_WITH_LOCK_WAIT(session, \ &S2C(session)->turtle_lock, WT_SESSION_LOCKED_TURTLE, op); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 83e238b0529..9161a215fdc 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -63,7 +63,7 @@ #define WT_MAX(a, b) ((a) < (b) ? (b) : (a)) /* Elements in an array. */ -#define WT_ELEMENTS(a) (sizeof(a) / sizeof(a[0])) +#define WT_ELEMENTS(a) (sizeof(a) / sizeof((a)[0])) /* 10 level skip lists, 1/4 have a link to the next element. */ #define WT_SKIP_MAXDEPTH 10 @@ -140,6 +140,7 @@ #define F_CLR(p, mask) FLD_CLR((p)->flags, mask) #define F_ISSET(p, mask) FLD_ISSET((p)->flags, mask) +#define F_ISSET_ALL(p, mask) (FLD_MASK((p)->flags, mask) == (mask)) #define F_MASK(p, mask) FLD_MASK((p)->flags, mask) #define F_SET(p, mask) FLD_SET((p)->flags, mask) @@ -180,14 +181,14 @@ */ #define WT_BINARY_SEARCH(key, arrayp, n, found) do { \ uint32_t __base, __indx, __limit; \ - found = false; \ + (found) = false; \ for (__base = 0, __limit = (n); __limit != 0; __limit >>= 1) { \ __indx = __base + (__limit >> 1); \ - if ((arrayp)[__indx] < key) { \ + if ((arrayp)[__indx] < (key)) { \ __base = __indx + 1; \ --__limit; \ - } else if ((arrayp)[__indx] == key) { \ - found = true; \ + } else if ((arrayp)[__indx] == (key)) { \ + (found) = true; \ break; \ } \ } \ @@ -206,8 +207,8 @@ /* Check if a string matches a prefix. */ #define WT_PREFIX_MATCH(str, pfx) \ - (((const char *)(str))[0] == ((const char *)pfx)[0] && \ - strncmp((str), (pfx), strlen(pfx)) == 0) + (((const char *)(str))[0] == ((const char *)(pfx))[0] && \ + strncmp(str, pfx, strlen(pfx)) == 0) /* Check if a string matches a prefix, and move past it. */ #define WT_PREFIX_SKIP(str, pfx) \ @@ -224,8 +225,8 @@ /* Check if a string matches a byte string of len bytes. */ #define WT_STRING_MATCH(str, bytes, len) \ - (((const char *)str)[0] == ((const char *)bytes)[0] && \ - strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0') + (((const char *)(str))[0] == ((const char *)(bytes))[0] && \ + strncmp(str, bytes, len) == 0 && (str)[len] == '\0') /* * Macro that produces a string literal that isn't wrapped in quotes, to avoid @@ -275,6 +276,3 @@ union __wt_rand_state { uint32_t w, z; } x; }; - -/* Shared array for converting to hex */ -extern const u_char __wt_hex[]; diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index befd480e085..7040886cf82 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -11,11 +11,22 @@ * Wait on a mutex, optionally timing out. */ static inline void -__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs) +__wt_cond_wait(WT_SESSION_IMPL *session, + WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *)) { bool notused; - __wt_cond_wait_signal(session, cond, usecs, ¬used); + __wt_cond_wait_signal(session, cond, usecs, run_func, ¬used); +} + +/* + * __wt_hex -- + * Convert a byte to a hex character. + */ +static inline u_char +__wt_hex(int c) +{ + return ((u_char)"0123456789abcdef"[c]); } /* @@ -75,3 +86,94 @@ __wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...) WT_UNUSED(fmt); #endif } + +/* + * __wt_snprintf -- + * snprintf convenience function, ignoring the returned size. + */ +static inline int +__wt_snprintf(char *buf, size_t size, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) +{ + WT_DECL_RET; + size_t len; + va_list ap; + + len = 0; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_incr(buf, size, &len, fmt, ap); + va_end(ap); + WT_RET(ret); + + /* It's an error if the buffer couldn't hold everything. */ + return (len >= size ? ERANGE : 0); +} + +/* + * __wt_vsnprintf -- + * vsnprintf convenience function, ignoring the returned size. + */ +static inline int +__wt_vsnprintf(char *buf, size_t size, const char *fmt, va_list ap) +{ + size_t len; + + len = 0; + + WT_RET(__wt_vsnprintf_len_incr(buf, size, &len, fmt, ap)); + + /* It's an error if the buffer couldn't hold everything. */ + return (len >= size ? ERANGE : 0); +} + +/* + * __wt_snprintf_len_set -- + * snprintf convenience function, setting the returned size. + */ +static inline int +__wt_snprintf_len_set( + char *buf, size_t size, size_t *retsizep, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) +{ + WT_DECL_RET; + va_list ap; + + *retsizep = 0; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap); + va_end(ap); + return (ret); +} + +/* + * __wt_vsnprintf_len_set -- + * vsnprintf convenience function, setting the returned size. + */ +static inline int +__wt_vsnprintf_len_set( + char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) +{ + *retsizep = 0; + + return (__wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap)); +} + +/* + * __wt_snprintf_len_incr -- + * snprintf convenience function, incrementing the returned size. + */ +static inline int +__wt_snprintf_len_incr( + char *buf, size_t size, size_t *retsizep, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) +{ + WT_DECL_RET; + va_list ap; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap); + va_end(ap); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index d5be5bd8c60..6c5c8b67647 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -19,8 +19,6 @@ /* * Add MSVC-specific attributes and pragmas to types and function declarations. */ -#define WT_COMPILER_TYPE_ALIGN(x) __declspec(align(x)) - #define WT_PACKED_STRUCT_BEGIN(name) \ __pragma(pack(push,1)) \ struct name { diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index 84f015d6b67..36acea810d9 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -21,41 +21,37 @@ struct __wt_condvar { int waiters; /* Numbers of waiters, or -1 if signalled with no waiters. */ /* - * The following fields are only used for automatically adjusting - * condition variables. They could be in a separate structure. + * The following fields are used for automatically adjusting condition + * variable wait times. */ uint64_t min_wait; /* Minimum wait duration */ uint64_t max_wait; /* Maximum wait duration */ uint64_t prev_wait; /* Wait duration used last time */ }; -/* - * !!! - * Don't modify this structure without understanding the read/write locking - * functions. - */ -typedef union { /* Read/write lock */ - uint64_t u; - struct { - uint32_t wr; /* Writers and readers */ - } i; - struct { - uint16_t writers; /* Now serving for writers */ - uint16_t readers; /* Now serving for readers */ - uint16_t next; /* Next available ticket number */ - uint16_t writers_active;/* Count of active writers */ - } s; -} wt_rwlock_t; - /* * Read/write locks: * * WiredTiger uses read/write locks for shared/exclusive access to resources. + * !!! + * Don't modify this structure without understanding the read/write locking + * functions. */ -struct __wt_rwlock { - const char *name; /* Lock name for debugging */ - - wt_rwlock_t rwlock; /* Read/write lock */ +struct __wt_rwlock { /* Read/write lock */ + volatile union { + uint64_t v; /* Full 64-bit value */ + struct { + uint8_t current; /* Current ticket */ + uint8_t next; /* Next available ticket */ + uint8_t reader; /* Read queue ticket */ + uint8_t __notused; /* Padding */ + uint16_t readers_active;/* Count of active readers */ + uint16_t readers_queued;/* Count of queued readers */ + } s; + } u; + + WT_CONDVAR *cond_readers; /* Blocking readers */ + WT_CONDVAR *cond_writers; /* Blocking writers */ }; /* @@ -70,28 +66,17 @@ struct __wt_rwlock { #define SPINLOCK_PTHREAD_MUTEX 2 #define SPINLOCK_PTHREAD_MUTEX_ADAPTIVE 3 +struct __wt_spinlock { #if SPINLOCK_TYPE == SPINLOCK_GCC - -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { + WT_CACHE_LINE_PAD_BEGIN volatile int lock; - - /* - * We track acquisitions and time spent waiting for some locks. For - * performance reasons and to make it possible to write generic code - * that tracks statistics for different locks, we store the offset - * of the statistics fields to be updated during lock acquisition. - */ - int16_t stat_count_off; /* acquisitions offset */ - int16_t stat_app_usecs_off; /* waiting application threads offset */ - int16_t stat_int_usecs_off; /* waiting server threads offset */ -}; - #elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\ SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE ||\ SPINLOCK_TYPE == SPINLOCK_MSVC - -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { wt_mutex_t lock; +#else +#error Unknown spinlock type +#endif const char *name; /* Mutex name */ @@ -106,10 +91,8 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { int16_t stat_int_usecs_off; /* waiting server threads offset */ int8_t initialized; /* Lock initialized, for cleanup */ -}; - -#else - -#error Unknown spinlock type +#if SPINLOCK_TYPE == SPINLOCK_GCC + WT_CACHE_LINE_PAD_END #endif +}; diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i index a6309e0976b..2d483972ed2 100644 --- a/src/third_party/wiredtiger/src/include/mutex.i +++ b/src/third_party/wiredtiger/src/include/mutex.i @@ -14,6 +14,18 @@ * of instructions. */ +/* + * __spin_init_internal -- + * Initialize the WT portion of a spinlock. + */ +static inline void +__spin_init_internal(WT_SPINLOCK *t, const char *name) +{ + t->name = name; + t->stat_count_off = t->stat_app_usecs_off = t->stat_int_usecs_off = -1; + t->initialized = 1; +} + #if SPINLOCK_TYPE == SPINLOCK_GCC /* Default to spinning 1000 times before yielding. */ @@ -29,10 +41,9 @@ static inline int __wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t, const char *name) { WT_UNUSED(session); - WT_UNUSED(name); t->lock = 0; - t->stat_count_off = t->stat_app_usecs_off = t->stat_int_usecs_off = -1; + __spin_init_internal(t, name); return (0); } @@ -110,10 +121,7 @@ __wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t, const char *name) #else WT_RET(pthread_mutex_init(&t->lock, NULL)); #endif - - t->name = name; - t->stat_count_off = t->stat_app_usecs_off = t->stat_int_usecs_off = -1; - t->initialized = 1; + __spin_init_internal(t, name); WT_UNUSED(session); return (0); @@ -195,8 +203,7 @@ __wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t, const char *name) return (__wt_map_windows_error(windows_error)); } - t->name = name; - t->initialized = 1; + __spin_init_internal(t, name); return (0); } @@ -300,3 +307,22 @@ __wt_spin_lock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) } else __wt_spin_lock(session, t); } + +/* + * __wt_spin_trylock_track -- + * Try to lock a spinlock or fail immediately if it is busy. + * Track if successful. + */ +static inline int +__wt_spin_trylock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + int64_t **stats; + + if (t->stat_count_off != -1 && WT_STAT_ENABLED(session)) { + WT_RET(__wt_spin_trylock(session, t)); + stats = (int64_t **)S2C(session)->stats; + stats[session->stat_bucket][t->stat_count_off]++; + return (0); + } + return (__wt_spin_trylock(session, t)); +} diff --git a/src/third_party/wiredtiger/src/include/os.h b/src/third_party/wiredtiger/src/include/os.h index 7a8e47ed81f..73d89268392 100644 --- a/src/third_party/wiredtiger/src/include/os.h +++ b/src/third_party/wiredtiger/src/include/os.h @@ -11,8 +11,14 @@ * A call returning 0 indicates success; any call where \ * 0 is not the only successful return must provide an \ * expression evaluating to 0 in all successful cases. \ + * \ + * XXX \ + * Casting the call's return to int is because CentOS 7.3.1611 \ + * complains about syscall returning a long and the loss of \ + * integer precision in the assignment to ret. The cast should \ + * be a no-op everywhere. \ */ \ - if (((ret) = (call)) == 0) \ + if (((ret) = (int)(call)) == 0) \ break; \ /* \ * The call's error was either returned by the call or \ @@ -61,7 +67,7 @@ #define WT_TIMECMP(t1, t2) \ ((t1).tv_sec < (t2).tv_sec ? -1 : \ - (t1).tv_sec == (t2.tv_sec) ? \ + (t1).tv_sec == (t2).tv_sec ? \ (t1).tv_nsec < (t2).tv_nsec ? -1 : \ (t1).tv_nsec == (t2).tv_nsec ? 0 : 1 : 1) diff --git a/src/third_party/wiredtiger/src/include/os_windows.h b/src/third_party/wiredtiger/src/include/os_windows.h index 65938ac9f17..c1e5f788dc6 100644 --- a/src/third_party/wiredtiger/src/include/os_windows.h +++ b/src/third_party/wiredtiger/src/include/os_windows.h @@ -43,16 +43,6 @@ typedef uint32_t u_int; typedef unsigned char u_char; typedef uint64_t u_long; -/* <= VS 2013 is not C99 compat */ -#if _MSC_VER < 1900 -#define snprintf _wt_snprintf - -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...); -#endif - /* * Windows does have ssize_t * Python headers declare also though so we need to guard it @@ -61,18 +51,6 @@ _Check_return_opt_ int __cdecl _wt_snprintf( typedef int ssize_t; #endif -/* - * Provide a custom version of vsnprintf that returns the - * needed buffer length instead of -1 on truncation - */ -#define vsnprintf _wt_vsnprintf - -_Check_return_opt_ int __cdecl _wt_vsnprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, - va_list _ArgList); - /* Provide a custom version of localtime_r */ struct tm *localtime_r(const time_t* timer, struct tm* result); diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index 17ca261bcfc..0eadb2f2027 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -104,8 +104,8 @@ __pack_name_next(WT_PACK_NAME *pn, WT_CONFIG_ITEM *name) WT_CONFIG_ITEM ignore; if (pn->genname) { - (void)snprintf(pn->buf, sizeof(pn->buf), - (pn->iskey ? "key%d" : "value%d"), pn->count); + WT_RET(__wt_snprintf(pn->buf, sizeof(pn->buf), + (pn->iskey ? "key%d" : "value%d"), pn->count)); WT_CLEAR(*name); name->str = pn->buf; name->len = strlen(pn->buf); @@ -168,10 +168,15 @@ next: if (pack->cur == pack->end) (int)(pack->end - pack->orig), pack->orig); return (0); case 'u': - case 'U': /* Special case for items with a size prefix. */ pv->type = (!pv->havesize && *pack->cur != '\0') ? 'U' : 'u'; return (0); + case 'U': + /* + * Don't change the type. 'U' is used internally, so this type + * was already changed to explicitly include the size. + */ + return (0); case 'b': case 'h': case 'i': @@ -193,7 +198,7 @@ next: if (pack->cur == pack->end) return (0); default: WT_RET_MSG(pack->session, EINVAL, - "Invalid type '%c' found in format '%.*s'", + "Invalid type '%c' found in format '%.*s'", pv->type, (int)(pack->end - pack->orig), pack->orig); } @@ -201,43 +206,43 @@ next: if (pack->cur == pack->end) #define WT_PACK_GET(session, pv, ap) do { \ WT_ITEM *__item; \ - switch (pv.type) { \ + switch ((pv).type) { \ case 'x': \ break; \ case 's': \ case 'S': \ - pv.u.s = va_arg(ap, const char *); \ + (pv).u.s = va_arg(ap, const char *); \ break; \ case 'U': \ case 'u': \ __item = va_arg(ap, WT_ITEM *); \ - pv.u.item.data = __item->data; \ - pv.u.item.size = __item->size; \ + (pv).u.item.data = __item->data; \ + (pv).u.item.size = __item->size; \ break; \ case 'b': \ case 'h': \ case 'i': \ - pv.u.i = va_arg(ap, int); \ + (pv).u.i = va_arg(ap, int); \ break; \ case 'B': \ case 'H': \ case 'I': \ case 't': \ - pv.u.u = va_arg(ap, unsigned int); \ + (pv).u.u = va_arg(ap, unsigned int); \ break; \ case 'l': \ - pv.u.i = va_arg(ap, long); \ + (pv).u.i = va_arg(ap, long); \ break; \ case 'L': \ - pv.u.u = va_arg(ap, unsigned long); \ + (pv).u.u = va_arg(ap, unsigned long); \ break; \ case 'q': \ - pv.u.i = va_arg(ap, int64_t); \ + (pv).u.i = va_arg(ap, int64_t); \ break; \ case 'Q': \ case 'r': \ case 'R': \ - pv.u.u = va_arg(ap, uint64_t); \ + (pv).u.u = va_arg(ap, uint64_t); \ break; \ /* User format strings have already been validated. */ \ WT_ILLEGAL_VALUE(session); \ @@ -551,47 +556,47 @@ __unpack_read(WT_SESSION_IMPL *session, #define WT_UNPACK_PUT(session, pv, ap) do { \ WT_ITEM *__item; \ - switch (pv.type) { \ + switch ((pv).type) { \ case 'x': \ break; \ case 's': \ case 'S': \ - *va_arg(ap, const char **) = pv.u.s; \ + *va_arg(ap, const char **) = (pv).u.s; \ break; \ case 'U': \ case 'u': \ __item = va_arg(ap, WT_ITEM *); \ - __item->data = pv.u.item.data; \ - __item->size = pv.u.item.size; \ + __item->data = (pv).u.item.data; \ + __item->size = (pv).u.item.size; \ break; \ case 'b': \ - *va_arg(ap, int8_t *) = (int8_t)pv.u.i; \ + *va_arg(ap, int8_t *) = (int8_t)(pv).u.i; \ break; \ case 'h': \ - *va_arg(ap, int16_t *) = (short)pv.u.i; \ + *va_arg(ap, int16_t *) = (short)(pv).u.i; \ break; \ case 'i': \ case 'l': \ - *va_arg(ap, int32_t *) = (int32_t)pv.u.i; \ + *va_arg(ap, int32_t *) = (int32_t)(pv).u.i; \ break; \ case 'q': \ - *va_arg(ap, int64_t *) = pv.u.i; \ + *va_arg(ap, int64_t *) = (pv).u.i; \ break; \ case 'B': \ case 't': \ - *va_arg(ap, uint8_t *) = (uint8_t)pv.u.u; \ + *va_arg(ap, uint8_t *) = (uint8_t)(pv).u.u; \ break; \ case 'H': \ - *va_arg(ap, uint16_t *) = (uint16_t)pv.u.u; \ + *va_arg(ap, uint16_t *) = (uint16_t)(pv).u.u; \ break; \ case 'I': \ case 'L': \ - *va_arg(ap, uint32_t *) = (uint32_t)pv.u.u; \ + *va_arg(ap, uint32_t *) = (uint32_t)(pv).u.u; \ break; \ case 'Q': \ case 'r': \ case 'R': \ - *va_arg(ap, uint64_t *) = pv.u.u; \ + *va_arg(ap, uint64_t *) = (pv).u.u; \ break; \ /* User format strings have already been validated. */ \ WT_ILLEGAL_VALUE(session); \ diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 6a5ce67a867..50e141d9921 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -78,6 +78,14 @@ struct __wt_table { */ #define WT_COLGROUPS(t) WT_MAX((t)->ncolgroups, 1) +/* Helpers for the locked state of the handle list and table locks. */ +#define WT_SESSION_LOCKED_HANDLE_LIST \ + (WT_SESSION_LOCKED_HANDLE_LIST_READ | \ + WT_SESSION_LOCKED_HANDLE_LIST_WRITE) +#define WT_SESSION_LOCKED_TABLE \ + (WT_SESSION_LOCKED_TABLE_READ | \ + WT_SESSION_LOCKED_TABLE_WRITE) + /* * WT_WITH_LOCK_WAIT -- * Wait for a lock, perform an operation, drop the lock. @@ -85,7 +93,7 @@ struct __wt_table { #define WT_WITH_LOCK_WAIT(session, lock, flag, op) do { \ if (F_ISSET(session, (flag))) { \ op; \ - } else { \ + } else { \ __wt_spin_lock_track(session, lock); \ F_SET(session, (flag)); \ op; \ @@ -95,76 +103,159 @@ struct __wt_table { } while (0) /* - * WT_WITH_LOCK -- - * Acquire a lock, perform an operation, drop the lock. + * WT_WITH_LOCK_NOWAIT -- + * Acquire a lock if available, perform an operation, drop the lock. */ -#define WT_WITH_LOCK(session, ret, lock, flag, op) do { \ - ret = 0; \ - if (!F_ISSET(session, (flag)) && \ - F_ISSET(session, WT_SESSION_LOCK_NO_WAIT)) { \ - if ((ret = __wt_spin_trylock(session, lock)) == 0) { \ - F_SET(session, (flag)); \ - op; \ - F_CLR(session, (flag)); \ - __wt_spin_unlock(session, lock); \ - } \ - } else \ - WT_WITH_LOCK_WAIT(session, lock, flag, op); \ +#define WT_WITH_LOCK_NOWAIT(session, ret, lock, flag, op) do { \ + (ret) = 0; \ + if (F_ISSET(session, (flag))) { \ + op; \ + } else if (((ret) = \ + __wt_spin_trylock_track(session, lock)) == 0) { \ + F_SET(session, (flag)); \ + op; \ + F_CLR(session, (flag)); \ + __wt_spin_unlock(session, lock); \ + } \ } while (0) /* - * WT_WITH_CHECKPOINT_LOCK -- + * WT_WITH_CHECKPOINT_LOCK, WT_WITH_CHECKPOINT_LOCK_NOWAIT -- * Acquire the checkpoint lock, perform an operation, drop the lock. */ -#define WT_WITH_CHECKPOINT_LOCK(session, ret, op) \ - WT_WITH_LOCK(session, ret, \ +#define WT_WITH_CHECKPOINT_LOCK(session, op) \ + WT_WITH_LOCK_WAIT(session, \ + &S2C(session)->checkpoint_lock, WT_SESSION_LOCKED_CHECKPOINT, op) +#define WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, op) \ + WT_WITH_LOCK_NOWAIT(session, ret, \ &S2C(session)->checkpoint_lock, WT_SESSION_LOCKED_CHECKPOINT, op) /* - * WT_WITH_HANDLE_LIST_LOCK -- - * Acquire the data handle list lock, perform an operation, drop the lock. + * WT_WITH_HANDLE_LIST_READ_LOCK -- + * Acquire the data handle list lock in shared mode, perform an operation, + * drop the lock. The handle list lock is a read-write lock so the + * implementation is different to the other lock macros. * * Note: always waits because some operations need the handle list lock to * discard handles, and we only expect it to be held across short * operations. */ -#define WT_WITH_HANDLE_LIST_LOCK(session, op) \ - WT_WITH_LOCK_WAIT(session, \ - &S2C(session)->dhandle_lock, WT_SESSION_LOCKED_HANDLE_LIST, op) +#define WT_WITH_HANDLE_LIST_READ_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)) { \ + op; \ + } else { \ + __wt_readlock(session, &S2C(session)->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + __wt_readunlock(session, &S2C(session)->dhandle_lock); \ + } \ +} while (0) + +/* + * WT_WITH_HANDLE_LIST_WRITE_LOCK -- + * Acquire the data handle list lock in exclusive mode, perform an + * operation, drop the lock. The handle list lock is a read-write lock so + * the implementation is different to the other lock macros. + */ +#define WT_WITH_HANDLE_LIST_WRITE_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ));\ + __wt_writelock(session, &S2C(session)->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + __wt_writeunlock(session, &S2C(session)->dhandle_lock); \ + } \ +} while (0) /* * WT_WITH_METADATA_LOCK -- * Acquire the metadata lock, perform an operation, drop the lock. */ -#define WT_WITH_METADATA_LOCK(session, ret, op) \ - WT_WITH_LOCK(session, ret, \ +#define WT_WITH_METADATA_LOCK(session, op) \ + WT_WITH_LOCK_WAIT(session, \ &S2C(session)->metadata_lock, WT_SESSION_LOCKED_METADATA, op) /* - * WT_WITH_SCHEMA_LOCK -- + * WT_WITH_SCHEMA_LOCK, WT_WITH_SCHEMA_LOCK_NOWAIT -- * Acquire the schema lock, perform an operation, drop the lock. * Check that we are not already holding some other lock: the schema lock * must be taken first. */ -#define WT_WITH_SCHEMA_LOCK(session, ret, op) do { \ +#define WT_WITH_SCHEMA_LOCK(session, op) do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ + !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST | \ + WT_SESSION_NO_SCHEMA_LOCK | WT_SESSION_LOCKED_TABLE)); \ + WT_WITH_LOCK_WAIT(session, \ + &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ +} while (0) +#define WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, op) do { \ WT_ASSERT(session, \ F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST | \ WT_SESSION_NO_SCHEMA_LOCK | WT_SESSION_LOCKED_TABLE)); \ - WT_WITH_LOCK(session, ret, \ + WT_WITH_LOCK_NOWAIT(session, ret, \ &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ } while (0) /* - * WT_WITH_TABLE_LOCK -- + * WT_WITH_TABLE_READ_LOCK, WT_WITH_TABLE_WRITE_LOCK, + * WT_WITH_TABLE_WRITE_LOCK_NOWAIT -- * Acquire the table lock, perform an operation, drop the lock. + * The table lock is a read-write lock so the implementation is different + * to most other lock macros. + * + * Note: readlock always waits because some operations need the table lock + * to discard handles, and we only expect it to be held across short + * operations. */ -#define WT_WITH_TABLE_LOCK(session, ret, op) do { \ +#define WT_WITH_TABLE_READ_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_TABLE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); \ + __wt_readlock(session, &S2C(session)->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_READ); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_READ); \ + __wt_readunlock(session, &S2C(session)->table_lock); \ + } \ +} while (0) + +#define WT_WITH_TABLE_WRITE_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ | \ + WT_SESSION_LOCKED_HANDLE_LIST)); \ + __wt_writelock(session, &S2C(session)->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + __wt_writeunlock(session, &S2C(session)->table_lock); \ + } \ +} while (0) +#define WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, op) do { \ WT_ASSERT(session, \ - F_ISSET(session, WT_SESSION_LOCKED_TABLE) || \ - !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); \ - WT_WITH_LOCK(session, ret, \ - &S2C(session)->table_lock, WT_SESSION_LOCKED_TABLE, op); \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE) || \ + !F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ | \ + WT_SESSION_LOCKED_HANDLE_LIST)); \ + if (F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)) { \ + op; \ + } else if (((ret) = __wt_try_writelock(session, \ + &S2C(session)->table_lock)) == 0) { \ + F_SET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + __wt_writeunlock(session, &S2C(session)->table_lock); \ + } \ } while (0) /* @@ -176,19 +267,31 @@ struct __wt_table { WT_CONNECTION_IMPL *__conn = S2C(session); \ bool __checkpoint_locked = \ F_ISSET(session, WT_SESSION_LOCKED_CHECKPOINT); \ - bool __handle_locked = \ - F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST); \ - bool __table_locked = \ - F_ISSET(session, WT_SESSION_LOCKED_TABLE); \ + bool __handle_read_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + bool __handle_write_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + bool __table_read_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ); \ + bool __table_write_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ bool __schema_locked = \ F_ISSET(session, WT_SESSION_LOCKED_SCHEMA); \ - if (__handle_locked) { \ - F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST); \ - __wt_spin_unlock(session, &__conn->dhandle_lock); \ + if (__handle_read_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + __wt_readunlock(session, &__conn->dhandle_lock); \ + } \ + if (__handle_write_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + __wt_writeunlock(session, &__conn->dhandle_lock); \ + } \ + if (__table_read_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_READ); \ + __wt_readunlock(session, &__conn->table_lock); \ } \ - if (__table_locked) { \ - F_CLR(session, WT_SESSION_LOCKED_TABLE); \ - __wt_spin_unlock(session, &__conn->table_lock); \ + if (__table_write_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + __wt_writeunlock(session, &__conn->table_lock); \ } \ if (__schema_locked) { \ F_CLR(session, WT_SESSION_LOCKED_SCHEMA); \ @@ -207,12 +310,20 @@ struct __wt_table { __wt_spin_lock(session, &__conn->schema_lock); \ F_SET(session, WT_SESSION_LOCKED_SCHEMA); \ } \ - if (__table_locked) { \ - __wt_spin_lock(session, &__conn->table_lock); \ - F_SET(session, WT_SESSION_LOCKED_TABLE); \ + if (__table_read_locked) { \ + __wt_readlock(session, &__conn->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_READ); \ + } \ + if (__table_write_locked) { \ + __wt_writelock(session, &__conn->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + } \ + if (__handle_read_locked) { \ + __wt_readlock(session, &__conn->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ } \ - if (__handle_locked) { \ - __wt_spin_lock(session, &__conn->dhandle_lock); \ - F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST); \ + if (__handle_write_locked) { \ + __wt_writelock(session, &__conn->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ } \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/serial.i b/src/third_party/wiredtiger/src/include/serial.i index 982f196b0b8..0134e1a9c20 100644 --- a/src/third_party/wiredtiger/src/include/serial.i +++ b/src/third_party/wiredtiger/src/include/serial.i @@ -154,7 +154,7 @@ __col_append_serial_func(WT_SESSION_IMPL *session, WT_INSERT_HEAD *ins_head, static inline int __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **new_insp, - size_t new_ins_size, uint64_t *recnop, u_int skipdepth) + size_t new_ins_size, uint64_t *recnop, u_int skipdepth, bool exclusive) { WT_INSERT *new_ins = *new_insp; WT_DECL_RET; @@ -165,11 +165,16 @@ __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, /* Clear references to memory we now own and must free on error. */ *new_insp = NULL; - /* Acquire the page's spinlock, call the worker function. */ - WT_PAGE_LOCK(session, page); + /* + * Acquire the page's spinlock unless we already have exclusive access. + * Then call the worker function. + */ + if (!exclusive) + WT_PAGE_LOCK(session, page); ret = __col_append_serial_func( session, ins_head, ins_stack, new_ins, recnop, skipdepth); - WT_PAGE_UNLOCK(session, page); + if (!exclusive) + WT_PAGE_UNLOCK(session, page); if (ret != 0) { /* Free unused memory on error. */ @@ -198,7 +203,7 @@ __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, static inline int __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **new_insp, - size_t new_ins_size, u_int skipdepth) + size_t new_ins_size, u_int skipdepth, bool exclusive) { WT_INSERT *new_ins = *new_insp; WT_DECL_RET; @@ -220,10 +225,12 @@ __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, ret = __insert_simple_func( session, ins_stack, new_ins, skipdepth); else { - WT_PAGE_LOCK(session, page); + if (!exclusive) + WT_PAGE_LOCK(session, page); ret = __insert_serial_func( session, ins_head, ins_stack, new_ins, skipdepth); - WT_PAGE_UNLOCK(session, page); + if (!exclusive) + WT_PAGE_UNLOCK(session, page); } if (ret != 0) { @@ -252,7 +259,8 @@ __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, */ static inline int __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, - WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size) + WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size, + bool exclusive) { WT_DECL_RET; WT_UPDATE *obsolete, *upd = *updp; @@ -295,7 +303,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, /* * If there are no subsequent WT_UPDATE structures we are done here. */ - if (upd->next == NULL) + if (upd->next == NULL || exclusive) return (0); /* @@ -316,11 +324,11 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, } /* If we can't lock it, don't scan, that's okay. */ - if (__wt_try_writelock(session, &page->page_lock) != 0) + if (WT_PAGE_TRYLOCK(session, page) != 0) return (0); obsolete = __wt_update_obsolete_check(session, page, upd->next); - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); if (obsolete != NULL) __wt_update_obsolete_free(session, page, obsolete); diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 3f9f495c134..1b2dfd1ed2b 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -23,7 +23,7 @@ struct __wt_data_handle_cache { * A hazard pointer. */ struct __wt_hazard { - WT_PAGE *page; /* Page address */ + WT_REF *ref; /* Page reference */ #ifdef HAVE_DIAGNOSTIC const char *file; /* File/line where hazard acquired */ int line; @@ -41,7 +41,7 @@ struct __wt_hazard { * WT_SESSION_IMPL -- * Implementation of WT_SESSION. */ -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { +struct __wt_session_impl { WT_SESSION iface; void *lang_private; /* Language specific private storage */ @@ -52,8 +52,6 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { const char *lastop; /* Last operation */ uint32_t id; /* UID, offset in session array */ - WT_CONDVAR *cond; /* Condition variable */ - WT_EVENT_HANDLER *event_handler;/* Application's event handlers */ WT_DATA_HANDLE *dhandle; /* Current data handle */ @@ -69,13 +67,12 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { TAILQ_HEAD(__dhandles, __wt_data_handle_cache) dhandles; time_t last_sweep; /* Last sweep for dead handles */ - WT_CURSOR *cursor; /* Current cursor */ /* Cursors closed with the session */ TAILQ_HEAD(__cursors, __wt_cursor) cursors; WT_CURSOR_BACKUP *bkp_cursor; /* Hot backup cursor */ - WT_COMPACT *compact; /* Compaction information */ + WT_COMPACT_STATE *compact; /* Compaction information */ enum { WT_COMPACT_NONE=0, WT_COMPACT_RUNNING, WT_COMPACT_SUCCESS } compact_state; @@ -90,7 +87,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { void *meta_track_sub; /* Child transaction / save point */ size_t meta_track_alloc; /* Currently allocated */ int meta_track_nest; /* Nesting level of meta transaction */ -#define WT_META_TRACKING(session) (session->meta_track_next != NULL) +#define WT_META_TRACKING(session) ((session)->meta_track_next != NULL) /* * Each session keeps a cache of table handles. The set of handles @@ -100,6 +97,10 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { */ TAILQ_HEAD(__tables, __wt_table) tables; + /* Current rwlock for callback. */ + WT_RWLOCK *current_rwlock; + uint8_t current_rwticket; + WT_ITEM **scratch; /* Temporary memory for any function */ u_int scratch_alloc; /* Currently allocated */ size_t scratch_cached; /* Scratch bytes cached */ @@ -153,20 +154,16 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { uint32_t flags; /* - * The split stash memory and hazard information persist past session - * close because they are accessed by threads of control other than the - * thread owning the session. - * + * All of the following fields live at the end of the structure so it's + * easier to clear everything but the fields that persist. + */ +#define WT_SESSION_CLEAR_SIZE (offsetof(WT_SESSION_IMPL, rnd)) + + /* * The random number state persists past session close because we don't - * want to repeatedly allocate repeated values for skiplist depth if the + * want to repeatedly use the same values for skiplist depth when the * application isn't caching sessions. - * - * All of these fields live at the end of the structure so it's easier - * to clear everything but the fields that persist. */ -#define WT_SESSION_CLEAR_SIZE(s) \ - (WT_PTRDIFF(&(s)->rnd, s)) - WT_RAND_STATE rnd; /* Random number generation state */ /* Hashed handle reference list array */ @@ -175,6 +172,9 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { TAILQ_HEAD(__tables_hash, __wt_table) *tablehash; /* + * Split stash memory persists past session close because it's accessed + * by threads of control other than the thread owning the session. + * * Splits can "free" memory that may still be in use, and we use a * split generation number to track it, that is, the session stores a * reference to the memory and allocates a split generation; when no @@ -194,15 +194,22 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { /* * Hazard pointers. * + * Hazard information persists past session close because it's accessed + * by threads of control other than the thread owning the session. + * * Use the non-NULL state of the hazard field to know if the session has * previously been initialized. */ #define WT_SESSION_FIRST_USE(s) \ ((s)->hazard == NULL) - /* The number of hazard pointers grows dynamically. */ -#define WT_HAZARD_INCR 1 - uint32_t hazard_size; /* Allocated slots in hazard array. */ + /* + * The hazard pointer array grows as necessary, initialize with 250 + * slots. + */ +#define WT_SESSION_INITIAL_HAZARD_SLOTS 250 + uint32_t hazard_size; /* Hazard pointer array slots */ + uint32_t hazard_inuse; /* Hazard pointer array slots in-use */ uint32_t nhazard; /* Count of active hazard pointers */ WT_HAZARD *hazard; /* Hazard pointer array */ }; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index e53414fc0c9..6c274484bcb 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -72,7 +72,7 @@ * and the session ID is a small, monotonically increasing number. */ #define WT_STATS_SLOT_ID(session) \ - ((session)->id) % WT_COUNTER_SLOTS + (((session)->id) % WT_COUNTER_SLOTS) /* * Statistic structures are arrays of int64_t's. We have functions to read/write @@ -310,10 +310,15 @@ struct __wt_connection_stats { int64_t cache_eviction_slow; int64_t cache_eviction_state; int64_t cache_eviction_walks_abandoned; + int64_t cache_eviction_active_workers; + int64_t cache_eviction_worker_created; int64_t cache_eviction_worker_evicting; + int64_t cache_eviction_worker_removed; + int64_t cache_eviction_stable_state_workers; int64_t cache_eviction_force_fail; int64_t cache_eviction_walks_active; int64_t cache_eviction_walks_started; + int64_t cache_eviction_force_retune; int64_t cache_eviction_hazard; int64_t cache_hazard_checks; int64_t cache_hazard_walks; @@ -388,9 +393,7 @@ struct __wt_connection_stats { int64_t lock_checkpoint_count; int64_t lock_checkpoint_wait_application; int64_t lock_checkpoint_wait_internal; - int64_t lock_handle_list_count; - int64_t lock_handle_list_wait_application; - int64_t lock_handle_list_wait_internal; + int64_t lock_handle_list_wait_eviction; int64_t lock_metadata_count; int64_t lock_metadata_wait_application; int64_t lock_metadata_wait_internal; @@ -402,9 +405,11 @@ struct __wt_connection_stats { int64_t lock_table_wait_internal; int64_t log_slot_switch_busy; int64_t log_slot_closes; + int64_t log_slot_active_closed; int64_t log_slot_races; int64_t log_slot_transitions; int64_t log_slot_joins; + int64_t log_slot_no_free_slots; int64_t log_slot_unbuffered; int64_t log_bytes_payload; int64_t log_bytes_written; @@ -445,6 +450,9 @@ struct __wt_connection_stats { int64_t rec_split_stashed_objects; int64_t session_cursor_open; int64_t session_open; + int64_t session_table_alter_fail; + int64_t session_table_alter_success; + int64_t session_table_alter_skip; int64_t session_table_compact_fail; int64_t session_table_compact_success; int64_t session_table_create_fail; @@ -561,6 +569,7 @@ struct __wt_dsrc_stats { int64_t cache_pages_requested; int64_t cache_write; int64_t cache_write_restore; + int64_t cache_bytes_dirty; int64_t cache_eviction_clean; int64_t cache_state_gen_avg_gap; int64_t cache_state_avg_written_size; diff --git a/src/third_party/wiredtiger/src/include/thread_group.h b/src/third_party/wiredtiger/src/include/thread_group.h index 76758a090c4..77cff00dc8d 100644 --- a/src/third_party/wiredtiger/src/include/thread_group.h +++ b/src/third_party/wiredtiger/src/include/thread_group.h @@ -40,7 +40,7 @@ struct __wt_thread_group { const char *name; /* Name */ - WT_RWLOCK *lock; /* Protects group changes */ + WT_RWLOCK lock; /* Protects group changes */ /* * Condition signalled when wanting to wake up threads that are diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 8128e8e4cc2..7e802c188ab 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -49,8 +49,11 @@ WT_ASSERT((s), (s)->txn.forced_iso > 0); \ (s)->txn.forced_iso--; \ WT_ASSERT((s), txn_state->id == saved_state.id && \ + (txn_state->metadata_pinned == saved_state.metadata_pinned ||\ + saved_state.metadata_pinned == WT_TXN_NONE) && \ (txn_state->pinned_id == saved_state.pinned_id || \ saved_state.pinned_id == WT_TXN_NONE)); \ + txn_state->metadata_pinned = saved_state.metadata_pinned; \ txn_state->pinned_id = saved_state.pinned_id; \ } while (0) @@ -59,14 +62,17 @@ struct __wt_named_snapshot { TAILQ_ENTRY(__wt_named_snapshot) q; - uint64_t pinned_id, snap_min, snap_max; + uint64_t id, pinned_id, snap_min, snap_max; uint64_t *snapshot; uint32_t snapshot_count; }; -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_txn_state { +struct __wt_txn_state { + WT_CACHE_LINE_PAD_BEGIN volatile uint64_t id; volatile uint64_t pinned_id; + volatile uint64_t metadata_pinned; + WT_CACHE_LINE_PAD_END }; struct __wt_txn_global { @@ -86,7 +92,7 @@ struct __wt_txn_global { * Prevents the oldest ID moving forwards while threads are scanning * the global transaction state. */ - WT_RWLOCK *scan_rwlock; + WT_RWLOCK scan_rwlock; /* * Track information about the running checkpoint. The transaction @@ -94,14 +100,21 @@ struct __wt_txn_global { * for a long time so we keep them out of regular visibility checks. * Eviction and checkpoint operations know when they need to be aware * of checkpoint transactions. + * + * We rely on the fact that (a) the only table a checkpoint updates is + * the metadata; and (b) once checkpoint has finished reading a table, + * it won't revisit it. */ + volatile bool checkpoint_running; /* Checkpoint running */ volatile uint32_t checkpoint_id; /* Checkpoint's session ID */ - volatile uint64_t checkpoint_gen; - volatile uint64_t checkpoint_pinned; + volatile uint64_t checkpoint_gen; /* Checkpoint generation */ + volatile uint64_t checkpoint_pinned; /* Oldest ID for checkpoint */ volatile uint64_t checkpoint_txnid; /* Checkpoint's txn ID */ + volatile uint64_t metadata_pinned; /* Oldest ID for metadata */ + /* Named snapshot state. */ - WT_RWLOCK *nsnap_rwlock; + WT_RWLOCK nsnap_rwlock; volatile uint64_t nsnap_oldest_id; TAILQ_HEAD(__wt_nsnap_qh, __wt_named_snapshot) nsnaph; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index cf7e2eafc65..314c948e4d1 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -62,7 +62,6 @@ __wt_txn_unmodify(WT_SESSION_IMPL *session) static inline int __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) { - WT_DECL_RET; WT_TXN_OP *op; WT_TXN *txn; @@ -77,7 +76,7 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_TXN_OP_INMEM : WT_TXN_OP_BASIC; op->u.upd = upd; upd->txnid = session->txn.id; - return (ret); + return (0); } /* @@ -111,6 +110,13 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) txn_global = &S2C(session)->txn_global; btree = S2BT_SAFE(session); + /* + * The metadata is tracked specially because of optimizations for + * checkpoints. + */ + if (session->dhandle != NULL && WT_IS_METADATA(session->dhandle)) + return (txn_global->metadata_pinned); + /* * Take a local copy of these IDs in case they are updated while we are * checking visibility. The read of the transaction ID pinned by a @@ -119,7 +125,8 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) * minimum of it with the oldest ID, which is what we want. */ oldest_id = txn_global->oldest_id; - include_checkpoint_txn = btree == NULL || btree->include_checkpoint_txn; + include_checkpoint_txn = btree == NULL || + btree->checkpoint_gen != txn_global->checkpoint_gen; WT_READ_BARRIER(); checkpoint_pinned = txn_global->checkpoint_pinned; @@ -262,7 +269,7 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) * eviction, it's better to do it beforehand. */ WT_RET(__wt_cache_eviction_check(session, false, NULL)); - WT_RET(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); } F_SET(txn, WT_TXN_RUNNING); @@ -451,7 +458,7 @@ __wt_txn_read_last(WT_SESSION_IMPL *session) * __wt_txn_cursor_op -- * Called for each cursor operation. */ -static inline int +static inline void __wt_txn_cursor_op(WT_SESSION_IMPL *session) { WT_TXN *txn; @@ -482,10 +489,10 @@ __wt_txn_cursor_op(WT_SESSION_IMPL *session) if (txn->isolation == WT_ISO_READ_UNCOMMITTED) { if (txn_state->pinned_id == WT_TXN_NONE) txn_state->pinned_id = txn_global->last_running; + if (txn_state->metadata_pinned == WT_TXN_NONE) + txn_state->metadata_pinned = txn_state->pinned_id; } else if (!F_ISSET(txn, WT_TXN_HAS_SNAPSHOT)) - WT_RET(__wt_txn_get_snapshot(session)); - - return (0); + __wt_txn_get_snapshot(session); } /* diff --git a/src/third_party/wiredtiger/src/include/verify_build.h b/src/third_party/wiredtiger/src/include/verify_build.h index 477b9b7c134..a657b9ac460 100644 --- a/src/third_party/wiredtiger/src/include/verify_build.h +++ b/src/third_party/wiredtiger/src/include/verify_build.h @@ -6,9 +6,6 @@ * See the file LICENSE for redistribution information. */ -#undef ALIGN_CHECK -#undef SIZE_CHECK - /* * NOTE: If you see a compile failure in this file, your compiler is laying out * structs in memory in a way WiredTiger does not expect. Please refer to the @@ -36,12 +33,12 @@ */ #define WT_STATIC_ASSERT(cond) (void)sizeof(char[1 - 2 * !(cond)]) -#define SIZE_CHECK(type, e) do { \ +#define WT_SIZE_CHECK(type, e) do { \ char __check_##type[1 - 2 * !(sizeof(type) == (e))]; \ (void)__check_##type; \ } while (0) -#define ALIGN_CHECK(type, a) \ +#define WT_ALIGN_CHECK(type, a) \ WT_STATIC_ASSERT(WT_ALIGN(sizeof(type), (a)) == sizeof(type)) /* @@ -53,8 +50,16 @@ static inline void __wt_verify_build(void) { /* Check specific structures weren't padded. */ - SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); - SIZE_CHECK(WT_REF, WT_REF_SIZE); + WT_SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); + WT_SIZE_CHECK(WT_REF, WT_REF_SIZE); + + /* Check specific structures were padded. */ +#define WT_PADDING_CHECK(s) \ + WT_STATIC_ASSERT( \ + sizeof(s) > WT_CACHE_LINE_ALIGNMENT || \ + sizeof(s) % WT_CACHE_LINE_ALIGNMENT == 0) + WT_PADDING_CHECK(WT_LOGSLOT); + WT_PADDING_CHECK(WT_TXN_STATE); /* * The btree code encodes key/value pairs in size_t's, and requires at @@ -71,6 +76,3 @@ __wt_verify_build(void) */ WT_STATIC_ASSERT(sizeof(wt_off_t) == 8); } - -#undef ALIGN_CHECK -#undef SIZE_CHECK diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 665e8eaf4b0..ddecb2ac765 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -36,7 +36,7 @@ extern "C" { #if defined(DOXYGEN) || defined(SWIG) #define __F(func) func #else -#define __F(func) (*func) +#define __F(func) (*(func)) #endif #ifdef SWIG @@ -114,16 +114,16 @@ struct __wt_item { size_t size; #ifndef DOXYGEN -#define WT_ITEM_ALIGNED 0x00000001 -#define WT_ITEM_INUSE 0x00000002 - /* This appears in the middle of the struct to avoid padding. */ - /*! Object flags (internal use). */ - uint32_t flags; - /*! Managed memory chunk (internal use). */ void *mem; + /*! Managed memory size (internal use). */ size_t memsize; + +#define WT_ITEM_ALIGNED 0x00000001 +#define WT_ITEM_INUSE 0x00000002 + /*! Object flags (internal use). */ + uint32_t flags; #endif }; @@ -427,7 +427,7 @@ struct __wt_cursor { * * @param cursor the cursor handle * @errors - * In particular, if \c overwrite is not configured and a record with + * In particular, if \c overwrite=false is configured and a record with * the specified key already exists, ::WT_DUPLICATE_KEY is returned. * Also, if \c in_memory is configured for the database and the insert * requires more than the configured cache size to complete, @@ -452,7 +452,9 @@ struct __wt_cursor { * * On success, the cursor ends positioned at the modified record; to * minimize cursor resources, the WT_CURSOR::reset method should be - * called as soon as the cursor no longer needs that position. + * called as soon as the cursor no longer needs that position. (The + * WT_CURSOR::insert method never keeps a cursor position and may be + * more efficient for that reason.) * * The maximum length of a single column stored in a table is not fixed * (as it partially depends on the underlying file configuration), but @@ -460,7 +462,7 @@ struct __wt_cursor { * * @param cursor the cursor handle * @errors - * In particular, if \c overwrite is not configured and no record with + * In particular, if \c overwrite=false is configured and no record with * the specified key exists, ::WT_NOTFOUND is returned. * Also, if \c in_memory is configured for the database and the insert * requires more than the configured cache size to complete, @@ -477,8 +479,18 @@ struct __wt_cursor { * * @snippet ex_all.c Remove a record * - * If the cursor was not configured with "overwrite=true", the key must - * be set and the key's record must exist; the record will be removed. + * If the cursor was configured with "overwrite=false" (not the + * default), the key must be set and the key's record must exist; the + * record will be removed. + * + * Any cursor position does not change: if the cursor was positioned + * before the WT_CURSOR::remove call, the cursor remains positioned + * at the removed record; to minimize cursor resources, the + * WT_CURSOR::reset method should be called as soon as the cursor no + * longer needs that position. If the cursor was not positioned before + * the WT_CURSOR::remove call, the cursor ends with no position, and a + * subsequent call to the WT_CURSOR::next (WT_CURSOR::prev) method will + * iterate from the beginning (end) of the table. * * @snippet ex_all.c Remove a record and fail if DNE * @@ -486,14 +498,10 @@ struct __wt_cursor { * (that is, a store with an 'r' type key and 't' type value) is * identical to setting the record's value to 0. * - * On success, the cursor ends positioned at the removed record; to - * minimize cursor resources, the WT_CURSOR::reset method should be - * called as soon as the cursor no longer needs that position. - * * @param cursor the cursor handle * @errors - * In particular, if \c overwrite is not configured and no record with - * the specified key exists, ::WT_NOTFOUND is returned. + * In particular, if \c overwrite=false is configured and no record + * with the specified key exists, ::WT_NOTFOUND is returned. */ int __F(remove)(WT_CURSOR *cursor); /*! @} */ @@ -576,8 +584,9 @@ struct __wt_cursor { #define WT_CURSTD_OPEN 0x00200 #define WT_CURSTD_OVERWRITE 0x00400 #define WT_CURSTD_RAW 0x00800 -#define WT_CURSTD_VALUE_EXT 0x01000 /* Value points out of the tree. */ -#define WT_CURSTD_VALUE_INT 0x02000 /* Value points into the tree. */ +#define WT_CURSTD_RAW_SEARCH 0x01000 +#define WT_CURSTD_VALUE_EXT 0x02000 /* Value points out of the tree. */ +#define WT_CURSTD_VALUE_INT 0x04000 /* Value points into the tree. */ #define WT_CURSTD_VALUE_SET (WT_CURSTD_VALUE_EXT | WT_CURSTD_VALUE_INT) uint32_t flags; #endif @@ -799,6 +808,34 @@ struct __wt_session { void *app_private; #endif + /*! + * Alter a table. + * + * This will allow modification of some table settings after + * creation. + * + * @snippet ex_all.c Alter a table + * + * @param session the session handle + * @param name the URI of the object to alter, such as \c "table:stock" + * @configstart{WT_SESSION.alter, see dist/api_data.py} + * @config{access_pattern_hint, It is recommended that workloads that + * consist primarily of updates and/or point queries specify \c random. + * Workloads that do many cursor scans through large ranges of data + * specify \c sequential and other workloads specify \c none. The + * option leads to an advisory call to an appropriate operating system + * API where available., a string\, chosen from the following options: + * \c "none"\, \c "random"\, \c "sequential"; default \c none.} + * @config{cache_resident, do not ever evict the object's pages from + * cache. Not compatible with LSM tables; see @ref + * tuning_cache_resident for more information., a boolean flag; default + * \c false.} + * @configend + * @errors + */ + int __F(alter)(WT_HANDLE_CLOSED(WT_SESSION) *session, + const char *name, const char *config); + /*! * Close the session handle. * @@ -827,6 +864,11 @@ struct __wt_session { * * @param session the session handle * @configstart{WT_SESSION.reconfigure, see dist/api_data.py} + * @config{ignore_cache_size, when set\, operations performed by this + * session ignore the cache size and are not blocked when the cache is + * full. Note that use of this option for operations that create cache + * pressure can starve ordinary sessions that obey the cache size., a + * boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c * "read-uncommitted"\, \c "read-committed"\, \c "snapshot"; default \c @@ -990,6 +1032,13 @@ struct __wt_session { * \c "table:stock". For a description of URI formats * see @ref data_sources. * @configstart{WT_SESSION.create, see dist/api_data.py} + * @config{access_pattern_hint, It is recommended that workloads that + * consist primarily of updates and/or point queries specify \c random. + * Workloads that do many cursor scans through large ranges of data + * specify \c sequential and other workloads specify \c none. The + * option leads to an advisory call to an appropriate operating system + * API where available., a string\, chosen from the following options: + * \c "none"\, \c "random"\, \c "sequential"; default \c none.} * @config{allocation_size, the file unit allocation size\, in bytes\, * must a power-of-two; smaller values decrease the file space required * by overflow items\, and the default value of 4KB is a good choice @@ -1193,8 +1242,8 @@ struct __wt_session { * @config{split_pct, the Btree page split size as a percentage of the * maximum Btree page size\, that is\, when a Btree page is split\, it * will be split into smaller pages\, where each page is the specified - * percentage of the maximum Btree page size., an integer between 25 and - * 100; default \c 75.} + * percentage of the maximum Btree page size., an integer between 50 and + * 100; default \c 90.} * @config{type, set the type of data source used to store a column * group\, index or simple table. By default\, a \c "file:" URI is * derived from the object name. The \c type configuration can be used @@ -1431,6 +1480,10 @@ struct __wt_session { * contains. * @snippet ex_all.c Truncate a range * + * Any specified cursors end with no position, and subsequent calls to + * the WT_CURSOR::next (WT_CURSOR::prev) method will iterate from the + * beginning (end) of the table. + * * @param session the session handle * @param name the URI of the file or table to truncate * @param start optional cursor marking the first record discarded; @@ -1636,6 +1689,10 @@ struct __wt_session { * including the specified name., a string; default empty.} * @config{ * ),,} + * @config{include_updates, make updates from the current transaction + * visible to users of the named snapshot. Transactions started with + * such a named snapshot are restricted to being read-only., a boolean + * flag; default \c false.} * @config{name, specify a name for the snapshot., a string; default * empty.} * @configend @@ -1811,7 +1868,7 @@ struct __wt_connection { * threads WiredTiger will start to help evict pages from cache. The * number of threads started will vary depending on the current eviction * load. Each eviction worker thread uses a session from the configured - * session_max., an integer between 1 and 20; default \c 1.} + * session_max., an integer between 1 and 20; default \c 8.} * @config{    threads_min, minimum number of * threads WiredTiger will start to help evict pages from cache. The * number of threads currently running will vary depending on the @@ -1938,12 +1995,13 @@ struct __wt_connection { * as a list\, such as "verbose=[evictserver\,read]"., a * list\, with values chosen from the following options: \c "api"\, \c * "block"\, \c "checkpoint"\, \c "compact"\, \c "evict"\, \c - * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lsm"\, - * \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c - * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c - * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c - * "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c - * "write"; default empty.} + * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c + * "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c + * "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c + * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c + * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c + * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default + * empty.} * @configend * @errors */ @@ -2005,6 +2063,11 @@ struct __wt_connection { * connection's error handler is used. See @ref error_handling_event * for more information. * @configstart{WT_CONNECTION.open_session, see dist/api_data.py} + * @config{ignore_cache_size, when set\, operations performed by this + * session ignore the cache size and are not blocked when the cache is + * full. Note that use of this option for operations that create cache + * pressure can starve ordinary sessions that obey the cache size., a + * boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c * "read-uncommitted"\, \c "read-committed"\, \c "snapshot"; default \c @@ -2204,6 +2267,11 @@ struct __wt_connection { * I/O. The default value of -1 indicates a platform-specific alignment value * should be used (4KB on Linux systems when direct I/O is configured\, zero * elsewhere)., an integer between -1 and 1MB; default \c -1.} + * @config{builtin_extension_config, A structure where the keys are the names of + * builtin extensions and the values are passed to WT_CONNECTION::load_extension + * as the \c config parameter (for example\, + * builtin_extension_config={zlib={compression_level=3}})., a + * string; default empty.} * @config{cache_overhead, assume the heap allocator overhead is the specified * percentage\, and adjust the cache usage by that amount (for example\, if * there is 10GB of data in cache\, a percentage of 10 means WiredTiger treats @@ -2277,7 +2345,7 @@ struct __wt_connection { * WiredTiger will start to help evict pages from cache. The number of threads * started will vary depending on the current eviction load. Each eviction * worker thread uses a session from the configured session_max., an integer - * between 1 and 20; default \c 1.} + * between 1 and 20; default \c 8.} * @config{    threads_min, * minimum number of threads WiredTiger will start to help evict pages from * cache. The number of threads currently running will vary depending on the @@ -2307,7 +2375,7 @@ struct __wt_connection { * @config{exclusive, fail if the database already exists\, generally used with * the \c create option., a boolean flag; default \c false.} * @config{extensions, list of shared library extensions to load (using dlopen). - * Any values specified to an library extension are passed to + * Any values specified to a library extension are passed to * WT_CONNECTION::load_extension as the \c config parameter (for example\, * extensions=(/path/ext.so={entry=my_entry}))., a list of strings; * default empty.} @@ -2329,8 +2397,6 @@ struct __wt_connection { * in seconds at which to check for files that are inactive and close them., an * integer between 1 and 100000; default \c 10.} * @config{ ),,} - * @config{hazard_max, maximum number of simultaneous hazard pointers per - * session handle., an integer greater than or equal to 15; default \c 1000.} * @config{in_memory, keep data in-memory only. See @ref in_memory for more * information., a boolean flag; default \c false.} * @config{log = (, enable logging. Enabling logging uses three sessions from @@ -2461,12 +2527,13 @@ struct __wt_connection { * WiredTiger is configured with --enable-verbose. Options are given as a * list\, such as "verbose=[evictserver\,read]"., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c - * "checkpoint"\, \c "compact"\, \c "evict"\, \c "evictserver"\, \c "fileops"\, - * \c "handleops"\, \c "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c - * "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c - * "recovery"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, - * \c "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c - * "write"; default empty.} + * "checkpoint"\, \c "compact"\, \c "evict"\, \c "evict_stuck"\, \c + * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lsm"\, \c + * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c + * "rebalance"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c + * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c + * "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; + * default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as "write_through=[data]". Configuring \c write_through requires @@ -3010,27 +3077,27 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * transaction is in progress, it should be rolled back and the operation * retried in a new transaction. */ -#define WT_ROLLBACK -31800 +#define WT_ROLLBACK (-31800) /*! * Attempt to insert an existing key. * This error is generated when the application attempts to insert a record with * the same key as an existing record without the 'overwrite' configuration to * WT_SESSION::open_cursor. */ -#define WT_DUPLICATE_KEY -31801 +#define WT_DUPLICATE_KEY (-31801) /*! * Non-specific WiredTiger error. * This error is returned when an error is not covered by a specific error * return. */ -#define WT_ERROR -31802 +#define WT_ERROR (-31802) /*! * Item not found. * This error indicates an operation did not find a value to return. This * includes cursor search and other operations where no record matched the * cursor's search key such as WT_CURSOR::update or WT_CURSOR::remove. */ -#define WT_NOTFOUND -31803 +#define WT_NOTFOUND (-31803) /*! * WiredTiger library panic. * This error indicates an underlying problem that requires the application exit @@ -3038,17 +3105,17 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * returned from a WiredTiger interface, no further WiredTiger calls are * required. */ -#define WT_PANIC -31804 +#define WT_PANIC (-31804) /*! @cond internal */ /*! Restart the operation (internal). */ -#define WT_RESTART -31805 +#define WT_RESTART (-31805) /*! @endcond */ /*! * Recovery must be run to continue. * This error is generated when wiredtiger_open is configured to return an error * if recovery is required to use the database. */ -#define WT_RUN_RECOVERY -31806 +#define WT_RUN_RECOVERY (-31806) /*! * Operation would overflow cache. * This error is only generated when wiredtiger_open is configured to run in- @@ -3057,7 +3124,7 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * progress, it should be rolled back and the operation retried in a new * transaction. */ -#define WT_CACHE_FULL -31807 +#define WT_CACHE_FULL (-31807) /* * Error return section: END * DO NOT EDIT: automatically built by dist/api_err.py. @@ -3412,6 +3479,14 @@ struct __wt_compressor { * @snippet ex_data_source.c WT_DATA_SOURCE register */ struct __wt_data_source { + /*! + * Callback to alter an object. + * + * @snippet ex_data_source.c WT_DATA_SOURCE alter + */ + int (*alter)(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, WT_CONFIG_ARG *config); + /*! * Callback to create a new object. * @@ -3715,21 +3790,25 @@ typedef enum { WT_FS_OPEN_FILE_TYPE_REGULAR /*!< open a regular file */ } WT_FS_OPEN_FILE_TYPE; +/*! WT_FILE_SYSTEM::open_file flags: random access pattern */ +#define WT_FS_OPEN_ACCESS_RAND 0x001 +/*! WT_FILE_SYSTEM::open_file flags: sequential access pattern */ +#define WT_FS_OPEN_ACCESS_SEQ 0x002 /*! WT_FILE_SYSTEM::open_file flags: create if does not exist */ -#define WT_FS_OPEN_CREATE 0x001 +#define WT_FS_OPEN_CREATE 0x004 /*! WT_FILE_SYSTEM::open_file flags: direct I/O requested */ -#define WT_FS_OPEN_DIRECTIO 0x002 +#define WT_FS_OPEN_DIRECTIO 0x008 /*! WT_FILE_SYSTEM::open_file flags: file creation must be durable */ -#define WT_FS_OPEN_DURABLE 0x004 +#define WT_FS_OPEN_DURABLE 0x010 /*! * WT_FILE_SYSTEM::open_file flags: return EBUSY if exclusive use not available */ -#define WT_FS_OPEN_EXCLUSIVE 0x008 +#define WT_FS_OPEN_EXCLUSIVE 0x020 #ifndef DOXYGEN -#define WT_FS_OPEN_FIXED 0x010 /* Path not home relative (internal) */ +#define WT_FS_OPEN_FIXED 0x040 /* Path not home relative (internal) */ #endif /*! WT_FILE_SYSTEM::open_file flags: open is read-only */ -#define WT_FS_OPEN_READONLY 0x020 +#define WT_FS_OPEN_READONLY 0x080 /*! * WT_FILE_SYSTEM::remove or WT_FILE_SYSTEM::rename flags: the remove or rename @@ -4365,390 +4444,406 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_STATE 1051 /*! cache: eviction walks abandoned */ #define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1052 +/*! cache: eviction worker thread active */ +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1053 +/*! cache: eviction worker thread created */ +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1054 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1053 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1055 +/*! cache: eviction worker thread removed */ +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1056 +/*! cache: eviction worker thread stable number */ +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1057 /*! cache: failed eviction of pages that exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1054 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1058 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1055 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1059 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1056 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1060 +/*! cache: force re-tuning of eviction workers once in a while */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1061 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1057 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1062 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1058 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1063 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1059 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1064 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1060 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1065 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1061 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1066 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1062 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1067 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1063 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1068 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1064 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1069 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1065 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1070 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1066 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1071 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1067 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1072 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1068 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1073 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1069 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1074 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1070 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1075 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1071 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1076 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1072 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1077 /*! cache: overflow values cached in memory */ -#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1073 +#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1078 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1074 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1079 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1075 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1080 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1076 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1081 /*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1077 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1082 /*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1078 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1083 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1079 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1084 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1080 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1085 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1081 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1086 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1082 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1087 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1083 +#define WT_STAT_CONN_CACHE_READ 1088 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1084 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1089 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1085 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1090 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1086 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1091 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1092 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1088 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1093 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1089 +#define WT_STAT_CONN_CACHE_WRITE 1094 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1090 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1095 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1091 +#define WT_STAT_CONN_CACHE_OVERHEAD 1096 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1092 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1097 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1093 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1098 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1094 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1099 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1095 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1100 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1096 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1101 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1097 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1102 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1098 +#define WT_STAT_CONN_COND_AUTO_WAIT 1103 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1099 +#define WT_STAT_CONN_FILE_OPEN 1104 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1100 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1105 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1101 +#define WT_STAT_CONN_MEMORY_FREE 1106 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1102 +#define WT_STAT_CONN_MEMORY_GROW 1107 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1103 +#define WT_STAT_CONN_COND_WAIT 1108 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1104 +#define WT_STAT_CONN_RWLOCK_READ 1109 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1105 +#define WT_STAT_CONN_RWLOCK_WRITE 1110 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1106 +#define WT_STAT_CONN_FSYNC_IO 1111 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1107 +#define WT_STAT_CONN_READ_IO 1112 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1108 +#define WT_STAT_CONN_WRITE_IO 1113 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1109 +#define WT_STAT_CONN_CURSOR_CREATE 1114 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1110 +#define WT_STAT_CONN_CURSOR_INSERT 1115 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1111 +#define WT_STAT_CONN_CURSOR_NEXT 1116 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1112 +#define WT_STAT_CONN_CURSOR_PREV 1117 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1113 +#define WT_STAT_CONN_CURSOR_REMOVE 1118 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1114 +#define WT_STAT_CONN_CURSOR_RESET 1119 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1115 +#define WT_STAT_CONN_CURSOR_RESTART 1120 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1116 +#define WT_STAT_CONN_CURSOR_SEARCH 1121 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1117 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1122 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1118 +#define WT_STAT_CONN_CURSOR_UPDATE 1123 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1119 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1124 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1120 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1125 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1121 +#define WT_STAT_CONN_DH_SWEEP_REF 1126 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1122 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1127 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1123 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1128 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1124 +#define WT_STAT_CONN_DH_SWEEP_TOD 1129 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1125 +#define WT_STAT_CONN_DH_SWEEPS 1130 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1126 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1131 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1127 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1132 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1128 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1133 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1129 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1134 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1130 -/*! lock: handle-list lock acquisitions */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_COUNT 1131 -/*! lock: handle-list lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_APPLICATION 1132 -/*! lock: handle-list lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_INTERNAL 1133 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1135 +/*! lock: handle-list lock eviction thread wait time (usecs) */ +#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1136 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1134 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1137 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1135 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1138 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1136 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1139 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1137 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1140 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1138 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1141 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1139 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1142 /*! lock: table lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_COUNT 1140 +#define WT_STAT_CONN_LOCK_TABLE_COUNT 1143 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1141 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1144 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1142 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1145 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1143 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1146 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1144 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1147 +/*! log: consolidated slot join active slot closed */ +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1148 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1145 +#define WT_STAT_CONN_LOG_SLOT_RACES 1149 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1146 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1150 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1147 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1151 +/*! log: consolidated slot transitions unable to find free slot */ +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1152 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1148 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1153 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1149 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1154 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1150 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1155 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1151 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1156 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1152 +#define WT_STAT_CONN_LOG_FLUSH 1157 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1153 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1158 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1154 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1159 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1155 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1160 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1156 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1161 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1157 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1162 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1158 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1163 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1159 +#define WT_STAT_CONN_LOG_SCANS 1164 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1160 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1165 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1161 +#define WT_STAT_CONN_LOG_WRITE_LSN 1166 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1162 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1167 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1163 +#define WT_STAT_CONN_LOG_SYNC 1168 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1164 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1169 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1165 +#define WT_STAT_CONN_LOG_SYNC_DIR 1170 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1166 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1171 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1167 +#define WT_STAT_CONN_LOG_WRITES 1172 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1168 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1173 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1169 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1174 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1170 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1175 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1171 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1176 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1172 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1177 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1173 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1178 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1174 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1179 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1175 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1180 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1176 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1181 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1177 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1182 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1178 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1183 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1179 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1184 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1180 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1185 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1181 +#define WT_STAT_CONN_REC_PAGES 1186 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1182 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1187 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1183 +#define WT_STAT_CONN_REC_PAGE_DELETE 1188 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1184 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1189 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1185 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1190 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1186 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1191 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1187 +#define WT_STAT_CONN_SESSION_OPEN 1192 +/*! session: table alter failed calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1193 +/*! session: table alter successful calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1194 +/*! session: table alter unchanged and skipped */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1195 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1188 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1196 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1189 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1197 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1190 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1198 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1191 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1199 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1192 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1200 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1193 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1201 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1194 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1202 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1195 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1203 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1196 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1204 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1197 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1205 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1198 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1206 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1199 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1207 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1200 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1208 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1201 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1209 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1202 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1210 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1203 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1211 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1204 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1212 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1205 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1213 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1206 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1214 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1207 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1215 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1208 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1216 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1209 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1217 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1210 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1218 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1211 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1219 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1212 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1220 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1213 +#define WT_STAT_CONN_PAGE_SLEEP 1221 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1214 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1222 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1215 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1223 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1216 +#define WT_STAT_CONN_TXN_BEGIN 1224 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1217 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1225 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1218 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1226 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1219 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1227 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1220 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1228 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1221 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1229 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1222 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1230 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1223 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1231 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1224 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1232 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1225 +#define WT_STAT_CONN_TXN_CHECKPOINT 1233 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1234 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1227 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1235 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1236 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1237 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1230 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1238 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1231 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1239 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1232 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1240 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1233 +#define WT_STAT_CONN_TXN_SYNC 1241 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1234 +#define WT_STAT_CONN_TXN_COMMIT 1242 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1235 +#define WT_STAT_CONN_TXN_ROLLBACK 1243 /*! * @} @@ -4908,181 +5003,183 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CACHE_WRITE 2059 /*! cache: pages written requiring in-memory restoration */ #define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060 +/*! cache: tracked dirty bytes in the cache */ +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2061 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2061 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2062 /*! * cache_walk: Average difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2062 +#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2063 /*! * cache_walk: Average on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2063 +#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2064 /*! * cache_walk: Clean pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2064 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2065 /*! * cache_walk: Current eviction generation, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2065 +#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2066 /*! * cache_walk: Dirty pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2066 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2067 /*! * cache_walk: Entries in the root page, only reported if cache_walk or * all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2067 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2068 /*! * cache_walk: Internal pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2068 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2069 /*! * cache_walk: Leaf pages currently in cache, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2069 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2070 /*! * cache_walk: Maximum difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2070 +#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2071 /*! * cache_walk: Maximum page size seen, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2071 +#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2072 /*! * cache_walk: Minimum on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2072 +#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2073 /*! * cache_walk: On-disk page image sizes smaller than a single allocation * unit, only reported if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2073 +#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2074 /*! * cache_walk: Pages created in memory and never written, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2074 +#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2075 /*! * cache_walk: Pages currently queued for eviction, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2075 +#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2076 /*! * cache_walk: Pages that could not be queued for eviction, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2076 +#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2077 /*! * cache_walk: Refs skipped during cache traversal, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2077 +#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2078 /*! * cache_walk: Size of the root page, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2078 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2079 /*! * cache_walk: Total number of pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES 2079 +#define WT_STAT_DSRC_CACHE_STATE_PAGES 2080 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2080 +#define WT_STAT_DSRC_COMPRESS_READ 2081 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2081 +#define WT_STAT_DSRC_COMPRESS_WRITE 2082 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2082 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2083 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2083 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2084 /*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2084 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2085 /*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2085 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2086 /*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2086 +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2087 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2087 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2088 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2088 +#define WT_STAT_DSRC_CURSOR_CREATE 2089 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2089 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2090 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2090 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2091 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2091 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2092 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2092 +#define WT_STAT_DSRC_CURSOR_INSERT 2093 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2093 +#define WT_STAT_DSRC_CURSOR_NEXT 2094 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2094 +#define WT_STAT_DSRC_CURSOR_PREV 2095 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2095 +#define WT_STAT_DSRC_CURSOR_REMOVE 2096 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2096 +#define WT_STAT_DSRC_CURSOR_RESET 2097 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2097 +#define WT_STAT_DSRC_CURSOR_RESTART 2098 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2098 +#define WT_STAT_DSRC_CURSOR_SEARCH 2099 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2099 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2100 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2100 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2101 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2101 +#define WT_STAT_DSRC_CURSOR_UPDATE 2102 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2102 +#define WT_STAT_DSRC_REC_DICTIONARY 2103 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2103 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2104 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2104 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2105 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2105 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2106 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2106 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2107 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2107 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2108 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2108 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2109 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2109 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2110 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2110 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2111 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2111 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2112 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2112 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2113 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2113 +#define WT_STAT_DSRC_REC_PAGES 2114 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2114 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2115 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2115 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2116 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2116 +#define WT_STAT_DSRC_SESSION_COMPACT 2117 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2117 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2118 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2118 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2119 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wiredtiger_ext.h b/src/third_party/wiredtiger/src/include/wiredtiger_ext.h index 3d65cd1fc24..236d4e07e67 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger_ext.h +++ b/src/third_party/wiredtiger/src/include/wiredtiger_ext.h @@ -204,25 +204,47 @@ struct __wt_extension_api { WT_COLLATOR *collator, WT_ITEM *first, WT_ITEM *second, int *cmp); /*! - * @copydoc wiredtiger_config_parser_open + * Return the value of a configuration key. + * + * @param wt_api the extension handle + * @param session the session handle (or NULL if none available) + * @param config the configuration information passed to an application + * @param key configuration key string + * @param value the returned value + * @errors + * + * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); /*! - * Return the value of a configuration string. + * Return the value of a configuration key from a string. * * @param wt_api the extension handle * @param session the session handle (or NULL if none available) + * @param config the configuration string * @param key configuration key string - * @param config the configuration information passed to an application * @param value the returned value * @errors * * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); + int (*config_get_string)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, const char *key, WT_CONFIG_ITEM *value); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open_arg)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, WT_CONFIG_ARG *config, + WT_CONFIG_PARSER **config_parserp); /*! * Insert a row into the metadata if it does not already exist. diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index dc3e3a0b6c0..cf79578985b 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -106,10 +106,12 @@ struct __wt_col; typedef struct __wt_col WT_COL; struct __wt_col_rle; typedef struct __wt_col_rle WT_COL_RLE; +struct __wt_col_var_repeat; + typedef struct __wt_col_var_repeat WT_COL_VAR_REPEAT; struct __wt_colgroup; typedef struct __wt_colgroup WT_COLGROUP; -struct __wt_compact; - typedef struct __wt_compact WT_COMPACT; +struct __wt_compact_state; + typedef struct __wt_compact_state WT_COMPACT_STATE; struct __wt_condvar; typedef struct __wt_condvar WT_CONDVAR; struct __wt_config; diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 96b593ec706..803d3e8dfab 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -24,7 +24,7 @@ static int __log_write_internal( * __log_wait_for_earlier_slot -- * Wait for write_lsn to catch up to this slot. */ -static void +static int __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) { WT_CONNECTION_IMPL *conn; @@ -41,16 +41,18 @@ __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) * unlock in case an earlier thread is trying to switch its * slot and complete its operation. */ + WT_RET(WT_SESSION_CHECK_PANIC(session)); if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_unlock(session, &log->log_slot_lock); - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); + __wt_cond_signal(session, conn->log_wrlsn_cond); if (++yield_count < WT_THOUSAND) __wt_yield(); else - __wt_cond_wait(session, log->log_write_cond, 200); + __wt_cond_wait(session, log->log_write_cond, 200, NULL); if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_lock(session, &log->log_slot_lock); } + return (0); } /* @@ -62,16 +64,21 @@ static int __log_fs_write(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, wt_off_t offset, size_t len, const void *buf) { + WT_DECL_RET; + /* * If we're writing into a new log file, we have to wait for all * writes to the previous log file to complete otherwise there could * be a hole at the end of the previous log file that we cannot detect. */ if (slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { - __log_wait_for_earlier_slot(session, slot); + WT_RET(__log_wait_for_earlier_slot(session, slot)); WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } - return (__wt_write(session, slot->slot_fh, offset, len, buf)); + if ((ret = __wt_write(session, slot->slot_fh, offset, len, buf)) != 0) + WT_PANIC_MSG(session, ret, + "%s: fatal log failure", slot->slot_fh->name); + return (ret); } /* @@ -89,7 +96,7 @@ __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn) log = conn->log; log->ckpt_lsn = *ckp_lsn; if (conn->log_cond != NULL) - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); } /* @@ -105,6 +112,7 @@ __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) conn = S2C(session); log = conn->log; + WT_RET(WT_SESSION_CHECK_PANIC(session)); WT_RET(__wt_log_force_write(session, 1, NULL)); __wt_log_wrlsn(session, NULL); if (start) @@ -159,6 +167,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) uint64_t fsync_duration_usecs; log = S2C(session)->log; + log_fh = NULL; /* * We need to wait for the previous log file to get written @@ -168,8 +177,9 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) * log file ready to close. */ while (log->sync_lsn.l.file < min_lsn->l.file) { + WT_RET(WT_SESSION_CHECK_PANIC(session)); __wt_cond_signal(session, S2C(session)->log_file_cond); - __wt_cond_wait(session, log->log_sync_cond, 10000); + __wt_cond_wait(session, log->log_sync_cond, 10000, NULL); } __wt_spin_lock(session, &log->log_sync_lock); WT_ASSERT(session, log->log_dir_fh != NULL); @@ -214,11 +224,12 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) WT_STAT_CONN_INCR(session, log_sync); WT_STAT_CONN_INCRV(session, log_sync_duration, fsync_duration_usecs); - WT_ERR(__wt_close(session, &log_fh)); __wt_cond_signal(session, log->log_sync_cond); } err: __wt_spin_unlock(session, &log->log_sync_lock); + if (log_fh != NULL) + WT_TRET(__wt_close(session, &log_fh)); return (ret); } @@ -298,14 +309,11 @@ void __wt_log_written_reset(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; - WT_LOG *log; conn = S2C(session); - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) - return; - log = conn->log; - log->log_written = 0; - return; + + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) + conn->log->log_written = 0; } /* @@ -425,8 +433,6 @@ __wt_log_extract_lognum( { const char *p; - WT_UNUSED(session); - if (id == NULL || name == NULL) return (WT_ERROR); if ((p = strrchr(name, '.')) == NULL || @@ -777,8 +783,8 @@ __log_openfile(WT_SESSION_IMPL *session, __wt_log_desc_byteswap(desc); if (desc->log_magic != WT_LOG_MAGIC) WT_PANIC_RET(session, WT_ERROR, - "log file %s corrupted: Bad magic number %" PRIu32, - (*fhp)->name, desc->log_magic); + "log file %s corrupted: Bad magic number %" PRIu32, + (*fhp)->name, desc->log_magic); if (desc->majorv > WT_LOG_MAJOR_VERSION || (desc->majorv == WT_LOG_MAJOR_VERSION && desc->minorv > WT_LOG_MINOR_VERSION)) @@ -895,12 +901,12 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) */ create_log = true; if (conn->log_prealloc > 0 && !conn->hot_backup) { - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (conn->hot_backup) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); else { ret = __log_alloc_prealloc(session, log->fileid); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); /* * If ret is 0 it means we found a pre-allocated file. @@ -915,7 +921,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) else { WT_STAT_CONN_INCR(session, log_prealloc_missed); if (conn->log_cond != NULL) - __wt_cond_auto_signal( + __wt_cond_signal( session, conn->log_cond); } } @@ -1029,12 +1035,12 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) log = conn->log; if (!F_ISSET(log, WT_LOG_TRUNCATE_NOTSUP) && !conn->hot_backup) { - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (conn->hot_backup) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); else { ret = __wt_ftruncate(session, log_fh, offset); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); if (ret != ENOTSUP) return (ret); F_SET(log, WT_LOG_TRUNCATE_NOTSUP); @@ -1276,7 +1282,7 @@ __wt_log_open(WT_SESSION_IMPL *session) * where the previous log file ends. */ if (!F_ISSET(conn, WT_CONN_READONLY)) { - WT_WITH_SLOT_LOCK(session, log, ret, + WT_WITH_SLOT_LOCK(session, log, ret = __log_newfile(session, true, NULL)); WT_ERR(ret); } @@ -1462,7 +1468,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * be holes in the log file. */ WT_STAT_CONN_INCR(session, log_release_write_lsn); - __log_wait_for_earlier_slot(session, slot); + WT_ERR(__log_wait_for_earlier_slot(session, slot)); log->write_start_lsn = slot->slot_start_lsn; log->write_lsn = slot->slot_end_lsn; @@ -1483,6 +1489,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * current fsync completes and advance log->sync_lsn. */ while (F_ISSET(slot, WT_SLOT_SYNC | WT_SLOT_SYNC_DIR)) { + WT_ERR(WT_SESSION_CHECK_PANIC(session)); /* * We have to wait until earlier log files have finished their * sync operations. The most recent one will set the LSN to the @@ -1490,7 +1497,8 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) */ if (log->sync_lsn.l.file < slot->slot_end_lsn.l.file || __wt_spin_trylock(session, &log->log_sync_lock) != 0) { - __wt_cond_wait(session, log->log_sync_cond, 10000); + __wt_cond_wait( + session, log->log_sync_cond, 10000, NULL); continue; } locked = true; @@ -1655,10 +1663,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_RET(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); if (logcount == 0) - /* - * Return it is not supported if none don't exist. - */ - return (ENOTSUP); + WT_RET_MSG(session, ENOTSUP, "no log files found"); for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); @@ -1674,6 +1679,10 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, &log_fh, WT_LOG_FILENAME, start_lsn.l.file, WT_LOG_OPEN_VERIFY)); WT_ERR(__wt_filesize(session, log_fh, &log_size)); rd_lsn = start_lsn; + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, + "Recovering log %" PRIu32 " through %" PRIu32, + rd_lsn.l.file, end_lsn.l.file); WT_ERR(__wt_scr_alloc(session, WT_LOG_ALIGN, &buf)); WT_ERR(__wt_scr_alloc(session, 0, &decryptitem)); @@ -1722,6 +1731,11 @@ advance: WT_ERR(__log_openfile(session, &log_fh, WT_LOG_FILENAME, rd_lsn.l.file, WT_LOG_OPEN_VERIFY)); + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, + "Recovering log %" PRIu32 + " through %" PRIu32, + rd_lsn.l.file, end_lsn.l.file); WT_ERR(__wt_filesize(session, log_fh, &log_size)); eol = false; continue; @@ -1758,9 +1772,8 @@ advance: if (eol) /* Found a hole. This LSN is the end. */ break; - else - /* Last record in log. Look for more. */ - goto advance; + /* Last record in log. Look for more. */ + goto advance; } rdup_len = __wt_rduppo2(reclen, allocsize); if (reclen > allocsize) { @@ -1906,7 +1919,6 @@ __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) { WT_LOG *log; WT_MYSLOT myslot; - uint32_t joined; log = S2C(session)->log; memset(&myslot, 0, sizeof(myslot)); @@ -1914,14 +1926,7 @@ __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) if (did_work != NULL) *did_work = true; myslot.slot = log->active_slot; - joined = WT_LOG_SLOT_JOINED(log->active_slot->slot_state); - if (joined == 0) { - WT_STAT_CONN_INCR(session, log_force_write_skip); - if (did_work != NULL) - *did_work = false; - return (0); - } - return (__wt_log_slot_switch(session, &myslot, retry, true)); + return (__wt_log_slot_switch(session, &myslot, retry, true, did_work)); } /* @@ -2120,7 +2125,11 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_STAT_CONN_INCR(session, log_writes); - __wt_log_slot_join(session, rdup_len, flags, &myslot); + /* + * The only time joining a slot should ever return an error is if it + * detects a panic. + */ + WT_ERR(__wt_log_slot_join(session, rdup_len, flags, &myslot)); /* * If the addition of this record crosses the buffer boundary, * switch in a new slot. @@ -2129,7 +2138,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ret = 0; if (myslot.end_offset >= WT_LOG_SLOT_BUF_MAX || F_ISSET(&myslot, WT_MYSLOT_UNBUFFERED) || force) - ret = __wt_log_slot_switch(session, &myslot, true, false); + ret = __wt_log_slot_switch(session, &myslot, true, false, NULL); if (ret == 0) ret = __log_fill(session, &myslot, false, record, &lsn); release_size = __wt_log_slot_release( @@ -2154,7 +2163,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, * XXX I've seen times when conditions are NULL. */ if (conn->log_cond != NULL) { - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); __wt_yield(); } else WT_ERR(__wt_log_force_write(session, 1, NULL)); @@ -2162,13 +2171,19 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, if (LF_ISSET(WT_LOG_FLUSH)) { /* Wait for our writes to reach the OS */ while (__wt_log_cmp(&log->write_lsn, &lsn) <= 0 && - myslot.slot->slot_error == 0) - __wt_cond_wait(session, log->log_write_cond, 10000); + myslot.slot->slot_error == 0) { + WT_ERR(WT_SESSION_CHECK_PANIC(session)); + __wt_cond_wait( + session, log->log_write_cond, 10000, NULL); + } } else if (LF_ISSET(WT_LOG_FSYNC)) { /* Wait for our writes to reach disk */ while (__wt_log_cmp(&log->sync_lsn, &lsn) <= 0 && - myslot.slot->slot_error == 0) - __wt_cond_wait(session, log->log_sync_cond, 10000); + myslot.slot->slot_error == 0) { + WT_ERR(WT_SESSION_CHECK_PANIC(session)); + __wt_cond_wait( + session, log->log_sync_cond, 10000, NULL); + } } /* @@ -2193,12 +2208,12 @@ err: /* * If one of the sync flags is set, assert the proper LSN has moved to - * match. + * match on success. */ - WT_ASSERT(session, !LF_ISSET(WT_LOG_FLUSH) || + WT_ASSERT(session, ret != 0 || !LF_ISSET(WT_LOG_FLUSH) || __wt_log_cmp(&log->write_lsn, &lsn) >= 0); - WT_ASSERT(session, - !LF_ISSET(WT_LOG_FSYNC) || __wt_log_cmp(&log->sync_lsn, &lsn) >= 0); + WT_ASSERT(session, ret != 0 || !LF_ISSET(WT_LOG_FSYNC) || + __wt_log_cmp(&log->sync_lsn, &lsn) >= 0); return (ret); } @@ -2223,8 +2238,10 @@ __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) return (0); va_copy(ap_copy, ap); - len = (size_t)vsnprintf(NULL, 0, fmt, ap_copy) + 1; + len = 1; + ret = __wt_vsnprintf_len_incr(NULL, 0, &len, fmt, ap_copy); va_end(ap_copy); + WT_RET(ret); WT_RET( __wt_logrec_alloc(session, sizeof(WT_LOG_RECORD) + len, &logrec)); @@ -2241,7 +2258,8 @@ __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) rec_fmt, rectype)); logrec->size += (uint32_t)header_size; - (void)vsnprintf((char *)logrec->data + logrec->size, len, fmt, ap); + WT_ERR(__wt_vsnprintf( + (char *)logrec->data + logrec->size, len, fmt, ap)); __wt_verbose(session, WT_VERB_LOG, "log_printf: %s", (char *)logrec->data + logrec->size); diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index c79805f5371..97e317ce68c 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -8,6 +8,49 @@ #include "wt_internal.h" +#ifdef HAVE_DIAGNOSTIC +/* + * __log_slot_dump -- + * Dump the entire slot state. + */ +static void +__log_slot_dump(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_LOGSLOT *slot; + int earliest, i; + + conn = S2C(session); + log = conn->log; + earliest = 0; + for (i = 0; i < WT_SLOT_POOL; i++) { + slot = &log->slot_pool[i]; + if (__wt_log_cmp(&slot->slot_release_lsn, + &log->slot_pool[earliest].slot_release_lsn) < 0) + earliest = i; + __wt_errx(session, "Slot %d:", i); + __wt_errx(session, " State: %" PRIx64 " Flags: %" PRIx32, + slot->slot_state, slot->flags); + __wt_errx(session, " Start LSN: %" PRIu32 "/%" PRIu32, + slot->slot_start_lsn.l.file, slot->slot_start_lsn.l.offset); + __wt_errx(session, " End LSN: %" PRIu32 "/%" PRIu32, + slot->slot_end_lsn.l.file, slot->slot_end_lsn.l.offset); + __wt_errx(session, " Release LSN: %" PRIu32 "/%" PRIu32, + slot->slot_release_lsn.l.file, + slot->slot_release_lsn.l.offset); + __wt_errx(session, " Offset: start: %" PRIuMAX + " last:%" PRIuMAX, (uintmax_t)slot->slot_start_offset, + (uintmax_t)slot->slot_last_offset); + __wt_errx(session, " Unbuffered: %" PRId64 + " error: %" PRId32, slot->slot_unbuffered, + slot->slot_error); + } + __wt_errx(session, "Earliest slot: %d", earliest); + +} +#endif + /* * __wt_log_slot_activate -- * Initialize a slot to become active. @@ -21,7 +64,6 @@ __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) conn = S2C(session); log = conn->log; - slot->slot_state = 0; /* * !!! slot_release_lsn must be set outside this function because * this function may be called after a log file switch and the @@ -30,12 +72,19 @@ __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) * set for closing the file handle on a log file switch. The flags * are reset when the slot is freed. See log_slot_free. */ + slot->slot_unbuffered = 0; slot->slot_start_lsn = slot->slot_end_lsn = log->alloc_lsn; slot->slot_start_offset = log->alloc_lsn.l.offset; slot->slot_last_offset = log->alloc_lsn.l.offset; slot->slot_fh = log->log_fh; slot->slot_error = 0; - slot->slot_unbuffered = 0; + WT_DIAGNOSTIC_YIELD; + /* + * Set the slot state last. Other threads may have a stale pointer + * to this slot and could try to alter the state and other fields once + * they see the state cleared. + */ + WT_PUBLISH(slot->slot_state, 0); } /* @@ -50,6 +99,10 @@ __log_slot_close( WT_CONNECTION_IMPL *conn; WT_LOG *log; int64_t end_offset, new_state, old_state; +#ifdef HAVE_DIAGNOSTIC + struct timespec begin, now; + int count; +#endif WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SLOT)); WT_ASSERT(session, releasep != NULL); @@ -101,9 +154,33 @@ retry: * that value. If the state is unbuffered, wait for the unbuffered * size to be set. */ - while (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state) && - slot->slot_unbuffered == 0) - __wt_yield(); +#ifdef HAVE_DIAGNOSTIC + count = 0; + __wt_epoch(session, &begin); +#endif + if (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state)) { + while (slot->slot_unbuffered == 0) { + WT_RET(WT_SESSION_CHECK_PANIC(session)); + __wt_yield(); +#ifdef HAVE_DIAGNOSTIC + ++count; + if (count > WT_MILLION) { + __wt_epoch(session, &now); + if (WT_TIMEDIFF_SEC(now, begin) > 10) { + __wt_errx(session, "SLOT_CLOSE: Slot %" + PRIu32 " Timeout unbuffered, state 0x%" + PRIx64 " unbuffered %" PRIu64, + (uint32_t)(slot - &log->slot_pool[0]), + slot->slot_state, + slot->slot_unbuffered); + __log_slot_dump(session); + __wt_abort(session); + } + count = 0; + } +#endif + } + } end_offset = WT_LOG_SLOT_JOINED_BUFFERED(old_state) + slot->slot_unbuffered; @@ -117,18 +194,105 @@ retry: return (0); } +/* + * __log_slot_new -- + * Find a free slot and switch it as the new active slot. + * Must be called holding the slot lock. + */ +static int +__log_slot_new(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_LOGSLOT *slot; + int32_t i, pool_i; +#ifdef HAVE_DIAGNOSTIC + struct timespec begin, now; + int count; +#endif + + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SLOT)); + conn = S2C(session); + log = conn->log; + /* + * Although this function is single threaded, multiple threads could + * be trying to set a new active slot sequentially. If we find an + * active slot that is valid, return. + */ + if ((slot = log->active_slot) != NULL && + WT_LOG_SLOT_OPEN(slot->slot_state)) + return (0); + +#ifdef HAVE_DIAGNOSTIC + count = 0; + __wt_epoch(session, &begin); +#endif + /* + * Keep trying until we can find a free slot. + */ + for (;;) { + /* + * Rotate among the slots to lessen collisions. + */ + for (i = 0, pool_i = log->pool_index; i < WT_SLOT_POOL; + i++, pool_i++) { + if (pool_i >= WT_SLOT_POOL) + pool_i = 0; + slot = &log->slot_pool[pool_i]; + if (slot->slot_state == WT_LOG_SLOT_FREE) { + /* + * Acquire our starting position in the + * log file. Assume the full buffer size. + */ + WT_RET(__wt_log_acquire(session, + log->slot_buf_size, slot)); + /* + * We have a new, initialized slot to use. + * Set it as the active slot. + */ + WT_STAT_CONN_INCR(session, + log_slot_transitions); + log->active_slot = slot; + log->pool_index = pool_i; + return (0); + } + } + /* + * If we didn't find any free slots signal the worker thread. + */ + WT_STAT_CONN_INCR(session, log_slot_no_free_slots); + __wt_cond_signal(session, conn->log_wrlsn_cond); + __wt_yield(); +#ifdef HAVE_DIAGNOSTIC + ++count; + if (count > WT_MILLION) { + __wt_epoch(session, &now); + if (WT_TIMEDIFF_SEC(now, begin) > 10) { + __wt_errx(session, + "SLOT_NEW: Timeout free slot"); + __log_slot_dump(session); + __wt_abort(session); + } + count = 0; + } +#endif + } + /* NOTREACHED */ +} + /* * __log_slot_switch_internal -- * Switch out the current slot and set up a new one. */ static int __log_slot_switch_internal( - WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool forced) + WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool forced, bool *did_work) { WT_DECL_RET; WT_LOG *log; WT_LOGSLOT *slot; bool free_slot, release; + uint32_t joined; log = S2C(session)->log; release = false; @@ -142,10 +306,23 @@ __log_slot_switch_internal( */ if (slot != log->active_slot) return (0); + /* + * If the current active slot is unused and this is a forced switch, + * we're done. If this is a non-forced switch we always switch + * because the slot could be part of an unbuffered operation. + */ + joined = WT_LOG_SLOT_JOINED(slot->slot_state); + if (joined == 0 && forced) { + WT_STAT_CONN_INCR(session, log_force_write_skip); + if (did_work != NULL) + *did_work = false; + return (0); + } + WT_RET(WT_SESSION_CHECK_PANIC(session)); /* - * We may come through here multiple times if we were able to close - * a slot but could not set up a new one. If we closed it already, + * We may come through here multiple times if we were not able to + * set up a new one. If we closed it already, * don't try to do it again but still set up the new slot. */ if (!F_ISSET(myslot, WT_MYSLOT_CLOSE)) { @@ -157,20 +334,30 @@ __log_slot_switch_internal( if (ret == WT_NOTFOUND) return (0); WT_RET(ret); - if (release) { - WT_RET(__wt_log_release(session, slot, &free_slot)); - if (free_slot) - __wt_log_slot_free(session, slot); - } + /* + * Set that we have closed this slot because we may call in here + * multiple times if we retry creating a new slot. Similarly + * set retain whether this slot needs releasing so that we don't + * lose that information if we retry. + */ + F_SET(myslot, WT_MYSLOT_CLOSE); + if (release) + F_SET(myslot, WT_MYSLOT_NEEDS_RELEASE); } /* - * Set that we have closed this slot because we may call in here - * multiple times if we retry creating a new slot. + * Now that the slot is closed, set up a new one so that joining + * threads don't have to wait on writing the previous slot if we + * release it. Release after setting a new one. */ - F_SET(myslot, WT_MYSLOT_CLOSE); - WT_RET(__wt_log_slot_new(session)); + WT_RET(__log_slot_new(session)); F_CLR(myslot, WT_MYSLOT_CLOSE); - return (0); + if (F_ISSET(myslot, WT_MYSLOT_NEEDS_RELEASE)) { + WT_RET(__wt_log_release(session, slot, &free_slot)); + F_CLR(myslot, WT_MYSLOT_NEEDS_RELEASE); + if (free_slot) + __wt_log_slot_free(session, slot); + } + return (ret); } /* @@ -178,13 +365,14 @@ __log_slot_switch_internal( * Switch out the current slot and set up a new one. */ int -__wt_log_slot_switch( - WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced) +__wt_log_slot_switch(WT_SESSION_IMPL *session, + WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) { WT_DECL_RET; WT_LOG *log; log = S2C(session)->log; + /* * !!! Since the WT_WITH_SLOT_LOCK macro is a do-while loop, the * compiler does not like it combined directly with the while loop @@ -197,8 +385,9 @@ __wt_log_slot_switch( * because we are responsible for setting up the new slot. */ do { - WT_WITH_SLOT_LOCK(session, log, ret, - ret = __log_slot_switch_internal(session, myslot, forced)); + WT_WITH_SLOT_LOCK(session, log, + ret = __log_slot_switch_internal( + session, myslot, forced, did_work)); if (ret == EBUSY) { WT_STAT_CONN_INCR(session, log_slot_switch_busy); __wt_yield(); @@ -207,67 +396,6 @@ __wt_log_slot_switch( return (ret); } -/* - * __wt_log_slot_new -- - * Find a free slot and switch it as the new active slot. - * Must be called holding the slot lock. - */ -int -__wt_log_slot_new(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_LOG *log; - WT_LOGSLOT *slot; - int32_t i; - - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SLOT)); - conn = S2C(session); - log = conn->log; - /* - * Although this function is single threaded, multiple threads could - * be trying to set a new active slot sequentially. If we find an - * active slot that is valid, return. - */ - if ((slot = log->active_slot) != NULL && - WT_LOG_SLOT_OPEN(slot->slot_state)) - return (0); - - /* - * Keep trying until we can find a free slot. - */ - for (;;) { - /* - * For now just restart at 0. We could use log->pool_index - * if that is inefficient. - */ - for (i = 0; i < WT_SLOT_POOL; i++) { - slot = &log->slot_pool[i]; - if (slot->slot_state == WT_LOG_SLOT_FREE) { - /* - * Acquire our starting position in the - * log file. Assume the full buffer size. - */ - WT_RET(__wt_log_acquire(session, - log->slot_buf_size, slot)); - /* - * We have a new, initialized slot to use. - * Set it as the active slot. - */ - WT_STAT_CONN_INCR(session, - log_slot_transitions); - log->active_slot = slot; - return (0); - } - } - /* - * If we didn't find any free slots signal the worker thread. - */ - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); - __wt_yield(); - } - /* NOTREACHED */ -} - /* * __wt_log_slot_init -- * Initialize the slot array. @@ -283,7 +411,6 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) conn = S2C(session); log = conn->log; - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, log->slot_pool); for (i = 0; i < WT_SLOT_POOL; i++) log->slot_pool[i].slot_state = WT_LOG_SLOT_FREE; @@ -312,10 +439,13 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) /* * We cannot initialize the release LSN in the activate function * because that function can be called after a log file switch. + * The release LSN is usually the same as the slot_start_lsn except + * around a log file switch. */ slot->slot_release_lsn = log->alloc_lsn; __wt_log_slot_activate(session, slot); log->active_slot = slot; + log->pool_index = 0; if (0) { err: while (--i >= 0) @@ -360,10 +490,9 @@ __wt_log_slot_destroy(WT_SESSION_IMPL *session) /* * __wt_log_slot_join -- - * Join a consolidated logging slot. Must be called with - * the read lock held. + * Join a consolidated logging slot. */ -void +int __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) { @@ -372,71 +501,76 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, WT_LOGSLOT *slot; int64_t flag_state, new_state, old_state, released; int32_t join_offset, new_join; -#ifdef HAVE_DIAGNOSTIC - bool unbuf_force; -#endif + bool unbuffered, yld; conn = S2C(session); log = conn->log; - /* - * Make sure the length cannot overflow. The caller should not - * even call this function if it doesn't fit but use direct - * writes. - */ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SLOT)); + WT_ASSERT(session, mysize != 0); /* * There should almost always be a slot open. */ + unbuffered = false; #ifdef HAVE_DIAGNOSTIC - unbuf_force = (++log->write_calls % WT_THOUSAND) == 0; + yld = (++log->write_calls % 7) == 0; + if ((log->write_calls % WT_THOUSAND) == 0 || + mysize > WT_LOG_SLOT_BUF_MAX) { +#else + yld = false; + if (mysize > WT_LOG_SLOT_BUF_MAX) { #endif + unbuffered = true; + F_SET(myslot, WT_MYSLOT_UNBUFFERED); + } for (;;) { WT_BARRIER(); + WT_RET(WT_SESSION_CHECK_PANIC(session)); slot = log->active_slot; old_state = slot->slot_state; - /* - * Try to join our size into the existing size and - * atomically write it back into the state. - */ - flag_state = WT_LOG_SLOT_FLAGS(old_state); - released = WT_LOG_SLOT_RELEASED(old_state); - join_offset = WT_LOG_SLOT_JOINED(old_state); -#ifdef HAVE_DIAGNOSTIC - if (unbuf_force || mysize > WT_LOG_SLOT_BUF_MAX) { -#else - if (mysize > WT_LOG_SLOT_BUF_MAX) { -#endif - new_join = join_offset + WT_LOG_SLOT_UNBUFFERED; - F_SET(myslot, WT_MYSLOT_UNBUFFERED); - myslot->slot = slot; + if (WT_LOG_SLOT_OPEN(old_state)) { + /* + * Try to join our size into the existing size and + * atomically write it back into the state. + */ + flag_state = WT_LOG_SLOT_FLAGS(old_state); + released = WT_LOG_SLOT_RELEASED(old_state); + join_offset = WT_LOG_SLOT_JOINED(old_state); + if (unbuffered) + new_join = join_offset + WT_LOG_SLOT_UNBUFFERED; + else + new_join = join_offset + (int32_t)mysize; + new_state = (int64_t)WT_LOG_SLOT_JOIN_REL( + (int64_t)new_join, (int64_t)released, + (int64_t)flag_state); + + /* + * Braces used due to potential empty body warning. + */ + if (yld) { + WT_DIAGNOSTIC_YIELD; + } + /* + * Attempt to swap our size into the state. + */ + if (__wt_atomic_casiv64( + &slot->slot_state, old_state, new_state)) + break; + WT_STAT_CONN_INCR(session, log_slot_races); } else - new_join = join_offset + (int32_t)mysize; - new_state = (int64_t)WT_LOG_SLOT_JOIN_REL( - (int64_t)new_join, (int64_t)released, (int64_t)flag_state); - - /* - * Check if the slot is open for joining and we are able to - * swap in our size into the state. - */ - if (WT_LOG_SLOT_OPEN(old_state) && - __wt_atomic_casiv64( - &slot->slot_state, old_state, new_state)) - break; + WT_STAT_CONN_INCR(session, log_slot_active_closed); /* * The slot is no longer open or we lost the race to * update it. Yield and try again. */ - WT_STAT_CONN_INCR(session, log_slot_races); __wt_yield(); } /* * We joined this slot. Fill in our information to return to * the caller. */ - if (mysize != 0) - WT_STAT_CONN_INCR(session, log_slot_joins); + WT_STAT_CONN_INCR(session, log_slot_joins); if (LF_ISSET(WT_LOG_DSYNC | WT_LOG_FSYNC)) F_SET(slot, WT_SLOT_SYNC_DIR); if (LF_ISSET(WT_LOG_FLUSH)) @@ -451,6 +585,7 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, myslot->slot = slot; myslot->offset = join_offset; myslot->end_offset = (wt_off_t)((uint64_t)join_offset + mysize); + return (0); } /* @@ -466,10 +601,15 @@ __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size) wt_off_t cur_offset, my_start; int64_t my_size, rel_size; - WT_UNUSED(session); slot = myslot->slot; my_start = slot->slot_start_offset + myslot->offset; + /* + * We maintain the last starting offset within this slot. + * This is used to know the offset of the last record that + * was written rather than the beginning record of the slot. + */ while ((cur_offset = slot->slot_last_offset) < my_start) { + WT_RET(WT_SESSION_CHECK_PANIC(session)); /* * Set our offset if we are larger. */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index c20673563d9..4d00143b28f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -10,10 +10,10 @@ #define WT_FORALL_CURSORS(clsm, c, i) \ for ((i) = (clsm)->nchunks; (i) > 0;) \ - if (((c) = (clsm)->chunks[--i]->cursor) != NULL) + if (((c) = (clsm)->chunks[--(i)]->cursor) != NULL) #define WT_LSM_CURCMP(s, lsm_tree, c1, c2, cmp) \ - __wt_compare(s, (lsm_tree)->collator, &(c1)->key, &(c2)->key, &cmp) + __wt_compare(s, (lsm_tree)->collator, &(c1)->key, &(c2)->key, &(cmp)) static int __clsm_lookup(WT_CURSOR_LSM *, WT_ITEM *); static int __clsm_open_cursors(WT_CURSOR_LSM *, bool, u_int, uint32_t); @@ -34,7 +34,7 @@ __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) lsm_tree = clsm->lsm_tree; session = (WT_SESSION_IMPL *)clsm->iface.session; - if (!F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH)) { + if (!lsm_tree->need_switch) { /* * Check that we are up-to-date: don't set the switch if the * tree has changed since we last opened cursors: that can lead @@ -44,8 +44,8 @@ __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) __wt_lsm_tree_readlock(session, lsm_tree); if (lsm_tree->nchunks == 0 || (clsm->dsk_gen == lsm_tree->dsk_gen && - !F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH))) { - F_SET(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + !lsm_tree->need_switch)) { + lsm_tree->need_switch = true; ret = __wt_lsm_manager_push_entry( session, WT_LSM_WORK_SWITCH, 0, lsm_tree); } @@ -129,7 +129,7 @@ __clsm_enter_update(WT_CURSOR_LSM *clsm) * chunk grows twice as large as the configured size, block until it * can be switched. */ - hard_limit = F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + hard_limit = lsm_tree->need_switch; if (have_primary) { WT_ENTER_PAGE_INDEX(session); @@ -178,20 +178,12 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) if (reset) { WT_ASSERT(session, !F_ISSET(&clsm->iface, - WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT)); + WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT)); WT_RET(__clsm_reset_cursors(clsm, NULL)); } for (;;) { - /* - * If the cursor looks up-to-date, check if the cache is full. - * In case this call blocks, the check will be repeated before - * proceeding. - */ - if (clsm->dsk_gen != lsm_tree->dsk_gen && - lsm_tree->nchunks != 0) - goto open; - + /* Check if the cursor looks up-to-date. */ if (clsm->dsk_gen != lsm_tree->dsk_gen && lsm_tree->nchunks != 0) goto open; @@ -215,7 +207,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) goto open; if (txn->isolation == WT_ISO_SNAPSHOT) - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); /* * Figure out how many updates are required for @@ -267,7 +259,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) (!update && F_ISSET(clsm, WT_CLSM_OPEN_READ)))) break; -open: WT_WITH_SCHEMA_LOCK(session, ret, +open: WT_WITH_SCHEMA_LOCK(session, ret = __clsm_open_cursors(clsm, update, 0, 0)); WT_RET(ret); } @@ -304,7 +296,7 @@ __clsm_leave(WT_CURSOR_LSM *clsm) * byte, if the application uses two leading DC4 byte for some reason, we'll do * a wasted data copy each time a new value is inserted into the object. */ -static const WT_ITEM __tombstone = { "\x14\x14", 2, 0, NULL, 0 }; +static const WT_ITEM __tombstone = { "\x14\x14", 2, NULL, 0, 0 }; /* * __clsm_deleted -- @@ -409,13 +401,11 @@ static int __clsm_resize_chunks( WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, u_int nchunks) { - WT_DECL_RET; WT_LSM_CURSOR_CHUNK *chunk; /* Don't allocate more iterators if we don't need them. */ - if (clsm->chunks_count >= nchunks) { - return (ret); - } + if (clsm->chunks_count >= nchunks) + return (0); WT_RET(__wt_realloc_def(session, &clsm->chunks_alloc, nchunks, &clsm->chunks)); @@ -423,7 +413,7 @@ __clsm_resize_chunks( WT_RET(__wt_calloc_one(session, &chunk)); clsm->chunks[clsm->chunks_count] = chunk; } - return (ret); + return (0); } /* @@ -434,9 +424,10 @@ static void __clsm_free_chunks(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm) { size_t i; - for (i = 0; i < clsm->chunks_count; i++) { + + for (i = 0; i < clsm->chunks_count; i++) __wt_free(session, clsm->chunks[i]); - } + __wt_free(session, clsm->chunks); } @@ -667,7 +658,7 @@ retry: if (F_ISSET(clsm, WT_CLSM_MERGE)) { */ if (i != nchunks - 1) clsm->chunks[i]->cursor->insert = - __wt_curfile_update_check; + __wt_curfile_insert_check; if (!F_ISSET(clsm, WT_CLSM_MERGE) && F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) @@ -689,19 +680,29 @@ retry: if (F_ISSET(clsm, WT_CLSM_MERGE)) { if (chunk != NULL && !F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && chunk->switch_txn == WT_TXN_NONE) { - clsm->primary_chunk = chunk; primary = clsm->chunks[clsm->nchunks - 1]->cursor; + btree = ((WT_CURSOR_BTREE *)primary)->btree; + /* - * Disable eviction for the in-memory chunk. Also clear the - * bulk load flag here, otherwise eviction will be enabled by - * the first update. + * If the primary is not yet set as the primary, do that now. + * Note that eviction was configured off when the underlying + * object was created, which is what we want, leave it alone. + * + * We don't have to worry about races here: every thread that + * modifies the tree will have to come through here, at worse + * we set the flag repeatedly. We don't use a WT_BTREE handle + * flag, however, we could race doing the read-modify-write of + * the flags field. + * + * If something caused the chunk to be closed and reopened + * since it was created, we can no longer use it as a primary + * chunk and we need to force a switch. We detect the tree was + * created when it was opened by checking the "original" flag. */ - btree = ((WT_CURSOR_BTREE *)(primary))->btree; - if (btree->bulk_load_ok) { - btree->bulk_load_ok = false; - WT_WITH_BTREE(session, btree, - __wt_btree_evictable(session, false)); - } + if (!btree->lsm_primary && btree->original) + btree->lsm_primary = true; + if (btree->lsm_primary) + clsm->primary_chunk = chunk; } clsm->dsk_gen = lsm_tree->dsk_gen; @@ -763,7 +764,7 @@ __wt_clsm_init_merge( F_SET(clsm, WT_CLSM_MINOR_MERGE); clsm->nchunks = nchunks; - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __clsm_open_cursors(clsm, false, start_chunk, start_id)); return (ret); } @@ -1214,7 +1215,8 @@ __clsm_lookup(WT_CURSOR_LSM *clsm, WT_ITEM *value) WT_LSM_TREE_STAT_INCR( session, clsm->lsm_tree->bloom_miss); continue; - } else if (ret == 0) + } + if (ret == 0) WT_LSM_TREE_STAT_INCR( session, clsm->lsm_tree->bloom_hit); WT_ERR(ret); @@ -1240,10 +1242,10 @@ __clsm_lookup(WT_CURSOR_LSM *clsm, WT_ITEM *value) WT_ERR(WT_NOTFOUND); done: -err: F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if (ret == 0) { - clsm->current = c; +err: if (ret == 0) { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); F_SET(cursor, WT_CURSTD_KEY_INT); + clsm->current = c; if (value == &cursor->value) F_SET(cursor, WT_CURSTD_VALUE_INT); } else if (c != NULL) @@ -1319,7 +1321,8 @@ __clsm_search_near(WT_CURSOR *cursor, int *exactp) if ((ret = c->search_near(c, &cmp)) == WT_NOTFOUND) { ret = 0; continue; - } else if (ret != 0) + } + if (ret != 0) goto err; /* Do we have an exact match? */ @@ -1339,7 +1342,8 @@ __clsm_search_near(WT_CURSOR *cursor, int *exactp) if ((ret = c->next(c)) == WT_NOTFOUND) { ret = 0; continue; - } else if (ret != 0) + } + if (ret != 0) goto err; } @@ -1565,12 +1569,23 @@ __clsm_update(WT_CURSOR *cursor) WT_CURSOR_NEEDVALUE(cursor); WT_ERR(__clsm_enter(clsm, false, true)); - if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) || - (ret = __clsm_lookup(clsm, &value)) == 0) { - WT_ERR(__clsm_deleted_encode( - session, &cursor->value, &value, &buf)); - ret = __clsm_put(session, clsm, &cursor->key, &value, true); - } + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + WT_ERR(__clsm_lookup(clsm, &value)); + WT_ERR(__clsm_deleted_encode(session, &cursor->value, &value, &buf)); + WT_ERR(__clsm_put(session, clsm, &cursor->key, &value, true)); + + /* + * Set the cursor to reference the internal key/value of the positioned + * cursor. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_ITEM_SET(cursor->key, clsm->current->key); + WT_ITEM_SET(cursor->value, clsm->current->value); + WT_ASSERT(session, + F_MASK(clsm->current, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + WT_ASSERT(session, + F_MASK(clsm->current, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); err: __wt_scr_free(session, &buf); __clsm_leave(clsm); @@ -1589,18 +1604,34 @@ __clsm_remove(WT_CURSOR *cursor) WT_DECL_RET; WT_ITEM value; WT_SESSION_IMPL *session; + bool positioned; clsm = (WT_CURSOR_LSM *)cursor; + /* Check if the cursor is positioned. */ + positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); + CURSOR_REMOVE_API_CALL(cursor, session, NULL); WT_CURSOR_NEEDKEY(cursor); WT_CURSOR_NOVALUE(cursor); WT_ERR(__clsm_enter(clsm, false, true)); - if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) || - (ret = __clsm_lookup(clsm, &value)) == 0) - ret = __clsm_put( - session, clsm, &cursor->key, &__tombstone, true); + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + WT_ERR(__clsm_lookup(clsm, &value)); + WT_ERR(__clsm_put( + session, clsm, &cursor->key, &__tombstone, positioned)); + + /* + * If the cursor was positioned, it stays positioned with a key but no + * no value, otherwise, there's no position, key or value. This isn't + * just cosmetic, without a reset, iteration on this cursor won't start + * at the beginning/end of the table. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (positioned) + F_SET(cursor, WT_CURSTD_KEY_INT); + else + WT_TRET(cursor->reset(cursor)); err: __clsm_leave(clsm); CURSOR_UPDATE_API_END(session, ret); @@ -1630,8 +1661,6 @@ __wt_clsm_close(WT_CURSOR *cursor) /* In case we were somehow left positioned, clear that. */ __clsm_leave(clsm); - /* The WT_LSM_TREE owns the URI. */ - cursor->uri = NULL; if (clsm->lsm_tree != NULL) __wt_lsm_tree_release(session, clsm->lsm_tree); WT_TRET(__wt_cursor_close(cursor)); @@ -1693,8 +1722,8 @@ __wt_clsm_open(WT_SESSION_IMPL *session, bulk = cval.val != 0; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, bulk, &lsm_tree)); + ret = __wt_lsm_tree_get(session, uri, bulk, &lsm_tree); + /* * Check whether the exclusive open for a bulk load succeeded, and * if it did ensure that it's safe to bulk load into the tree. @@ -1713,7 +1742,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, cursor = &clsm->iface; *cursor = iface; cursor->session = &session->iface; - cursor->uri = lsm_tree->name; + WT_ERR(__wt_strdup(session, lsm_tree->name, &cursor->uri)); cursor->key_format = lsm_tree->key_format; cursor->value_format = lsm_tree->value_format; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c index 319426de3f0..7a6a40e380f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c @@ -45,7 +45,7 @@ __clsm_close_bulk(WT_CURSOR *cursor) total_chunks /= avg_chunks) ++chunk->generation; - WT_RET(__wt_lsm_meta_write(session, lsm_tree)); + WT_RET(__wt_lsm_meta_write(session, lsm_tree, NULL)); ++lsm_tree->dsk_gen; /* Close the LSM cursor */ @@ -113,7 +113,7 @@ __wt_clsm_open_bulk(WT_CURSOR_LSM *clsm, const char *cfg[]) * switch inline, since switch needs a schema lock and online index * creation opens a bulk cursor while holding the schema lock. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_lsm_tree_switch(session, lsm_tree)); WT_RET(ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index cbd83a5cd30..b7d9086d10e 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -89,7 +89,6 @@ __lsm_general_worker_start(WT_SESSION_IMPL *session) if (manager->lsm_workers % 2 == 0) FLD_SET(worker_args->type, WT_LSM_WORK_MERGE); } - F_SET(worker_args, WT_LSM_WORKER_RUN); WT_RET(__wt_lsm_worker_start(session, worker_args)); } @@ -129,17 +128,13 @@ __lsm_stop_workers(WT_SESSION_IMPL *session) manager->lsm_workers--) { worker_args = &manager->lsm_worker_cookies[manager->lsm_workers - 1]; - /* - * Clear this worker's flag so it stops. - */ - F_CLR(worker_args, WT_LSM_WORKER_RUN); - WT_ASSERT(session, worker_args->tid != 0); - WT_RET(__wt_thread_join(session, worker_args->tid)); - worker_args->tid = 0; + WT_ASSERT(session, worker_args->tid_set); + + WT_RET(__wt_lsm_worker_stop(session, worker_args)); worker_args->type = 0; - worker_args->flags = 0; + /* - * We do not clear the session because they are allocated + * We do not clear the other fields because they are allocated * statically when the connection was opened. */ } @@ -237,12 +232,12 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) manager->lsm_worker_cookies[i].session = worker_session; } + F_SET(conn, WT_CONN_SERVER_LSM); + /* Start the LSM manager thread. */ WT_ERR(__wt_thread_create(session, &manager->lsm_worker_cookies[0].tid, __lsm_worker_manager, &manager->lsm_worker_cookies[0])); - F_SET(conn, WT_CONN_SERVER_LSM); - if (0) { err: for (i = 0; (worker_session = @@ -289,13 +284,18 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) manager = &conn->lsm_manager; removed = 0; + /* + * Clear the LSM server flag and flush to ensure running threads see + * the state change. + */ + F_CLR(conn, WT_CONN_SERVER_LSM); + WT_FULL_BARRIER(); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || manager->lsm_workers == 0); if (manager->lsm_workers > 0) { - /* - * Stop the main LSM manager thread first. - */ - while (F_ISSET(conn, WT_CONN_SERVER_LSM)) + /* Wait for the main LSM manager thread to finish. */ + while (!F_ISSET(manager, WT_LSM_MANAGER_SHUTDOWN)) __wt_yield(); /* Clean up open LSM handles. */ @@ -303,7 +303,6 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join( session, manager->lsm_worker_cookies[0].tid)); - manager->lsm_worker_cookies[0].tid = 0; /* Release memory from any operations left on the queue. */ while ((current = TAILQ_FIRST(&manager->switchqh)) != NULL) { @@ -335,14 +334,14 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &manager->switch_lock); __wt_spin_destroy(session, &manager->app_lock); __wt_spin_destroy(session, &manager->manager_lock); - WT_TRET(__wt_cond_destroy(session, &manager->work_cond)); + __wt_cond_destroy(session, &manager->work_cond); return (ret); } /* * __lsm_manager_worker_shutdown -- - * Shutdown the LSM manager and worker threads. + * Shutdown the LSM worker threads. */ static int __lsm_manager_worker_shutdown(WT_SESSION_IMPL *session) @@ -354,14 +353,13 @@ __lsm_manager_worker_shutdown(WT_SESSION_IMPL *session) manager = &S2C(session)->lsm_manager; /* - * Wait for the rest of the LSM workers to shutdown. Stop at index + * Wait for the rest of the LSM workers to shutdown. Start at index * one - since we (the manager) are at index 0. */ for (i = 1; i < manager->lsm_workers; i++) { - WT_ASSERT(session, manager->lsm_worker_cookies[i].tid != 0); - __wt_cond_signal(session, manager->work_cond); - WT_TRET(__wt_thread_join( - session, manager->lsm_worker_cookies[i].tid)); + WT_ASSERT(session, manager->lsm_worker_cookies[i].tid_set); + WT_TRET(__wt_lsm_worker_stop( + session, &manager->lsm_worker_cookies[i])); } return (ret); } @@ -383,12 +381,12 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) conn = S2C(session); dhandle_locked = false; - while (F_ISSET(conn, WT_CONN_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LSM)) { __wt_sleep(0, 10000); if (TAILQ_EMPTY(&conn->lsmqh)) continue; - __wt_spin_lock(session, &conn->dhandle_lock); - F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST); + __wt_readlock(session, &conn->dhandle_lock); + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); dhandle_locked = true; TAILQ_FOREACH(lsm_tree, &S2C(session)->lsmqh, q) { if (!lsm_tree->active) @@ -448,14 +446,14 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) session, WT_LSM_WORK_MERGE, 0, lsm_tree)); } } - __wt_spin_unlock(session, &conn->dhandle_lock); - F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST); + __wt_readunlock(session, &conn->dhandle_lock); + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); dhandle_locked = false; } err: if (dhandle_locked) { - __wt_spin_unlock(session, &conn->dhandle_lock); - F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST); + __wt_readunlock(session, &conn->dhandle_lock); + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); } return (ret); } @@ -469,11 +467,13 @@ static WT_THREAD_RET __lsm_worker_manager(void *arg) { WT_DECL_RET; + WT_LSM_MANAGER *manager; WT_LSM_WORKER_ARGS *cookie; WT_SESSION_IMPL *session; cookie = (WT_LSM_WORKER_ARGS *)arg; session = cookie->session; + manager = &S2C(session)->lsm_manager; WT_ERR(__lsm_general_worker_start(session)); WT_ERR(__lsm_manager_run_server(session)); @@ -482,7 +482,11 @@ __lsm_worker_manager(void *arg) if (ret != 0) { err: WT_PANIC_MSG(session, ret, "LSM worker manager thread error"); } - F_CLR(S2C(session), WT_CONN_SERVER_LSM); + + /* Connection close waits on us to shutdown, let it know we're done. */ + F_SET(manager, WT_LSM_MANAGER_SHUTDOWN); + WT_FULL_BARRIER(); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index 493855d489a..8838638f388 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -46,7 +46,7 @@ __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, static void __lsm_merge_aggressive_clear(WT_LSM_TREE *lsm_tree) { - F_CLR(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER); + lsm_tree->aggressive_timer_enabled = false; lsm_tree->merge_aggressiveness = 0; } @@ -85,12 +85,12 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) } /* - * Start the timer if it isn't running. Use a flag to define whether + * Start the timer if it isn't running. Use a bool to define whether * the timer is running - since clearing and checking a special * timer value isn't simple. */ - if (!F_ISSET(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER)) { - F_SET(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER); + if (!lsm_tree->aggressive_timer_enabled) { + lsm_tree->aggressive_timer_enabled = true; __wt_epoch(session, &lsm_tree->merge_aggressive_ts); } @@ -187,7 +187,7 @@ __lsm_merge_span(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, continue; if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM) || chunk->generation > 0) break; - else if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && + if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) break; } @@ -434,7 +434,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) F_SET(src, WT_CURSTD_RAW); WT_ERR(__wt_clsm_init_merge(src, start_chunk, start_id, nchunks)); - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_lsm_tree_setup_chunk(session, lsm_tree, chunk)); WT_ERR(ret); if (create_bloom) { @@ -579,7 +579,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) * Any errors that happened after the tree was locked are * fatal - we can't guarantee the state of the tree. */ - if ((ret = __wt_lsm_meta_write(session, lsm_tree)) != 0) + if ((ret = __wt_lsm_meta_write(session, lsm_tree, NULL)) != 0) WT_PANIC_ERR(session, ret, "Failed finalizing LSM merge"); lsm_tree->dsk_gen++; @@ -604,13 +604,13 @@ err: if (locked) if (ret != 0 && created_chunk) { /* Drop the newly-created files on error. */ if (chunk->uri != NULL) { - WT_WITH_SCHEMA_LOCK(session, tret, + WT_WITH_SCHEMA_LOCK(session, tret = __wt_schema_drop( session, chunk->uri, drop_cfg)); WT_TRET(tret); } if (create_bloom && chunk->bloom_uri != NULL) { - WT_WITH_SCHEMA_LOCK(session, tret, + WT_WITH_SCHEMA_LOCK(session, tret = __wt_schema_drop( session, chunk->bloom_uri, drop_cfg)); WT_TRET(tret); @@ -625,7 +625,7 @@ err: if (locked) else __wt_verbose(session, WT_VERB_LSM, "Merge failed with %s", - __wt_strerror(session, ret, NULL, 0)); + __wt_strerror(session, ret, NULL, 0)); } F_CLR(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_EVICTION); return (ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index ec52af96231..fc4dde82470 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -229,7 +229,7 @@ __lsm_meta_read_v1( cv.len -= 2; } WT_ERR(__wt_config_check(session, - WT_CONFIG_REF(session, WT_SESSION_create), cv.str, cv.len)); + WT_CONFIG_REF(session, WT_SESSION_create), cv.str, cv.len)); WT_ERR(__wt_strndup(session, cv.str, cv.len, &lsm_tree->bloom_config)); WT_ERR(__wt_config_getones( session, lsmconf, "lsm.bloom_hash_count", &cv)); @@ -454,13 +454,14 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) * Write the metadata for an LSM tree. */ int -__wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +__wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, + const char *newconfig) { WT_DECL_ITEM(buf); WT_DECL_RET; WT_LSM_CHUNK *chunk; u_int i; - const char *new_cfg[] = { NULL, NULL, NULL }; + const char *new_cfg[] = { NULL, NULL, NULL, NULL, NULL }; char *new_metadata; bool first; @@ -504,8 +505,10 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_ERR(__wt_buf_catfmt(session, buf, "]")); /* Update the existing configuration with the new values. */ - new_cfg[0] = lsm_tree->config; - new_cfg[1] = buf->data; + new_cfg[0] = WT_CONFIG_BASE(session, lsm_meta); + new_cfg[1] = lsm_tree->config; + new_cfg[2] = buf->data; + new_cfg[3] = newconfig; WT_ERR(__wt_config_collapse(session, new_cfg, &new_metadata)); ret = __wt_metadata_update(session, lsm_tree->name, new_metadata); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_stat.c b/src/third_party/wiredtiger/src/lsm/lsm_stat.c index 3fe3ca1ba81..411655878af 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_stat.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_stat.c @@ -29,24 +29,22 @@ __curstat_lsm_init( const char *cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL, NULL }; const char *disk_cfg[] = { - WT_CONFIG_BASE(session, WT_SESSION_open_cursor), - "checkpoint=" WT_CHECKPOINT, NULL, NULL }; + WT_CONFIG_BASE(session, WT_SESSION_open_cursor), + "checkpoint=" WT_CHECKPOINT, NULL, NULL }; locked = false; - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree)); WT_ERR(__wt_scr_alloc(session, 0, &uribuf)); /* Propagate all, fast and/or clear to the cursors we open. */ if (cst->flags != 0) { - (void)snprintf(config, sizeof(config), + WT_ERR(__wt_snprintf(config, sizeof(config), "statistics=(%s%s%s%s)", F_ISSET(cst, WT_STAT_TYPE_ALL) ? "all," : "", F_ISSET(cst, WT_STAT_CLEAR) ? "clear," : "", !F_ISSET(cst, WT_STAT_TYPE_ALL) && F_ISSET(cst, WT_STAT_TYPE_FAST) ? "fast," : "", - F_ISSET(cst, WT_STAT_TYPE_SIZE) ? "size," : ""); + F_ISSET(cst, WT_STAT_TYPE_SIZE) ? "size," : "")); cfg[1] = disk_cfg[1] = config; } @@ -178,7 +176,7 @@ __wt_curstat_lsm_init( * Grab the schema lock because we will be locking the LSM tree and we * may need to open some files. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __curstat_lsm_init(session, uri, cst)); return (ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 0054dcd1583..36b31358c0b 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -38,7 +38,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) /* We may be destroying an lsm_tree before it was added. */ if (F_ISSET(lsm_tree, WT_LSM_TREE_OPEN)) { WT_ASSERT(session, final || - F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); TAILQ_REMOVE(&S2C(session)->lsmqh, lsm_tree, q); } @@ -251,7 +251,7 @@ __lsm_tree_cleanup_old(WT_SESSION_IMPL *session, const char *uri) WT_RET(__wt_fs_exist(session, uri + strlen("file:"), &exists)); if (exists) - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_drop(session, uri, cfg)); return (ret); } @@ -293,8 +293,6 @@ int __wt_lsm_tree_setup_bloom( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) { - WT_DECL_RET; - /* * The Bloom URI can be populated when the chunk is created, but * it isn't set yet on open or merge. @@ -302,8 +300,8 @@ __wt_lsm_tree_setup_bloom( if (chunk->bloom_uri == NULL) WT_RET(__wt_lsm_tree_bloom_name( session, lsm_tree, chunk->id, &chunk->bloom_uri)); - WT_RET(__lsm_tree_cleanup_old(session, chunk->bloom_uri)); - return (ret); + + return (__lsm_tree_cleanup_old(session, chunk->bloom_uri)); } /* @@ -323,9 +321,7 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, metadata = NULL; /* If the tree can be opened, it already exists. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)); - if (ret == 0) { + if ((ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)) == 0) { __wt_lsm_tree_release(session, lsm_tree); return (exclusive ? EEXIST : 0); } @@ -341,7 +337,7 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, * error: the returned handle is NULL on error, and the metadata * tracking macros handle cleaning up on failure. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __lsm_tree_open(session, uri, true, &lsm_tree)); if (ret == 0) __wt_lsm_tree_release(session, lsm_tree); @@ -406,6 +402,9 @@ __lsm_tree_find(WT_SESSION_IMPL *session, } *treep = lsm_tree; + + WT_ASSERT(session, lsm_tree->excl_session == + (exclusive ? session : NULL)); return (0); } @@ -458,7 +457,8 @@ __lsm_tree_open(WT_SESSION_IMPL *session, conn = S2C(session); lsm_tree = NULL; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + WT_ASSERT(session, + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); /* Start the LSM manager thread if it isn't running. */ if (__wt_atomic_cas32(&conn->lsm_manager.lsm_workers, 0, 1)) @@ -471,7 +471,7 @@ __lsm_tree_open(WT_SESSION_IMPL *session, /* Try to open the tree. */ WT_RET(__wt_calloc_one(session, &lsm_tree)); - WT_ERR(__wt_rwlock_alloc(session, &lsm_tree->rwlock, "lsm tree")); + WT_ERR(__wt_rwlock_init(session, &lsm_tree->rwlock)); WT_ERR(__lsm_tree_set_name(session, lsm_tree, uri)); @@ -522,14 +522,21 @@ __wt_lsm_tree_get(WT_SESSION_IMPL *session, { WT_DECL_RET; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); - - ret = __lsm_tree_find(session, uri, exclusive, treep); + /* + * Dropping and re-acquiring the lock is safe here, since the tree open + * call checks to see if another thread beat it to opening the tree + * before proceeding. + */ + if (exclusive) + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __lsm_tree_find(session, uri, exclusive, treep)); + else + WT_WITH_HANDLE_LIST_READ_LOCK(session, + ret = __lsm_tree_find(session, uri, exclusive, treep)); if (ret == WT_NOTFOUND) - ret = __lsm_tree_open(session, uri, exclusive, treep); + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __lsm_tree_open(session, uri, exclusive, treep)); - WT_ASSERT(session, ret != 0 || - (*treep)->excl_session == (exclusive ? session : NULL)); return (ret); } @@ -736,7 +743,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (!first_switch && (last_chunk = lsm_tree->chunk[nchunks - 1]) != NULL && !F_ISSET(last_chunk, WT_LSM_CHUNK_ONDISK) && - !F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH)) + !lsm_tree->need_switch) goto err; /* Update the throttle time. */ @@ -758,8 +765,8 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) lsm_tree->chunk[lsm_tree->nchunks++] = chunk; WT_ERR(__wt_lsm_tree_setup_chunk(session, lsm_tree, chunk)); - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); - F_CLR(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); + lsm_tree->need_switch = false; ++lsm_tree->dsk_gen; lsm_tree->modified = true; @@ -842,6 +849,45 @@ __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, return (0); } +/* + * __wt_lsm_tree_alter -- + * Alter an LSM tree. + */ +int +__wt_lsm_tree_alter( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DECL_RET; + WT_LSM_CHUNK *chunk; + WT_LSM_TREE *lsm_tree; + u_int i; + bool locked; + + locked = false; + + /* Get the LSM tree. */ + WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree)); + + /* Prevent any new opens. */ + __wt_lsm_tree_writelock(session, lsm_tree); + locked = true; + + /* Alter the chunks. */ + for (i = 0; i < lsm_tree->nchunks; i++) { + chunk = lsm_tree->chunk[i]; + WT_ERR(__wt_schema_alter(session, chunk->uri, cfg)); + if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) + WT_ERR( + __wt_schema_alter(session, chunk->bloom_uri, cfg)); + } + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); + +err: if (locked) + __wt_lsm_tree_writeunlock(session, lsm_tree); + __wt_lsm_tree_release(session, lsm_tree); + return (ret); +} + /* * __wt_lsm_tree_drop -- * Drop an LSM tree. @@ -860,9 +906,7 @@ __wt_lsm_tree_drop( locked = false; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, name, true, &lsm_tree)); WT_ASSERT(session, !lsm_tree->active); /* Prevent any new opens. */ @@ -895,7 +939,7 @@ __wt_lsm_tree_drop( WT_ASSERT(session, !lsm_tree->active); err: if (locked) __wt_lsm_tree_writeunlock(session, lsm_tree); - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __lsm_tree_discard(session, lsm_tree, false)); WT_TRET(tret); return (ret); @@ -921,9 +965,7 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, locked = false; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, olduri, true, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, olduri, true, &lsm_tree)); /* Prevent any new opens. */ __wt_lsm_tree_writelock(session, lsm_tree); @@ -955,7 +997,7 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, } } - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); locked = false; __wt_lsm_tree_writeunlock(session, lsm_tree); WT_ERR(__wt_metadata_remove(session, olduri)); @@ -968,7 +1010,7 @@ err: if (locked) * Discard this LSM tree structure. The first operation on the renamed * tree will create a new one. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __lsm_tree_discard(session, lsm_tree, false)); WT_TRET(tret); return (ret); @@ -993,9 +1035,7 @@ __wt_lsm_tree_truncate( locked = false; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, name, true, &lsm_tree)); /* Prevent any new opens. */ __wt_lsm_tree_writelock(session, lsm_tree); @@ -1010,7 +1050,7 @@ __wt_lsm_tree_truncate( WT_ERR(__wt_lsm_merge_update_tree( session, lsm_tree, 0, lsm_tree->nchunks, chunk)); - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); locked = false; __wt_lsm_tree_writeunlock(session, lsm_tree); @@ -1029,7 +1069,7 @@ err: if (locked) * the last good version of the metadata will be used, resulting * in a valid (not truncated) tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __lsm_tree_discard(session, lsm_tree, false)); WT_TRET(tret); } @@ -1043,7 +1083,7 @@ err: if (locked) void __wt_lsm_tree_readlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { - __wt_readlock(session, lsm_tree->rwlock); + __wt_readlock(session, &lsm_tree->rwlock); /* * Diagnostic: avoid deadlocks with the schema lock: if we need it for @@ -1061,7 +1101,7 @@ __wt_lsm_tree_readunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { F_CLR(session, WT_SESSION_NO_EVICTION | WT_SESSION_NO_SCHEMA_LOCK); - __wt_readunlock(session, lsm_tree->rwlock); + __wt_readunlock(session, &lsm_tree->rwlock); } /* @@ -1071,7 +1111,7 @@ __wt_lsm_tree_readunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) void __wt_lsm_tree_writelock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { - __wt_writelock(session, lsm_tree->rwlock); + __wt_writelock(session, &lsm_tree->rwlock); /* * Diagnostic: avoid deadlocks with the schema lock: if we need it for @@ -1089,7 +1129,7 @@ __wt_lsm_tree_writeunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { F_CLR(session, WT_SESSION_NO_EVICTION | WT_SESSION_NO_SCHEMA_LOCK); - __wt_writeunlock(session, lsm_tree->rwlock); + __wt_writeunlock(session, &lsm_tree->rwlock); } /* @@ -1102,7 +1142,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) WT_DECL_RET; WT_LSM_CHUNK *chunk; WT_LSM_TREE *lsm_tree; - time_t begin, end; uint64_t progress; uint32_t i; bool compacting, flushing, locked, ref; @@ -1119,9 +1158,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) /* Tell __wt_schema_worker not to look inside the LSM tree. */ *skipp = true; - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, name, false, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, name, false, &lsm_tree)); if (!F_ISSET(S2C(session), WT_CONN_LSM_MERGE)) WT_ERR_MSG(session, EINVAL, @@ -1139,8 +1176,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) return (0); } - __wt_seconds(session, &begin); - /* * Compacting has two distinct phases. * 1. All in-memory chunks up to and including the current @@ -1266,12 +1301,9 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) } else break; } + WT_ERR(__wt_session_compact_check_timeout(session)); __wt_sleep(1, 0); - __wt_seconds(session, &end); - if (session->compact->max_time > 0 && - session->compact->max_time < (uint64_t)(end - begin)) { - WT_ERR(ETIMEDOUT); - } + /* * Push merge operations while they are still getting work * done. If we are pushing merges, make sure they are @@ -1323,9 +1355,7 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, locked = false; exclusive = FLD_ISSET(open_flags, WT_DHANDLE_EXCLUSIVE); - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, exclusive, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, uri, exclusive, &lsm_tree)); /* * We mark that we're busy using the tree to coordinate diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 917104031fc..b55db012451 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -171,12 +171,12 @@ __wt_lsm_work_switch( *ran = false; *entryp = NULL; - if (F_ISSET(entry->lsm_tree, WT_LSM_TREE_NEED_SWITCH)) { - WT_WITH_SCHEMA_LOCK(session, ret, + if (entry->lsm_tree->need_switch) { + WT_WITH_SCHEMA_LOCK(session, ret = __wt_lsm_tree_switch(session, entry->lsm_tree)); /* Failing to complete the switch is fine */ if (ret == EBUSY) { - if (F_ISSET(entry->lsm_tree, WT_LSM_TREE_NEED_SWITCH)) + if (entry->lsm_tree->need_switch) WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_SWITCH, 0, entry->lsm_tree)); ret = 0; @@ -230,7 +230,7 @@ __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (__wt_atomic_cas32(&chunk->bloom_busy, 0, 1)) { if (!F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) { ret = __lsm_bloom_create( - session, lsm_tree, chunk, (u_int)i); + session, lsm_tree, chunk, i); /* * Record if we were successful so that we can * later push a merge work unit. @@ -265,9 +265,9 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, { WT_DECL_RET; WT_TXN_ISOLATION saved_isolation; - bool flush_set; + bool flush_set, release_btree; - flush_set = false; + flush_set = release_btree = false; /* * If the chunk is already checkpointed, make sure it is also evicted. @@ -276,7 +276,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && !F_ISSET(chunk, WT_LSM_CHUNK_STABLE) && !chunk->evicted) { - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __lsm_discard_handle(session, chunk->uri, NULL)); if (ret == 0) chunk->evicted = 1; @@ -318,19 +318,18 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * We can wait here for checkpoints and fsyncs to complete, which can * take a long time. */ - if ((ret = __wt_session_get_btree( - session, chunk->uri, NULL, NULL, 0)) == 0) { - /* - * Set read-uncommitted: we have already checked that all of the - * updates in this chunk are globally visible, use the cheapest - * possible check in reconciliation. - */ - saved_isolation = session->txn.isolation; - session->txn.isolation = WT_ISO_READ_UNCOMMITTED; - ret = __wt_cache_op(session, WT_SYNC_WRITE_LEAVES); - session->txn.isolation = saved_isolation; - WT_TRET(__wt_session_release_btree(session)); - } + WT_ERR(__wt_session_get_btree(session, chunk->uri, NULL, NULL, 0)); + release_btree = true; + + /* + * Set read-uncommitted: we have already checked that all of the updates + * in this chunk are globally visible, use the cheapest possible check + * in reconciliation. + */ + saved_isolation = session->txn.isolation; + session->txn.isolation = WT_ISO_READ_UNCOMMITTED; + ret = __wt_cache_op(session, WT_SYNC_WRITE_LEAVES); + session->txn.isolation = saved_isolation; WT_ERR(ret); __wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointing %s", @@ -346,14 +345,16 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * time, and our checkpoint operation should be very quick. */ WT_ERR(__wt_meta_track_on(session)); - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, chunk->uri, __wt_checkpoint, NULL, NULL, 0))); + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_checkpoint(session, NULL))); WT_TRET(__wt_meta_track_off(session, false, ret != 0)); if (ret != 0) WT_ERR_MSG(session, ret, "LSM checkpoint"); + release_btree = false; + WT_ERR(__wt_session_release_btree(session)); + /* Now the file is written, get the chunk size. */ WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); @@ -364,7 +365,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, /* Lock the tree, mark the chunk as on disk and update the metadata. */ __wt_lsm_tree_writelock(session, lsm_tree); F_SET(chunk, WT_LSM_CHUNK_ONDISK); - ret = __wt_lsm_meta_write(session, lsm_tree); + ret = __wt_lsm_meta_write(session, lsm_tree, NULL); ++lsm_tree->dsk_gen; /* Update the throttle time. */ @@ -376,16 +377,6 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_PUBLISH(chunk->flushing, 0); flush_set = false; - /* - * Clear the no-eviction flag so the primary can be evicted and - * eventually closed. Only do this once the checkpoint has succeeded: - * otherwise, accessing the leaf page during the checkpoint can trigger - * forced eviction. - */ - WT_ERR(__wt_session_get_btree(session, chunk->uri, NULL, NULL, 0)); - __wt_btree_evictable(session, true); - WT_ERR(__wt_session_release_btree(session)); - /* Make sure we aren't pinning a transaction ID. */ __wt_txn_release_snapshot(session); @@ -402,6 +393,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, err: if (flush_set) WT_PUBLISH(chunk->flushing, 0); + if (release_btree) + WT_TRET(__wt_session_release_btree(session)); return (ret); } @@ -469,7 +462,7 @@ __lsm_bloom_create(WT_SESSION_IMPL *session, /* Ensure the bloom filter is in the metadata. */ __wt_lsm_tree_writelock(session, lsm_tree); F_SET(chunk, WT_LSM_CHUNK_BLOOM); - ret = __wt_lsm_meta_write(session, lsm_tree); + ret = __wt_lsm_meta_write(session, lsm_tree, NULL); ++lsm_tree->dsk_gen; __wt_lsm_tree_writeunlock(session, lsm_tree); @@ -517,8 +510,8 @@ __lsm_drop_file(WT_SESSION_IMPL *session, const char *uri) * * This will fail with EBUSY if the file is still in use. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __lsm_discard_handle(session, uri, WT_CHECKPOINT)); + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __lsm_discard_handle(session, uri, WT_CHECKPOINT)); WT_RET(ret); /* @@ -526,7 +519,7 @@ __lsm_drop_file(WT_SESSION_IMPL *session, const char *uri) * results in the hot backup lock being taken when it updates the * metadata (which would be too late to prevent our drop). */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_drop(session, uri, drop_cfg)); if (ret == 0) @@ -610,7 +603,8 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (drop_ret == EBUSY) { ++skipped; continue; - } else if (drop_ret != ENOENT) + } + if (drop_ret != ENOENT) WT_ERR(drop_ret); flush_metadata = true; @@ -621,7 +615,8 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (drop_ret == EBUSY) { ++skipped; continue; - } else if (drop_ret != ENOENT) + } + if (drop_ret != ENOENT) WT_ERR(drop_ret); flush_metadata = true; } @@ -659,7 +654,7 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) err: /* Flush the metadata unless the system is in panic */ if (flush_metadata && ret != WT_PANIC) { __wt_lsm_tree_writelock(session, lsm_tree); - WT_TRET(__wt_lsm_meta_write(session, lsm_tree)); + WT_TRET(__wt_lsm_meta_write(session, lsm_tree, NULL)); __wt_lsm_tree_writeunlock(session, lsm_tree); } __lsm_unpin_chunks(session, &cookie); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_worker.c b/src/third_party/wiredtiger/src/lsm/lsm_worker.c index b0d0758775d..1cabbd4888d 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_worker.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_worker.c @@ -21,7 +21,23 @@ __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) { __wt_verbose(session, WT_VERB_LSM_MANAGER, "Start LSM worker %u type %#" PRIx32, args->id, args->type); - return (__wt_thread_create(session, &args->tid, __lsm_worker, args)); + + args->running = true; + WT_RET(__wt_thread_create(session, &args->tid, __lsm_worker, args)); + args->tid_set = true; + return (0); +} + +/* + * __wt_lsm_worker_stop -- + * A wrapper around the LSM worker thread stop. + */ +int +__wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) +{ + args->running = false; + args->tid_set = false; + return (__wt_thread_join(session, args->tid)); } /* @@ -84,7 +100,6 @@ err: __wt_lsm_manager_free_work_unit(session, entry); static WT_THREAD_RET __lsm_worker(void *arg) { - WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LSM_WORK_UNIT *entry; WT_LSM_WORKER_ARGS *cookie; @@ -93,11 +108,9 @@ __lsm_worker(void *arg) cookie = (WT_LSM_WORKER_ARGS *)arg; session = cookie->session; - conn = S2C(session); entry = NULL; - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(cookie, WT_LSM_WORKER_RUN)) { + while (cookie->running) { progress = false; /* @@ -154,7 +167,7 @@ __lsm_worker(void *arg) /* Don't busy wait if there was any work to do. */ if (!progress) { - __wt_cond_wait(session, cookie->work_cond, 10000); + __wt_cond_wait(session, cookie->work_cond, 10000, NULL); continue; } } diff --git a/src/third_party/wiredtiger/src/meta/meta_apply.c b/src/third_party/wiredtiger/src/meta/meta_apply.c index fb483c21dd9..dc93180a5e5 100644 --- a/src/third_party/wiredtiger/src/meta/meta_apply.c +++ b/src/third_party/wiredtiger/src/meta/meta_apply.c @@ -45,11 +45,7 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, session, uri, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - if (WT_META_TRACKING(session)) - WT_TRET(__wt_meta_track_handle_lock( - session, false)); - else - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_btree(session)); WT_RET(ret); } WT_RET_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index b985104c2eb..151bbe0e081 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -297,7 +297,7 @@ __wt_meta_ckptlist_get( *ckptbasep = ckptbase; if (0) { -err: __wt_meta_ckptlist_free(session, ckptbase); +err: __wt_meta_ckptlist_free(session, &ckptbase); } __wt_free(session, config); __wt_scr_free(session, &buf); @@ -463,16 +463,16 @@ err: __wt_scr_free(session, &buf); * Discard the checkpoint array. */ void -__wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT *ckptbase) +__wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep) { - WT_CKPT *ckpt; + WT_CKPT *ckpt, *ckptbase; - if (ckptbase == NULL) + if ((ckptbase = *ckptbasep) == NULL) return; WT_CKPT_FOREACH(ckptbase, ckpt) __wt_meta_checkpoint_free(session, ckpt); - __wt_free(session, ckptbase); + __wt_free(session, *ckptbasep); } /* diff --git a/src/third_party/wiredtiger/src/meta/meta_ext.c b/src/third_party/wiredtiger/src/meta/meta_ext.c index b48f7205807..aa1ea8b974d 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ext.c +++ b/src/third_party/wiredtiger/src/meta/meta_ext.c @@ -88,6 +88,7 @@ __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { return (__wt_meta_ckptlist_get( (WT_SESSION_IMPL *)session, name, ckptbasep)); @@ -99,6 +100,7 @@ __wt_metadata_get_ckptlist( */ void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - __wt_meta_ckptlist_free((WT_SESSION_IMPL *)session, ckptbase); + __wt_meta_ckptlist_free((WT_SESSION_IMPL *)session, &ckptbase); } diff --git a/src/third_party/wiredtiger/src/meta/meta_table.c b/src/third_party/wiredtiger/src/meta/meta_table.c index 76f4ef0c25f..aca69d0e6a2 100644 --- a/src/third_party/wiredtiger/src/meta/meta_table.c +++ b/src/third_party/wiredtiger/src/meta/meta_table.c @@ -68,9 +68,6 @@ __wt_metadata_cursor_open( if (F_ISSET(btree, WT_BTREE_NO_LOGGING)) F_CLR(btree, WT_BTREE_NO_LOGGING); - /* The metadata file always uses checkpoint IDs in visibility checks. */ - btree->include_checkpoint_txn = true; - return (0); } @@ -195,7 +192,7 @@ __wt_metadata_update( __metadata_turtle(key) ? "" : "not "); if (__metadata_turtle(key)) { - WT_WITH_TURTLE_LOCK(session, ret, + WT_WITH_TURTLE_LOCK(session, ret = __wt_turtle_update(session, key, value)); return (ret); } @@ -262,8 +259,17 @@ __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) key, WT_META_TRACKING(session) ? "true" : "false", __metadata_turtle(key) ? "" : "not "); - if (__metadata_turtle(key)) - return (__wt_turtle_read(session, key, valuep)); + if (__metadata_turtle(key)) { + /* + * The returned value should only be set if ret is non-zero, but + * Coverity is convinced otherwise. The code path is used enough + * that Coverity complains a lot, add an error check to get some + * peace and quiet. + */ + if ((ret = __wt_turtle_read(session, key, valuep)) != 0) + __wt_free(session, *valuep); + return (ret); + } /* * All metadata reads are at read-uncommitted isolation. That's diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index 9655a0b26a7..460b615b267 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -286,7 +286,7 @@ __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) */ ckpt_session->txn.id = session->txn.id; F_SET(ckpt_session, WT_SESSION_LOCKED_METADATA); - WT_WITH_METADATA_LOCK(session, ret, + WT_WITH_METADATA_LOCK(session, WT_WITH_DHANDLE(ckpt_session, WT_SESSION_META_DHANDLE(session), ret = __wt_checkpoint(ckpt_session, NULL))); diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index ace0fabab48..5a089471059 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -47,7 +47,7 @@ __metadata_init(WT_SESSION_IMPL *session) * We're single-threaded, but acquire the schema lock regardless: the * lower level code checks that it is appropriately synchronized. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_create(session, WT_METAFILE_URI, NULL)); return (ret); @@ -220,9 +220,8 @@ __wt_turtle_init(WT_SESSION_IMPL *session) /* Create the turtle file. */ WT_RET(__metadata_config(session, &metaconf)); - WT_WITH_TURTLE_LOCK(session, ret, - ret = __wt_turtle_update( - session, WT_METAFILE_URI, metaconf)); + WT_WITH_TURTLE_LOCK(session, ret = + __wt_turtle_update(session, WT_METAFILE_URI, metaconf)); WT_ERR(ret); } @@ -243,7 +242,7 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_DECL_ITEM(buf); WT_DECL_RET; WT_FSTREAM *fs; - bool exist, match; + bool exist; *valuep = NULL; @@ -259,22 +258,19 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) __metadata_config(session, valuep) : WT_NOTFOUND); WT_RET(__wt_fopen(session, WT_METADATA_TURTLE, 0, WT_STREAM_READ, &fs)); - /* Search for the key. */ WT_ERR(__wt_scr_alloc(session, 512, &buf)); - for (match = false;;) { + + /* Search for the key. */ + do { WT_ERR(__wt_getline(session, fs, buf)); if (buf->size == 0) WT_ERR(WT_NOTFOUND); - if (strcmp(key, buf->data) == 0) - match = true; + } while (strcmp(key, buf->data) != 0); - /* Key matched: read the subsequent line for the value. */ - WT_ERR(__wt_getline(session, fs, buf)); - if (buf->size == 0) - WT_ERR(__wt_illegal_value(session, WT_METADATA_TURTLE)); - if (match) - break; - } + /* Key matched: read the subsequent line for the value. */ + WT_ERR(__wt_getline(session, fs, buf)); + if (buf->size == 0) + WT_ERR(WT_NOTFOUND); /* Copy the value for the caller. */ WT_ERR(__wt_strdup(session, buf->data, valuep)); @@ -284,7 +280,12 @@ err: WT_TRET(__wt_fclose(session, &fs)); if (ret != 0) __wt_free(session, *valuep); - return (ret); + + /* + * A file error or a missing key/value pair in the turtle file means + * something has gone horribly wrong -- we're done. + */ + return (ret == 0 ? 0 : __wt_illegal_value(session, WT_METADATA_TURTLE)); } /* @@ -323,5 +324,9 @@ __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) err: WT_TRET(__wt_fclose(session, &fs)); WT_TRET(__wt_remove_if_exists(session, WT_METADATA_TURTLE_SET, false)); - return (ret); + /* + * An error updating the turtle file means something has gone horribly + * wrong -- we're done. + */ + return (ret == 0 ? 0 : __wt_illegal_value(session, WT_METADATA_TURTLE)); } diff --git a/src/third_party/wiredtiger/src/os_common/filename.c b/src/third_party/wiredtiger/src/os_common/filename.c index 8b6c1269829..d5695f63d91 100644 --- a/src/third_party/wiredtiger/src/os_common/filename.c +++ b/src/third_party/wiredtiger/src/os_common/filename.c @@ -29,6 +29,7 @@ int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) { + WT_DECL_RET; size_t len; char *buf; @@ -39,16 +40,17 @@ __wt_nfilename( * the exists API which is used by the test utilities. */ if (session == NULL || __wt_absolute_path(name)) - WT_RET(__wt_strndup(session, name, namelen, path)); - else { - len = strlen(S2C(session)->home) + 1 + namelen + 1; - WT_RET(__wt_calloc(session, 1, len, &buf)); - snprintf(buf, len, "%s%s%.*s", S2C(session)->home, - __wt_path_separator(), (int)namelen, name); - *path = buf; - } + return (__wt_strndup(session, name, namelen, path)); + len = strlen(S2C(session)->home) + 1 + namelen + 1; + WT_RET(__wt_calloc(session, 1, len, &buf)); + WT_ERR(__wt_snprintf(buf, len, "%s%s%.*s", + S2C(session)->home, __wt_path_separator(), (int)namelen, name)); + *path = buf; return (0); + +err: __wt_free(session, buf); + return (ret); } /* @@ -72,6 +74,7 @@ __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) */ int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_ITEM(tmp); WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/os_common/os_alloc.c b/src/third_party/wiredtiger/src/os_common/os_alloc.c index bfe5efafc8b..ef96ed09ea7 100644 --- a/src/third_party/wiredtiger/src/os_common/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_common/os_alloc.c @@ -30,6 +30,7 @@ */ int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { void *p; @@ -282,6 +283,7 @@ __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) */ void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { void *p; diff --git a/src/third_party/wiredtiger/src/os_common/os_errno.c b/src/third_party/wiredtiger/src/os_common/os_errno.c index a8e56b7f1aa..7ac89536e79 100644 --- a/src/third_party/wiredtiger/src/os_common/os_errno.c +++ b/src/third_party/wiredtiger/src/os_common/os_errno.c @@ -44,7 +44,7 @@ __wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) * Fallback to a generic message. */ if (session == NULL && - snprintf(errbuf, errlen, "error return: %d", error) > 0) + __wt_snprintf(errbuf, errlen, "error return: %d", error) == 0) return (errbuf); if (session != NULL && __wt_buf_fmt( session, &session->err, "error return: %d", error) == 0) diff --git a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c index 70a82007300..1670e97be45 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c +++ b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c @@ -515,8 +515,6 @@ __im_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) WT_FILE_SYSTEM_INMEM *im_fs; WT_SESSION_IMPL *session; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; im_fs = (WT_FILE_SYSTEM_INMEM *)file_system; diff --git a/src/third_party/wiredtiger/src/os_common/os_fstream.c b/src/third_party/wiredtiger/src/os_common/os_fstream.c index 5a368ea75e6..744da732d84 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fstream.c +++ b/src/third_party/wiredtiger/src/os_common/os_fstream.c @@ -144,7 +144,7 @@ __fstream_printf( p = (char *)((uint8_t *)buf->mem + buf->size); WT_ASSERT(session, buf->memsize >= buf->size); space = buf->memsize - buf->size; - len = (size_t)vsnprintf(p, space, fmt, ap_copy); + WT_RET(__wt_vsnprintf_len_set(p, space, &len, fmt, ap_copy)); va_end(ap_copy); if (len < space) { diff --git a/src/third_party/wiredtiger/src/os_common/os_getopt.c b/src/third_party/wiredtiger/src/os_common/os_getopt.c index 0306ad1d79d..960776c3999 100644 --- a/src/third_party/wiredtiger/src/os_common/os_getopt.c +++ b/src/third_party/wiredtiger/src/os_common/os_getopt.c @@ -79,6 +79,7 @@ char *__wt_optarg; /* argument associated with option */ int __wt_getopt( const char *progname, int nargc, char * const *nargv, const char *ostr) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { static const char *place = EMSG; /* option letter processing */ const char *oli; /* option letter list index */ diff --git a/src/third_party/wiredtiger/src/os_common/os_strtouq.c b/src/third_party/wiredtiger/src/os_common/os_strtouq.c index 0ae604fc761..cb4da0de058 100644 --- a/src/third_party/wiredtiger/src/os_common/os_strtouq.c +++ b/src/third_party/wiredtiger/src/os_common/os_strtouq.c @@ -14,6 +14,7 @@ */ uint64_t __wt_strtouq(const char *nptr, char **endptr, int base) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { #if defined(HAVE_STRTOUQ) return (strtouq(nptr, endptr, base)); diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 768a1324cd8..627278540d1 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -28,8 +28,6 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, int tret; char **entries; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; *dirlistp = NULL; diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index c6272b5da52..bc8cbf67025 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -575,7 +575,7 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, WT_FILE_HANDLE_POSIX *pfh; WT_SESSION_IMPL *session; mode_t mode; - int f; + int advise_flag, f; WT_UNUSED(file_system); @@ -676,17 +676,24 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, #if defined(HAVE_POSIX_FADVISE) /* - * Disable read-ahead on trees: it slows down random read workloads. + * If the user set an access pattern hint, call fadvise now. * Ignore fadvise when doing direct I/O, the kernel cache isn't * interesting. */ - if (!pfh->direct_io && file_type == WT_FS_OPEN_FILE_TYPE_DATA) { - WT_SYSCALL( - posix_fadvise(pfh->fd, 0, 0, POSIX_FADV_RANDOM), ret); + if (!pfh->direct_io && file_type == WT_FS_OPEN_FILE_TYPE_DATA && + LF_ISSET(WT_FS_OPEN_ACCESS_RAND | WT_FS_OPEN_ACCESS_SEQ)) { + advise_flag = 0; + if (LF_ISSET(WT_FS_OPEN_ACCESS_RAND)) + advise_flag = POSIX_FADV_RANDOM; + if (LF_ISSET(WT_FS_OPEN_ACCESS_SEQ)) + advise_flag = POSIX_FADV_SEQUENTIAL; + WT_SYSCALL(posix_fadvise(pfh->fd, 0, 0, advise_flag), ret); if (ret != 0) WT_ERR_MSG(session, ret, "%s: handle-open: posix_fadvise", name); } +#else + WT_UNUSED(advise_flag); #endif directory_open: @@ -746,8 +753,6 @@ __posix_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) { WT_SESSION_IMPL *session; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; __wt_free(session, file_system); diff --git a/src/third_party/wiredtiger/src/os_posix/os_getenv.c b/src/third_party/wiredtiger/src/os_posix/os_getenv.c index 7a086145cee..f779f90acee 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_getenv.c +++ b/src/third_party/wiredtiger/src/os_posix/os_getenv.c @@ -14,6 +14,7 @@ */ int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { const char *temp; diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index d4ee532a2af..fe010b62305 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -13,8 +13,7 @@ * Allocate and initialize a condition variable. */ int -__wt_cond_alloc(WT_SESSION_IMPL *session, - const char *name, bool is_signalled, WT_CONDVAR **condp) +__wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) { WT_CONDVAR *cond; WT_DECL_RET; @@ -27,7 +26,7 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, WT_ERR(pthread_cond_init(&cond->cond, NULL)); cond->name = name; - cond->waiters = is_signalled ? -1 : 0; + cond->waiters = 0; *condp = cond; return (0); @@ -42,8 +41,8 @@ err: __wt_free(session, cond); * out period expires, let the caller know. */ void -__wt_cond_wait_signal( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled) +__wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, + uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) { struct timespec ts; WT_DECL_RET; @@ -62,6 +61,23 @@ __wt_cond_wait_signal( WT_ERR(pthread_mutex_lock(&cond->mtx)); locked = true; + /* + * It's possible to race with threads waking us up. That's not a problem + * if there are multiple wakeups because the next wakeup will get us, or + * if we're only pausing for a short period. It's a problem if there's + * only a single wakeup, our waker is likely waiting for us to exit. + * After acquiring the mutex (so we're guaranteed to be awakened by any + * future wakeup call), optionally check if we're OK to keep running. + * This won't ensure our caller won't just loop and call us again, but + * at least it's not our fault. + * + * Assert we're not waiting longer than a second if not checking the + * run status. + */ + WT_ASSERT(session, run_func != NULL || usecs <= WT_MILLION); + if (run_func != NULL && !run_func(session)) + goto skipping; + if (usecs > 0) { __wt_epoch(session, &ts); ts.tv_sec += (time_t) @@ -81,7 +97,7 @@ __wt_cond_wait_signal( ret == ETIME || #endif ret == ETIMEDOUT) { - *signalled = false; +skipping: *signalled = false; ret = 0; } @@ -107,10 +123,13 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); /* - * Our callers are often setting flags to cause a thread to exit. Add - * a barrier to ensure the flags are seen by the threads. + * Our callers often set flags to cause a thread to exit. Add a barrier + * to ensure exit flags are seen by the sleeping threads, otherwise we + * can wake up a thread, it immediately goes back to sleep, and we'll + * hang. Use a full barrier (we may not write before waiting on thread + * join). */ - WT_WRITE_BARRIER(); + WT_FULL_BARRIER(); /* * Fast path if we are in (or can enter), a state where the next waiter @@ -134,7 +153,7 @@ err: * __wt_cond_destroy -- * Destroy a condition variable. */ -int +void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; @@ -142,11 +161,15 @@ __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) cond = *condp; if (cond == NULL) - return (0); + return; - ret = pthread_cond_destroy(&cond->cond); - WT_TRET(pthread_mutex_destroy(&cond->mtx)); - __wt_free(session, *condp); + if ((ret = pthread_cond_destroy(&cond->cond)) != 0) + WT_PANIC_MSG( + session, ret, "pthread_cond_destroy: %s", cond->name); - return (ret); + if ((ret = pthread_mutex_destroy(&cond->mtx)) != 0) + WT_PANIC_MSG( + session, ret, "pthread_mutex_destroy: %s", cond->name); + + __wt_free(session, *condp); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c b/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c index d6107115eb3..ac3958be22f 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c +++ b/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c @@ -14,6 +14,7 @@ */ void __wt_stream_set_line_buffer(FILE *fp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { /* * This function exists because MSVC doesn't support buffer sizes of 0 @@ -29,6 +30,7 @@ __wt_stream_set_line_buffer(FILE *fp) */ void __wt_stream_set_no_buffer(FILE *fp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { (void)setvbuf(fp, NULL, _IONBF, 0); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_sleep.c b/src/third_party/wiredtiger/src/os_posix/os_sleep.c index 8633b8d1ec0..2c60987ced7 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_sleep.c +++ b/src/third_party/wiredtiger/src/os_posix/os_sleep.c @@ -14,6 +14,7 @@ */ void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { struct timeval t; diff --git a/src/third_party/wiredtiger/src/os_posix/os_snprintf.c b/src/third_party/wiredtiger/src/os_posix/os_snprintf.c new file mode 100644 index 00000000000..390e2e0334a --- /dev/null +++ b/src/third_party/wiredtiger/src/os_posix/os_snprintf.c @@ -0,0 +1,27 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_vsnprintf_len_incr -- + * POSIX vsnprintf convenience function, incrementing the returned size. + */ +int +__wt_vsnprintf_len_incr( + char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + WT_DECL_RET; + + if ((ret = vsnprintf(buf, size, fmt, ap)) >= 0) { + *retsizep += (size_t)ret; + return (0); + } + return (__wt_errno()); +} diff --git a/src/third_party/wiredtiger/src/os_posix/os_thread.c b/src/third_party/wiredtiger/src/os_posix/os_thread.c index e57a308c9b0..18e4c347436 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_thread.c +++ b/src/third_party/wiredtiger/src/os_posix/os_thread.c @@ -18,6 +18,13 @@ __wt_thread_create(WT_SESSION_IMPL *session, { WT_DECL_RET; + /* + * Creating a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to start. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + /* Spawn a new thread of control. */ WT_SYSCALL_RETRY(pthread_create(tidret, NULL, func, arg), ret); if (ret == 0) @@ -34,6 +41,13 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { WT_DECL_RET; + /* + * Joining a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to halt. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + WT_SYSCALL(pthread_join(tid, NULL), ret); if (ret == 0) return (0); @@ -45,8 +59,9 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) * __wt_thread_id -- * Fill in a printable version of the process and thread IDs. */ -void +int __wt_thread_id(char *buf, size_t buflen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { pthread_t self; @@ -56,10 +71,10 @@ __wt_thread_id(char *buf, size_t buflen) */ self = pthread_self(); #ifdef __sun - (void)snprintf(buf, buflen, - "%" PRIuMAX ":%u", (uintmax_t)getpid(), self); + return (__wt_snprintf(buf, buflen, + "%" PRIuMAX ":%u", (uintmax_t)getpid(), self)); #else - (void)snprintf(buf, buflen, - "%" PRIuMAX ":%p", (uintmax_t)getpid(), (void *)self); + return (__wt_snprintf(buf, buflen, + "%" PRIuMAX ":%p", (uintmax_t)getpid(), (void *)self)); #endif } diff --git a/src/third_party/wiredtiger/src/os_posix/os_time.c b/src/third_party/wiredtiger/src/os_posix/os_time.c index 719e214696b..6f150ee8ffe 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_time.c +++ b/src/third_party/wiredtiger/src/os_posix/os_time.c @@ -14,6 +14,7 @@ */ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/os_posix/os_yield.c b/src/third_party/wiredtiger/src/os_posix/os_yield.c index 052a46940b7..f7c43aae746 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_yield.c +++ b/src/third_party/wiredtiger/src/os_posix/os_yield.c @@ -14,6 +14,15 @@ */ void __wt_yield(void) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { + /* + * Yielding the processor isn't documented as a memory barrier, and it's + * a reasonable expectation to have. There's no reason not to explicitly + * include a barrier since we're giving up the CPU, and ensures callers + * aren't ever surprised. + */ + WT_FULL_BARRIER(); + sched_yield(); } diff --git a/src/third_party/wiredtiger/src/os_win/os_dir.c b/src/third_party/wiredtiger/src/os_win/os_dir.c index f024d131387..47d4f95b793 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dir.c +++ b/src/third_party/wiredtiger/src/os_win/os_dir.c @@ -30,8 +30,6 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system, uint32_t count; char *dir_copy, **entries; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; *dirlistp = NULL; diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 7ab7178114b..5cf47ea5763 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -16,7 +16,6 @@ static int __win_fs_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, bool *existp) { - WT_DECL_RET; WT_DECL_ITEM(name_wide); WT_SESSION_IMPL *session; @@ -43,8 +42,8 @@ __win_fs_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, uint32_t flags) { DWORD windows_error; - WT_DECL_RET; WT_DECL_ITEM(name_wide); + WT_DECL_RET; WT_SESSION_IMPL *session; WT_UNUSED(file_system); @@ -88,22 +87,19 @@ __win_fs_rename(WT_FILE_SYSTEM *file_system, WT_ERR(__wt_to_utf16_string(session, to, &to_wide)); /* - * Check if file exists since Windows does not override the file if - * it exists. + * We want an atomic rename, but that's not guaranteed by MoveFileExW + * (or by any MSDN API). Don't set the MOVEFILE_COPY_ALLOWED flag to + * prevent the system from falling back to a copy and delete process. + * Do set the MOVEFILE_WRITE_THROUGH flag so the window is as small + * as possible, just in case. WiredTiger renames are done in a single + * directory and we expect that to be an atomic metadata update on any + * modern filesystem. */ - if (GetFileAttributesW(to_wide->data) != INVALID_FILE_ATTRIBUTES) - if (DeleteFileW(to_wide->data) == FALSE) { - windows_error = __wt_getlasterror(); - __wt_errx(session, - "%s: file-rename: DeleteFileW: %s", - to, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); - } - - if (MoveFileW(from_wide->data, to_wide->data) == FALSE) { + if (MoveFileExW(from_wide->data, to_wide->data, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) == FALSE) { windows_error = __wt_getlasterror(); __wt_errx(session, - "%s to %s: file-rename: MoveFileW: %s", + "%s to %s: file-rename: MoveFileExW: %s", from, to, __wt_formatmessage(session, windows_error)); WT_ERR(__wt_map_windows_error(windows_error)); } @@ -521,10 +517,16 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC)) f |= FILE_FLAG_WRITE_THROUGH; - /* Disable read-ahead on trees: it slows down random read workloads. */ - if (file_type == WT_FS_OPEN_FILE_TYPE_DATA) + /* If the user indicated a random workload, disable read-ahead. */ + if (file_type == WT_FS_OPEN_FILE_TYPE_DATA && + LF_ISSET(WT_FS_OPEN_ACCESS_RAND)) f |= FILE_FLAG_RANDOM_ACCESS; + /* If the user indicated a sequential workload, set that. */ + if (file_type == WT_FS_OPEN_FILE_TYPE_DATA && + LF_ISSET(WT_FS_OPEN_ACCESS_SEQ)) + f |= FILE_FLAG_SEQUENTIAL_SCAN; + win_fh->filehandle = CreateFileW(name_wide->data, desired_access, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, dwCreationDisposition, f, NULL); diff --git a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c index 79c62ccd7f2..c1b9f509d33 100644 --- a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c @@ -13,8 +13,7 @@ * Allocate and initialize a condition variable. */ int -__wt_cond_alloc(WT_SESSION_IMPL *session, - const char *name, bool is_signalled, WT_CONDVAR **condp) +__wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) { WT_CONDVAR *cond; @@ -26,7 +25,7 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, InitializeConditionVariable(&cond->cond); cond->name = name; - cond->waiters = is_signalled ? -1 : 0; + cond->waiters = 0; *condp = cond; return (0); @@ -38,8 +37,8 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, * out period expires, let the caller know. */ void -__wt_cond_wait_signal( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled) +__wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, + uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) { BOOL sleepret; DWORD milliseconds, windows_error; @@ -59,8 +58,26 @@ __wt_cond_wait_signal( EnterCriticalSection(&cond->mtx); locked = true; + /* + * It's possible to race with threads waking us up. That's not a problem + * if there are multiple wakeups because the next wakeup will get us, or + * if we're only pausing for a short period. It's a problem if there's + * only a single wakeup, our waker is likely waiting for us to exit. + * After acquiring the mutex (so we're guaranteed to be awakened by any + * future wakeup call), optionally check if we're OK to keep running. + * This won't ensure our caller won't just loop and call us again, but + * at least it's not our fault. + * + * Assert we're not waiting longer than a second if not checking the + * run status. + */ + WT_ASSERT(session, run_func != NULL || usecs <= WT_MILLION); + + if (run_func != NULL && !run_func(session)) + goto skipping; + if (usecs > 0) { - milliseconds64 = usecs / 1000; + milliseconds64 = usecs / WT_THOUSAND; /* * Check for 32-bit unsigned integer overflow @@ -90,7 +107,7 @@ __wt_cond_wait_signal( if (sleepret == 0) { windows_error = __wt_getlasterror(); if (windows_error == ERROR_TIMEOUT) { - *signalled = false; +skipping: *signalled = false; sleepret = 1; } } @@ -117,17 +134,17 @@ void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) { WT_DECL_RET; - bool locked; - - locked = false; __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); /* - * Our callers are often setting flags to cause a thread to exit. Add - * a barrier to ensure the flags are seen by the threads. + * Our callers often set flags to cause a thread to exit. Add a barrier + * to ensure exit flags are seen by the sleeping threads, otherwise we + * can wake up a thread, it immediately goes back to sleep, and we'll + * hang. Use a full barrier (we may not write before waiting on thread + * join). */ - WT_WRITE_BARRIER(); + WT_FULL_BARRIER(); /* * Fast path if we are in (or can enter), a state where the next waiter @@ -146,18 +163,16 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) * __wt_cond_destroy -- * Destroy a condition variable. */ -int +void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; cond = *condp; if (cond == NULL) - return (0); + return; /* Do nothing to delete Condition Variable */ DeleteCriticalSection(&cond->mtx); __wt_free(session, *condp); - - return (0); } diff --git a/src/third_party/wiredtiger/src/os_win/os_snprintf.c b/src/third_party/wiredtiger/src/os_win/os_snprintf.c index a6056ff9342..f3025b12a60 100644 --- a/src/third_party/wiredtiger/src/os_win/os_snprintf.c +++ b/src/third_party/wiredtiger/src/os_win/os_snprintf.c @@ -8,17 +8,47 @@ #include "wt_internal.h" -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...) +/* + * __wt_vsnprintf_len_incr -- + * POSIX vsnprintf convenience function, incrementing the returned size. + */ +int +__wt_vsnprintf_len_incr( + char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) { - va_list args; - WT_DECL_RET; + int len; + + /* + * WiredTiger calls with length 0 to get the needed buffer size. Call + * the count only version in this case, _vsnprintf_s will invoke the + * invalid parameter handler if count is less than or equal to zero. + */ + if (size == 0) { + *retsizep += (size_t)_vscprintf(fmt, ap); + return (0); + } + + /* + * Additionally, the invalid parameter handler is invoked if buffer or + * format is a NULL pointer. + */ + if (buf == NULL || fmt == NULL) + return (EINVAL); + + /* + * If the storage required to store the data and a terminating null + * exceeds size, the invalid parameter handler is invoked, unless + * count is _TRUNCATE, in which case as much of the string as will + * fit in the buffer is written and -1 returned. + */ + if ((len = _vsnprintf_s(buf, size, _TRUNCATE, fmt, ap)) >= 0) { + *retsizep += (size_t)len; + return (0); + } - va_start(args, _Format); - ret = _wt_vsnprintf(_DstBuf, _MaxCount, _Format, args); - va_end(args); + /* Return the buffer size required. */ + if (len == -1) + *retsizep += (size_t)_vscprintf(fmt, ap); - return (ret); + return (0); } diff --git a/src/third_party/wiredtiger/src/os_win/os_thread.c b/src/third_party/wiredtiger/src/os_win/os_thread.c index a34dff776b6..4c8f212bb4f 100644 --- a/src/third_party/wiredtiger/src/os_win/os_thread.c +++ b/src/third_party/wiredtiger/src/os_win/os_thread.c @@ -16,6 +16,13 @@ int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) { + /* + * Creating a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to start. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + /* Spawn a new thread of control. */ *tidret = (HANDLE)_beginthreadex(NULL, 0, func, arg, 0, NULL); if (*tidret != 0) @@ -33,6 +40,13 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { DWORD windows_error; + /* + * Joining a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to halt. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + if ((windows_error = WaitForSingleObject(tid, INFINITE)) != WAIT_OBJECT_0) { if (windows_error == WAIT_FAILED) @@ -58,10 +72,10 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) * __wt_thread_id -- * Fill in a printable version of the process and thread IDs. */ -void +int __wt_thread_id(char *buf, size_t buflen) { - (void)snprintf(buf, buflen, + return (__wt_snprintf(buf, buflen, "%" PRIu64 ":%" PRIu64, - (uint64_t)GetCurrentProcessId(), (uint64_t)GetCurrentThreadId); + (uint64_t)GetCurrentProcessId(), (uint64_t)GetCurrentThreadId)); } diff --git a/src/third_party/wiredtiger/src/os_win/os_utf8.c b/src/third_party/wiredtiger/src/os_win/os_utf8.c index f7d11c24f03..ccd8321aecf 100644 --- a/src/third_party/wiredtiger/src/os_win/os_utf8.c +++ b/src/third_party/wiredtiger/src/os_win/os_utf8.c @@ -18,7 +18,6 @@ __wt_to_utf16_string( { DWORD windows_error; int bufferSize; - WT_DECL_RET; bufferSize = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); windows_error = __wt_getlasterror(); @@ -55,7 +54,6 @@ __wt_to_utf8_string( { DWORD windows_error; int bufferSize; - WT_DECL_RET; bufferSize = WideCharToMultiByte( CP_UTF8, 0, wide, -1, NULL, 0, NULL, NULL); diff --git a/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c b/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c deleted file mode 100644 index 63f96e79d5b..00000000000 --- a/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c +++ /dev/null @@ -1,41 +0,0 @@ -/*- - * Copyright (c) 2014-2016 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -_Check_return_opt_ int __cdecl _wt_vsnprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, - va_list _ArgList) -{ - int len; - - /* - * WiredTiger will call with length 0 to get the needed buffer size - * We call the count only version in this case since vsnprintf_s assumes - * length is greater than zero or else it triggers the invalid_parameter - * handler. - */ - if (_MaxCount == 0) { - return _vscprintf(_Format, _ArgList); - } - - len = (size_t)_vsnprintf_s( - _DstBuf, _MaxCount, _TRUNCATE, _Format, _ArgList); - - /* - * The MSVC implementation returns -1 on truncation instead of what - * it would have written. We could let callers iteratively grow the - * buffer, or just ask us how big a buffer they would like. - */ - if (len == -1) - len = _vscprintf(_Format, _ArgList) + 1; - - return (len); -} diff --git a/src/third_party/wiredtiger/src/os_win/os_yield.c b/src/third_party/wiredtiger/src/os_win/os_yield.c index aab1559e072..038f2efe162 100644 --- a/src/third_party/wiredtiger/src/os_win/os_yield.c +++ b/src/third_party/wiredtiger/src/os_win/os_yield.c @@ -15,5 +15,13 @@ void __wt_yield(void) { + /* + * Yielding the processor isn't documented as a memory barrier, and it's + * a reasonable expectation to have. There's no reason not to explicitly + * include a barrier since we're giving up the CPU, and ensures callers + * aren't ever surprised. + */ + WT_FULL_BARRIER(); + SwitchToThread(); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_track.c b/src/third_party/wiredtiger/src/reconcile/rec_track.c index 10a64fdf116..5bf425b1b21 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_track.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_track.c @@ -35,7 +35,6 @@ __ovfl_discard_verbose( { WT_CELL_UNPACK *unpack, _unpack; WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 512, &tmp)); @@ -50,7 +49,7 @@ __ovfl_discard_verbose( __wt_addr_string(session, unpack->data, unpack->size, tmp)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -83,7 +82,6 @@ static int __ovfl_discard_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) { WT_CELL **cellp; - WT_DECL_RET; WT_OVFL_TRACK *track; uint32_t i; @@ -101,7 +99,7 @@ __ovfl_discard_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_free(session, track->discard); track->discard_entries = track->discard_allocated = 0; - return (ret); + return (0); } /* @@ -170,7 +168,6 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, WT_PAGE *page, WT_OVFL_REUSE *reuse, const char *tag) { WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 64, &tmp)); @@ -188,7 +185,7 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, WT_MIN(reuse->value_size, 40), (char *)WT_OVFL_REUSE_VALUE(reuse)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -568,7 +565,6 @@ __ovfl_txnc_verbose(WT_SESSION_IMPL *session, WT_PAGE *page, WT_OVFL_TXNC *txnc, const char *tag) { WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 64, &tmp)); @@ -583,7 +579,7 @@ __ovfl_txnc_verbose(WT_SESSION_IMPL *session, WT_MIN(txnc->value_size, 40), (char *)WT_OVFL_TXNC_VALUE(txnc)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -879,9 +875,9 @@ __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_RET(__ovfl_reuse_wrapup(session, page)); if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, S2BT(session)->ovfl_lock); + __wt_writelock(session, &S2BT(session)->ovfl_lock); ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, S2BT(session)->ovfl_lock); + __wt_writeunlock(session, &S2BT(session)->ovfl_lock); } return (ret); } @@ -907,9 +903,9 @@ __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_RET(__ovfl_reuse_wrapup_err(session, page)); if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, S2BT(session)->ovfl_lock); + __wt_writelock(session, &S2BT(session)->ovfl_lock); ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, S2BT(session)->ovfl_lock); + __wt_writeunlock(session, &S2BT(session)->ovfl_lock); } return (ret); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index fe288beed15..e59d9796352 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -26,6 +26,11 @@ typedef struct { uint32_t flags; /* Caller's configuration */ WT_ITEM disk_image; /* Temporary disk-image buffer */ + /* + * Temporary buffer used to write out a disk image when managing two + * chunks worth of data in memory + */ + WT_ITEM *interim_buf; /* * Track start/stop write generation to decide if all changes to the @@ -43,6 +48,10 @@ typedef struct { /* Track the page's maximum transaction ID. */ uint64_t max_txn; + /* Track if all updates were skipped. */ + uint64_t update_cnt; + uint64_t update_skip_cnt; + /* * When we can't mark the page clean (for example, checkpoint found some * uncommitted updates), there's a leave-dirty flag. @@ -123,6 +132,7 @@ typedef struct { * repeatedly split a packed page. */ uint32_t split_size; /* Split page size */ + uint32_t min_split_size; /* Minimum split page size */ /* * The problem with splits is we've done a lot of work by the time we @@ -147,16 +157,6 @@ typedef struct { */ size_t offset; /* Split's first byte */ - /* - * The recno and entries fields are the starting record number - * of the split chunk (for column-store splits), and the number - * of entries in the split chunk. These fields are used both - * to write the split chunk, and to create a new internal page - * to reference the split pages. - */ - uint64_t recno; /* Split's starting record */ - uint32_t entries; /* Split's entries */ - WT_ADDR addr; /* Split's written location */ uint32_t size; /* Split's size */ uint32_t checksum; /* Split's checksum */ @@ -178,39 +178,42 @@ typedef struct { size_t supd_allocated; /* + * While reconciling pages, at any given time, we maintain two + * split chunks in the memory to be written out as pages. As we + * get to the last two chunks, if the last one turns out to be + * smaller than the minimum split size, we go back into the + * penultimate chunk and split at this minimum split size + * boundary. This moves some data from the penultimate chunk to + * the last chunk, hence increasing the size of the last page + * written without decreasing the penultimate page size beyond + * the minimum split size. For this reason, we maintain both a + * maximum split percentage boundary and a minimum split + * percentage boundary. + * + * The recno and entries fields are the starting record number + * of the split chunk (for column-store splits), and the number + * of entries in the split chunk. These fields are used both to + * write the split chunk, and to create a new internal page to + * reference the split pages. + * * The key for a row-store page; no column-store key is needed * because the page's recno, stored in the recno field, is the * column-store key. */ - WT_ITEM key; /* Promoted row-store key */ + uint32_t max_bnd_entries; + uint64_t max_bnd_recno; + WT_ITEM max_bnd_key; + + size_t min_bnd_offset; + uint32_t min_bnd_entries; + uint64_t min_bnd_recno; + WT_ITEM min_bnd_key; } *bnd; /* Saved boundaries */ uint32_t bnd_next; /* Next boundary slot */ uint32_t bnd_next_max; /* Maximum boundary slots used */ size_t bnd_entries; /* Total boundary slots */ size_t bnd_allocated; /* Bytes allocated */ - /* - * We track the total number of page entries copied into split chunks - * so we can easily figure out how many entries in the current split - * chunk. - */ - uint32_t total_entries; /* Total entries in splits */ - - /* - * And there's state information as to where in this process we are: - * (1) tracking split boundaries because we can still fit more split - * chunks into the maximum page size, (2) tracking the maximum page - * size boundary because we can't fit any more split chunks into the - * maximum page size, (3) not performing boundary checks because it's - * either not useful with the current page size configuration, or - * because we've already been forced to split. - */ - enum { SPLIT_BOUNDARY=0, /* Next: a split page boundary */ - SPLIT_MAX=1, /* Next: the maximum page boundary */ - SPLIT_TRACKING_OFF=2, /* No boundary checks */ - SPLIT_TRACKING_RAW=3 } /* Underlying compression decides */ - bnd_state; - /* * We track current information about the current record number, the * number of entries copied into the temporary buffer, where we are @@ -222,6 +225,8 @@ typedef struct { uint32_t entries; /* Current number of entries */ uint8_t *first_free; /* Current first free byte */ size_t space_avail; /* Remaining space in this chunk */ + /* Remaining space in this chunk to put a minimum size boundary */ + size_t min_space_avail; /* * Saved update list, supporting the WT_EVICT_UPDATE_RESTORE and @@ -243,15 +248,14 @@ typedef struct { /* * WT_DICTIONARY -- - * We optionally build a dictionary of row-store values for leaf - * pages. Where two value cells are identical, only write the value - * once, the second and subsequent copies point to the original cell. - * The dictionary is fixed size, but organized in a skip-list to make - * searches faster. + * We optionally build a dictionary of values for leaf pages. Where + * two value cells are identical, only write the value once, the second + * and subsequent copies point to the original cell. The dictionary is + * fixed size, but organized in a skip-list to make searches faster. */ struct __rec_dictionary { uint64_t hash; /* Hash value */ - void *cell; /* Matching cell */ + uint32_t offset; /* Matching cell */ u_int depth; /* Skiplist */ WT_DICTIONARY *next[0]; @@ -289,6 +293,13 @@ typedef struct { uint32_t tested_ref_state; /* Debugging information */ } WT_RECONCILE; +#define WT_CROSSING_MIN_BND(r, next_len) \ + ((r)->bnd[(r)->bnd_next].min_bnd_offset == 0 && \ + (next_len) > (r)->min_space_avail) +#define WT_CROSSING_SPLIT_BND(r, next_len) ((next_len) > (r)->space_avail) +#define WT_CHECK_CROSSING_BND(r, next_len) \ + (WT_CROSSING_MIN_BND(r, next_len) || WT_CROSSING_SPLIT_BND(r, next_len)) + static void __rec_bnd_cleanup(WT_SESSION_IMPL *, WT_RECONCILE *, bool); static void __rec_cell_build_addr(WT_SESSION_IMPL *, WT_RECONCILE *, const void *, size_t, u_int, uint64_t); @@ -310,6 +321,7 @@ static int __rec_col_var(WT_SESSION_IMPL *, static int __rec_col_var_helper(WT_SESSION_IMPL *, WT_RECONCILE *, WT_SALVAGE_COOKIE *, WT_ITEM *, bool, uint8_t, uint64_t); static int __rec_destroy_session(WT_SESSION_IMPL *); +static uint32_t __rec_min_split_page_size(WT_BTREE *, uint32_t); static int __rec_root_write(WT_SESSION_IMPL *, WT_PAGE *, uint32_t); static int __rec_row_int(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_row_leaf(WT_SESSION_IMPL *, @@ -319,7 +331,6 @@ static int __rec_row_leaf_insert( static int __rec_row_merge(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_split_col(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_split_discard(WT_SESSION_IMPL *, WT_PAGE *); -static int __rec_split_fixup(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_split_row(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_split_row_promote( WT_SESSION_IMPL *, WT_RECONCILE *, WT_ITEM *, uint8_t); @@ -327,9 +338,10 @@ static int __rec_split_write(WT_SESSION_IMPL *, WT_RECONCILE *, WT_BOUNDARY *, WT_ITEM *, bool); static int __rec_update_las( WT_SESSION_IMPL *, WT_RECONCILE *, uint32_t, WT_BOUNDARY *); +static int __rec_write_check_complete(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_write_init(WT_SESSION_IMPL *, WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *); -static int __rec_write_status(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); +static void __rec_write_page_status(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_write_wrapup(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_write_wrapup_err( WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); @@ -345,8 +357,8 @@ static void __rec_dictionary_reset(WT_RECONCILE *); * Reconcile an in-memory page into its on-disk format, and write it. */ int -__wt_reconcile(WT_SESSION_IMPL *session, - WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags) +__wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, + WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) { WT_DECL_RET; WT_PAGE *page; @@ -356,6 +368,8 @@ __wt_reconcile(WT_SESSION_IMPL *session, page = ref->page; mod = page->modify; + if (lookaside_retryp != NULL) + *lookaside_retryp = false; __wt_verbose(session, WT_VERB_RECONCILE, "%s", __wt_page_type_string(page->type)); @@ -372,7 +386,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, * In-memory splits: reconciliation of an internal page cannot handle * a child page splitting during the reconciliation. */ - __wt_writelock(session, &page->page_lock); + WT_PAGE_LOCK(session, page); oldest_id = __wt_txn_oldest_id(session); if (LF_ISSET(WT_EVICTING)) @@ -391,7 +405,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, /* Initialize the reconciliation structure for each new run. */ if ((ret = __rec_write_init( session, ref, flags, salvage, &session->reconcile)) != 0) { - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); return (ret); } r = session->reconcile; @@ -421,18 +435,26 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_ILLEGAL_VALUE_SET(session); } - /* Get the final status for the reconciliation. */ + /* Checks for a successful reconciliation. */ if (ret == 0) - ret = __rec_write_status(session, r, page); + ret = __rec_write_check_complete(session, r); /* Wrap up the page reconciliation. */ - if (ret == 0) - ret = __rec_write_wrapup(session, r, page); + if (ret == 0 && (ret = __rec_write_wrapup(session, r, page)) == 0) + __rec_write_page_status(session, r); else WT_TRET(__rec_write_wrapup_err(session, r, page)); /* Release the reconciliation lock. */ - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); + + /* + * If our caller can configure lookaside table reconciliation, flag if + * that's worth trying. The lookaside table doesn't help if we skipped + * updates, it can only help with older readers preventing eviction. + */ + if (lookaside_retryp != NULL && r->update_cnt == r->update_skip_cnt) + *lookaside_retryp = true; /* Update statistics. */ WT_STAT_CONN_INCR(session, rec_pages); @@ -535,17 +557,14 @@ __rec_las_checkpoint_test(WT_SESSION_IMPL *session, WT_RECONCILE *r) } /* - * __rec_write_status -- - * Return the final status for reconciliation. + * __rec_write_check_complete -- + * Check that reconciliation should complete */ static int -__rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) +__rec_write_check_complete(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BTREE *btree; - WT_PAGE_MODIFY *mod; - - btree = S2BT(session); - mod = page->modify; + WT_BOUNDARY *bnd; + size_t i; /* * If we have used the lookaside table, check for a lookaside table and @@ -554,6 +573,37 @@ __rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) if (r->cache_write_lookaside && __rec_las_checkpoint_test(session, r)) return (EBUSY); + /* + * If we are doing update/restore based eviction, confirm part of the + * page is being discarded, or at least 10% of the updates won't have + * to be re-instantiated. Otherwise, it isn't progress, don't bother. + */ + if (F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) { + for (bnd = r->bnd, i = 0; i < r->bnd_entries; ++bnd, ++i) + if (bnd->supd == NULL) + break; + if (i == r->bnd_entries && + r->update_cnt / 10 >= r->update_skip_cnt) + return (EBUSY); + } + return (0); +} + +/* + * __rec_write_page_status -- + * Set the page status after reconciliation. + */ +static void +__rec_write_page_status(WT_SESSION_IMPL *session, WT_RECONCILE *r) +{ + WT_BTREE *btree; + WT_PAGE *page; + WT_PAGE_MODIFY *mod; + + btree = S2BT(session); + page = r->page; + mod = page->modify; + /* * Set the page's status based on whether or not we cleaned the page. */ @@ -612,8 +662,6 @@ __rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) else WT_ASSERT(session, !F_ISSET(r, WT_EVICTING)); } - - return (0); } /* @@ -675,6 +723,7 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) * pages in memory; it's not needed here, asserted for safety. */ WT_ASSERT(session, mod->mod_multi[i].supd == NULL); + WT_ASSERT(session, mod->mod_multi[i].disk_image == NULL); WT_ERR(__wt_multi_to_ref(session, next, &mod->mod_multi[i], &pindex->index[i], NULL, false)); @@ -700,7 +749,7 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) * Fake up a reference structure, and write the next root page. */ __wt_root_ref_init(&fake_ref, next, page->type == WT_PAGE_COL_INT); - return (__wt_reconcile(session, &fake_ref, NULL, flags)); + return (__wt_reconcile(session, &fake_ref, NULL, flags, NULL)); err: __wt_page_out(session, &next); return (ret); @@ -841,6 +890,9 @@ __rec_write_init(WT_SESSION_IMPL *session, /* Track the page's maximum transaction ID. */ r->max_txn = WT_TXN_NONE; + /* Track if all updates were skipped. */ + r->update_cnt = r->update_skip_cnt = 0; + /* Track if the page can be marked clean. */ r->leave_dirty = false; @@ -923,6 +975,7 @@ __rec_destroy(WT_SESSION_IMPL *session, void *reconcilep) *(WT_RECONCILE **)reconcilep = NULL; __wt_buf_free(session, &r->disk_image); + __wt_scr_free(session, &r->interim_buf); __wt_free(session, r->raw_entries); __wt_free(session, r->raw_offsets); @@ -987,7 +1040,8 @@ __rec_bnd_cleanup(WT_SESSION_IMPL *session, WT_RECONCILE *r, bool destroy) __wt_free(session, bnd->addr.addr); __wt_free(session, bnd->disk_image); __wt_free(session, bnd->supd); - __wt_buf_free(session, &bnd->key); + __wt_buf_free(session, &bnd->max_bnd_key); + __wt_buf_free(session, &bnd->min_bnd_key); } __wt_free(session, r->bnd); r->bnd_next = 0; @@ -1082,6 +1136,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, } else upd_list = ins->upd; + ++r->update_cnt; for (skipped = false, max_txn = WT_TXN_NONE, min_txn = UINT64_MAX, upd = upd_list; upd != NULL; upd = upd->next) { @@ -1172,6 +1227,12 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, txnid != S2C(session)->txn_global.checkpoint_txnid || WT_SESSION_IS_CHECKPOINT(session)); #endif + + /* + * Track how many update chains we saw vs. how many update + * chains had an entry we skipped. + */ + ++r->update_skip_cnt; return (0); } @@ -1343,7 +1404,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, */ #define WT_CHILD_RELEASE(session, hazard, ref) do { \ if (hazard) { \ - hazard = false; \ + (hazard) = false; \ WT_TRET( \ __wt_page_release(session, ref, WT_READ_NO_EVICT)); \ } \ @@ -1665,6 +1726,17 @@ __rec_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, uint32_t v, size_t size) r->entries += v; r->space_avail -= size; r->first_free += size; + + /* + * If offset for the minimum split size boundary is not set, we have not + * yet reached the minimum boundary, reduce the space available for it. + */ + if (r->bnd[r->bnd_next].min_bnd_offset == 0) { + if (r->min_space_avail >= size) + r->min_space_avail -= size; + else + r->min_space_avail = 0; + } } /* @@ -1685,7 +1757,7 @@ __rec_copy_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_KV *kv) * WT_CELLs are typically small, 1 or 2 bytes -- don't call memcpy, do * the copy in-line. */ - for (p = (uint8_t *)r->first_free, + for (p = r->first_free, t = (uint8_t *)&kv->cell, len = kv->cell_len; len > 0; --len) *p++ = *t++; @@ -1729,16 +1801,22 @@ __rec_dict_replace( return (0); /* - * If the dictionary cell reference is not set, we're creating a new - * entry in the dictionary, update its location. + * If the dictionary offset isn't set, we're creating a new entry in the + * dictionary, set its location. * - * If the dictionary cell reference is set, we have a matching value. - * Create a copy cell instead. + * If the dictionary offset is set, we have a matching value. Create a + * copy cell instead. */ - if (dp->cell == NULL) - dp->cell = r->first_free; + if (dp->offset == 0) + dp->offset = WT_PTRDIFF32(r->first_free, r->disk_image.mem); else { - offset = WT_PTRDIFF(r->first_free, dp->cell); + /* + * The offset is the byte offset from this cell to the previous, + * matching cell, NOT the byte offset from the beginning of the + * page. + */ + offset = (uint64_t)WT_PTRDIFF(r->first_free, + (uint8_t *)r->disk_image.mem + dp->offset); val->len = val->cell_len = __wt_cell_pack_copy(&val->cell, rle, offset); val->buf.data = NULL; @@ -1875,8 +1953,8 @@ static void __rec_split_bnd_init(WT_SESSION_IMPL *session, WT_BOUNDARY *bnd) { bnd->offset = 0; - bnd->recno = WT_RECNO_OOB; - bnd->entries = 0; + bnd->max_bnd_recno = WT_RECNO_OOB; + bnd->max_bnd_entries = 0; __wt_free(session, bnd->addr.addr); WT_CLEAR(bnd->addr); @@ -1891,6 +1969,10 @@ __rec_split_bnd_init(WT_SESSION_IMPL *session, WT_BOUNDARY *bnd) bnd->already_compressed = false; + bnd->min_bnd_offset = 0; + bnd->min_bnd_entries = 0; + bnd->min_bnd_recno = WT_RECNO_OOB; + /* * Don't touch the key, we re-use that memory in each new * reconciliation. @@ -1922,39 +2004,63 @@ __rec_split_bnd_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r) } /* - * __wt_split_page_size -- - * Split page size calculation: we don't want to repeatedly split every - * time a new entry is added, so we split to a smaller-than-maximum page size. + * __rec_split_page_size_from_pct -- + * Given a split percentage, calculate split page size in bytes. */ -uint32_t -__wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) -{ +static uint32_t +__rec_split_page_size_from_pct( + int split_pct, uint32_t maxpagesize, uint32_t allocsize) { uintmax_t a; uint32_t split_size; /* * Ideally, the split page size is some percentage of the maximum page - * size rounded to an allocation unit (round to an allocation unit so - * we don't waste space when we write). + * size rounded to an allocation unit (round to an allocation unit so we + * don't waste space when we write). */ a = maxpagesize; /* Don't overflow. */ split_size = (uint32_t)WT_ALIGN_NEAREST( - (a * (u_int)btree->split_pct) / 100, btree->allocsize); + (a * (u_int)split_pct) / 100, allocsize); /* - * Respect the configured split percentage if the calculated split - * size is either zero or a full page. The user has either configured - * an allocation size that matches the page size, or a split - * percentage that is close to zero or one hundred. Rounding is going - * to provide a worse outcome than having a split point that doesn't - * fall on an allocation size boundary in those cases. + * Respect the configured split percentage if the calculated split size + * is either zero or a full page. The user has either configured an + * allocation size that matches the page size, or a split percentage + * that is close to zero or one hundred. Rounding is going to provide a + * worse outcome than having a split point that doesn't fall on an + * allocation size boundary in those cases. */ if (split_size == 0 || split_size == maxpagesize) - split_size = (uint32_t)((a * (u_int)btree->split_pct) / 100); + split_size = (uint32_t)((a * (u_int)split_pct) / 100); return (split_size); } +/* + * __wt_split_page_size -- + * Split page size calculation: we don't want to repeatedly split every + * time a new entry is added, so we split to a smaller-than-maximum page size. + */ +uint32_t +__wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) +{ + return (__rec_split_page_size_from_pct( + btree->split_pct, maxpagesize, btree->allocsize)); +} + +/* + * __rec_min_split_page_size -- + * Minimum split size boundary calculation: To track a boundary at the + * minimum split size that we could have split at instead of splitting at + * the split page size. + */ +static uint32_t +__rec_min_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) +{ + return (__rec_split_page_size_from_pct( + WT_BTREE_MIN_SPLIT_PCT, maxpagesize, btree->allocsize)); +} + /* * __rec_split_init -- * Initialization for the reconciliation split functions. @@ -1966,7 +2072,7 @@ __rec_split_init(WT_SESSION_IMPL *session, WT_BM *bm; WT_BTREE *btree; WT_PAGE_HEADER *dsk; - size_t corrected_page_size; + size_t corrected_page_size, disk_img_buf_size; btree = S2BT(session); bm = btree->bm; @@ -2001,33 +2107,6 @@ __rec_split_init(WT_SESSION_IMPL *session, r->max_raw_page_size = r->page_size = (uint32_t)WT_MIN(r->page_size * 10, WT_MAX(r->page_size, btree->maxmempage / 2)); - - /* - * Ensure the disk image buffer is large enough for the max object, as - * corrected by the underlying block manager. - */ - corrected_page_size = r->page_size; - WT_RET(bm->write_size(bm, session, &corrected_page_size)); - WT_RET(__wt_buf_init(session, &r->disk_image, corrected_page_size)); - - /* - * Clear the disk page header to ensure all of it is initialized, even - * the unused fields. - * - * In the case of fixed-length column-store, clear the entire buffer: - * fixed-length column-store sets bits in bytes, where the bytes are - * assumed to initially be 0. - */ - memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? - corrected_page_size : WT_PAGE_HEADER_SIZE); - - /* - * Set the page type (the type doesn't change, and setting it later - * would require additional code in a few different places). - */ - dsk = r->disk_image.mem; - dsk->type = page->type; - /* * If we have to split, we want to choose a smaller page size for the * split pages, because otherwise we could end up splitting one large @@ -2047,22 +2126,28 @@ __rec_split_init(WT_SESSION_IMPL *session, * creating overflow items and compacted data, for example, as those * items have already been written to disk). So, the loop calls the * helper functions when approaching a split boundary, and we save the - * information at that point. That allows us to go back and split the - * page at the boundary points if we eventually overflow the maximum - * page size. + * information at that point. We also save the boundary information at + * the minimum split size. We maintain two chunks (each boundary + * represents a chunk that gets written as a page) in the memory, + * writing out the older one to the disk as a page when we need to make + * space for a new chunk. On reaching the last chunk, if it turns out to + * be smaller than the minimum split size, we go back into the + * penultimate chunk and split at this minimum split size boundary. This + * moves some data from the penultimate chunk to the last chunk, hence + * increasing the size of the last page written without decreasing the + * penultimate page size beyond the minimum split size. * * Finally, all this doesn't matter for fixed-size column-store pages, * raw compression, and salvage. Fixed-size column store pages can * split under (very) rare circumstances, but they're allocated at a * fixed page size, never anything smaller. In raw compression, the - * underlying compression routine decides when we split, so it's not - * our problem. In salvage, as noted above, we can't split at all. + * underlying compression routine decides when we split, so it's not our + * problem. In salvage, as noted above, we can't split at all. */ if (r->raw_compression || r->salvage != NULL) { r->split_size = 0; r->space_avail = r->page_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - } - else if (page->type == WT_PAGE_COL_FIX) { + } else if (page->type == WT_PAGE_COL_FIX) { r->split_size = r->page_size; r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); @@ -2070,32 +2155,55 @@ __rec_split_init(WT_SESSION_IMPL *session, r->split_size = __wt_split_page_size(btree, r->page_size); r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + r->min_split_size = + __rec_min_split_page_size(btree, r->page_size); + r->min_space_avail = + r->min_split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); } + + /* + * Ensure the disk image buffer is large enough for the max object, as + * corrected by the underlying block manager. + * + * The buffer that we build disk image in, needs to hold two chunks + * worth of data. Since we want to support split_size more than the page + * size (to allow for adjustments based on the compression), this buffer + * should be greater of twice of split_size and page_size. + */ + corrected_page_size = r->page_size; + disk_img_buf_size = 2 * WT_MAX(corrected_page_size, r->split_size); + WT_RET(bm->write_size(bm, session, &corrected_page_size)); + WT_RET(__wt_buf_init(session, &r->disk_image, disk_img_buf_size)); + + /* + * Clear the disk page header to ensure all of it is initialized, even + * the unused fields. + * + * In the case of fixed-length column-store, clear the entire buffer: + * fixed-length column-store sets bits in bytes, where the bytes are + * assumed to initially be 0. + */ + memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? + disk_img_buf_size : WT_PAGE_HEADER_SIZE); + + /* + * Set the page type (the type doesn't change, and setting it later + * would require additional code in a few different places). + */ + dsk = r->disk_image.mem; + dsk->type = page->type; + r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); /* Initialize the first boundary. */ r->bnd_next = 0; WT_RET(__rec_split_bnd_grow(session, r)); __rec_split_bnd_init(session, &r->bnd[0]); - r->bnd[0].recno = recno; + r->bnd[0].max_bnd_recno = recno; r->bnd[0].offset = WT_PAGE_HEADER_BYTE_SIZE(btree); - /* - * If the maximum page size is the same as the split page size, either - * because of the object type or application configuration, there isn't - * any need to maintain split boundaries within a larger page. - * - * No configuration for salvage here, because salvage can't split. - */ - if (r->raw_compression) - r->bnd_state = SPLIT_TRACKING_RAW; - else if (max == r->split_size) - r->bnd_state = SPLIT_TRACKING_OFF; - else - r->bnd_state = SPLIT_BOUNDARY; - - /* Initialize the entry counters. */ - r->entries = r->total_entries = 0; + /* Initialize the entry counter. */ + r->entries = 0; /* Initialize the starting record number. */ r->recno = recno; @@ -2298,19 +2406,112 @@ __rec_split_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t add_len) { WT_BM *bm; WT_BTREE *btree; - size_t corrected_page_size, len; + size_t corrected_page_size, inuse, len; btree = S2BT(session); bm = btree->bm; len = WT_PTRDIFF(r->first_free, r->disk_image.mem); - corrected_page_size = len + add_len; + inuse = (len - r->bnd[r->bnd_next].offset) + + WT_PAGE_HEADER_BYTE_SIZE(btree); + corrected_page_size = inuse + add_len; + WT_RET(bm->write_size(bm, session, &corrected_page_size)); - WT_RET(__wt_buf_grow(session, &r->disk_image, corrected_page_size)); + /* Need to account for buffer carrying two chunks worth of data */ + WT_RET(__wt_buf_grow(session, &r->disk_image, 2 * corrected_page_size)); + r->first_free = (uint8_t *)r->disk_image.mem + len; - WT_ASSERT(session, corrected_page_size >= len); - r->space_avail = corrected_page_size - len; + WT_ASSERT(session, corrected_page_size >= inuse); + r->space_avail = corrected_page_size - inuse; WT_ASSERT(session, r->space_avail >= add_len); + + return (0); +} + +/* + * __rec_split_write_prev_and_shift_cur -- + * Write the previous split chunk to the disk as a page. Shift the contents + * of the current chunk to the start of the buffer, making space for a new + * chunk to be written. + * If the caller asks for a chunk resizing, the boundary between the two + * chunks is readjusted to the minimum split size boundary details stored + * in the previous chunk, letting the current chunk grow at the cost of the + * previous chunk. + */ +static int +__rec_split_write_prev_and_shift_cur( + WT_SESSION_IMPL *session, WT_RECONCILE *r, bool resize_chunks) +{ + WT_BM *bm; + WT_BOUNDARY *bnd_cur, *bnd_prev; + WT_BTREE *btree; + WT_PAGE_HEADER *dsk, *dsk_tmp; + size_t cur_len, len; + uint8_t *dsk_start; + + WT_ASSERT(session, r->bnd_next != 0); + + btree = S2BT(session); + bm = btree->bm; + bnd_cur = &r->bnd[r->bnd_next]; + bnd_prev = bnd_cur - 1; + dsk = r->disk_image.mem; + cur_len = WT_PTRDIFF(r->first_free, dsk) - bnd_cur->offset; + + /* + * Resize chunks if the current is smaller than the minimum, and there + * are details on the minimum split size boundary available in the + * previous boundary details. + * + * There is a possibility that we do not have a minimum boundary set, in + * such a case we skip chunk resizing. Such a condition is possible for + * instance when we are building the image in the buffer and the first + * K/V pair is large enough that it surpasses both the minimum split + * size and the split size the application has set. In such a case we + * split the chunk without saving any minimum boundary. + */ + if (resize_chunks && + cur_len < r->min_split_size && bnd_prev->min_bnd_offset != 0) { + bnd_cur->offset = bnd_prev->min_bnd_offset; + bnd_cur->max_bnd_entries += + bnd_prev->max_bnd_entries - bnd_prev->min_bnd_entries; + bnd_prev->max_bnd_entries = bnd_prev->min_bnd_entries; + bnd_cur->max_bnd_recno = bnd_prev->min_bnd_recno; + + WT_RET(__wt_buf_set(session, &bnd_cur->max_bnd_key, + bnd_prev->min_bnd_key.data, bnd_prev->min_bnd_key.size)); + + /* Update current chunk's length */ + cur_len = WT_PTRDIFF(r->first_free, dsk) - bnd_cur->offset; + } + + /* + * Create an interim buffer if not already done to prepare the previous + * chunk's disk image. + */ + len = bnd_cur->offset; + WT_RET(bm->write_size(bm, session, &len)); + if (r->interim_buf == NULL) + WT_RET(__wt_scr_alloc(session, len, &r->interim_buf)); + else + WT_RET(__wt_buf_init(session, r->interim_buf, len)); + + dsk_tmp = r->interim_buf->mem; + memcpy(dsk_tmp, dsk, bnd_cur->offset); + dsk_tmp->recno = bnd_prev->max_bnd_recno; + dsk_tmp->u.entries = bnd_prev->max_bnd_entries; + dsk_tmp->mem_size = WT_STORE_SIZE(bnd_cur->offset); + r->interim_buf->size = dsk_tmp->mem_size; + WT_RET(__rec_split_write(session, r, bnd_prev, r->interim_buf, false)); + + /* Shift the current chunk to the start of the buffer */ + dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); + (void)memmove(dsk_start, (uint8_t *)dsk + bnd_cur->offset, cur_len); + + /* Fix boundary offset */ + bnd_cur->offset = WT_PAGE_HEADER_BYTE_SIZE(btree); + /* Fix where free points */ + r->first_free = dsk_start + cur_len; return (0); } @@ -2330,6 +2531,9 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) btree = S2BT(session); dsk = r->disk_image.mem; + /* Fixed length col store can call with next_len 0 */ + WT_ASSERT(session, next_len == 0 || r->space_avail < next_len); + /* * We should never split during salvage, and we're about to drop core * because there's no parent page. @@ -2339,147 +2543,63 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) "%s page too large, attempted split during salvage", __wt_page_type_string(r->page->type)); - /* Hitting a page boundary resets the dictionary, in all cases. */ - __rec_dictionary_reset(r); - - inuse = WT_PTRDIFF(r->first_free, dsk); - switch (r->bnd_state) { - case SPLIT_BOUNDARY: - /* - * We can get here if the first key/value pair won't fit. - * Additionally, grow the buffer to contain the current item if - * we haven't already consumed a reasonable portion of a split - * chunk. - */ - if (inuse < r->split_size / 2) - break; - - /* - * About to cross a split boundary but not yet forced to split - * into multiple pages. If we have to split, this is one of the - * split points, save information about where we are when the - * split would have happened. - */ - WT_RET(__rec_split_bnd_grow(session, r)); - last = &r->bnd[r->bnd_next++]; - next = last + 1; - - /* Set the number of entries for the just finished chunk. */ - last->entries = r->entries - r->total_entries; - r->total_entries = r->entries; - - /* Set the key for the next chunk. */ - next->recno = r->recno; - if (dsk->type == WT_PAGE_ROW_INT || - dsk->type == WT_PAGE_ROW_LEAF) - WT_RET(__rec_split_row_promote( - session, r, &next->key, dsk->type)); - - /* - * Set the starting buffer offset and clear the entries (the - * latter not required, but cleaner). - */ - next->offset = WT_PTRDIFF(r->first_free, dsk); - next->entries = 0; - - /* Set the space available to another split-size chunk. */ - r->space_avail = - r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - - /* - * Adjust the space available to handle two cases: - * - We don't have enough room for another full split-size - * chunk on the page. - * - We chose to fill past a page boundary because of a - * large item. - */ - if (inuse + r->space_avail > r->page_size) { - r->space_avail = - r->page_size > inuse ? (r->page_size - inuse) : 0; - - /* There are no further boundary points. */ - r->bnd_state = SPLIT_MAX; - } - - /* - * Return if the next object fits into this page, else we have - * to split the page. - */ - if (r->space_avail >= next_len) - return (0); - - /* FALLTHROUGH */ - case SPLIT_MAX: - /* - * We're going to have to split and create multiple pages. - * - * Cycle through the saved split-point information, writing the - * split chunks we have tracked. The underlying fixup function - * sets the space available and other information, and copied - * any unwritten chunk of data to the beginning of the buffer. - */ - WT_RET(__rec_split_fixup(session, r)); - - /* We're done saving split chunks. */ - r->bnd_state = SPLIT_TRACKING_OFF; - break; - case SPLIT_TRACKING_OFF: - /* - * We can get here if the first key/value pair won't fit. - * Additionally, grow the buffer to contain the current item if - * we haven't already consumed a reasonable portion of a split - * chunk. - */ - if (inuse < r->split_size / 2) - break; + last = &r->bnd[r->bnd_next]; + inuse = (WT_PTRDIFF(r->first_free, dsk) - last->offset) + + WT_PAGE_HEADER_BYTE_SIZE(btree); - /* - * The key/value pairs didn't fit into a single page, but either - * we've already noticed that and are now processing the rest of - * the pairs at split size boundaries, or the split size was the - * same as the page size, and we never bothered with split point - * information at all. - */ - WT_RET(__rec_split_bnd_grow(session, r)); - last = &r->bnd[r->bnd_next++]; - next = last + 1; + /* + * We can get here if the first key/value pair won't fit. + * Additionally, grow the buffer to contain the current item if we + * haven't already consumed a reasonable portion of a split chunk. + */ + if (inuse < r->split_size / 2) + goto done; - /* - * Set the key for the next chunk (before writing the block, a - * key range is needed in that code). - */ - next->recno = r->recno; - if (dsk->type == WT_PAGE_ROW_INT || - dsk->type == WT_PAGE_ROW_LEAF) - WT_RET(__rec_split_row_promote( - session, r, &next->key, dsk->type)); + /* All page boundaries reset the dictionary. */ + __rec_dictionary_reset(r); - /* Clear the entries (not required, but cleaner). */ - next->entries = 0; + /* Set the number of entries for the just finished chunk. */ + last->max_bnd_entries = r->entries; - /* Finalize the header information and write the page. */ - dsk->recno = last->recno; - dsk->u.entries = r->entries; - dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); + /* + * In case of bulk load, write out chunks as we get them. Otherwise we + * keep two chunks in memory at a given time. So, if there is a previous + * chunk, write it out, making space in the buffer for the next chunk to + * be written. + */ + if (r->is_bulk_load) { + dsk->recno = last->max_bnd_recno; + dsk->u.entries = last->max_bnd_entries; + dsk->mem_size = (uint32_t)inuse; r->disk_image.size = dsk->mem_size; - WT_RET( - __rec_split_write(session, r, last, &r->disk_image, false)); - - /* - * Set the caller's entry count and buffer information for the - * next chunk. We only get here if we're not splitting or have - * already split, so it's split-size chunks from here on out. - */ - r->entries = 0; + WT_RET(__rec_split_write( + session, r, last, &r->disk_image, false)); + /* Fix where free points */ r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); - r->space_avail = - r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - break; - case SPLIT_TRACKING_RAW: - return (__wt_illegal_value(session, NULL)); - } + } else if (r->bnd_next != 0) + WT_RET(__rec_split_write_prev_and_shift_cur(session, r, false)); - /* + /* Prepare the next boundary */ + WT_RET(__rec_split_bnd_grow(session, r)); + r->bnd_next++; + next = &r->bnd[r->bnd_next]; + next->offset = WT_PTRDIFF(r->first_free, dsk); + /* Set the key for the next chunk. */ + next->max_bnd_recno = r->recno; + if (dsk->type == WT_PAGE_ROW_INT || dsk->type == WT_PAGE_ROW_LEAF) + WT_RET(__rec_split_row_promote( + session, r, &next->max_bnd_key, dsk->type)); + + r->entries = 0; + /* + * Set the space available to another split-size and minimum split-size + * chunk. + */ + r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + r->min_space_avail = + r->min_split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + +done: /* * Overflow values can be larger than the maximum page size but still be * "on-page". If the next key/value pair is larger than space available * after a split has happened (in other words, larger than the maximum @@ -2496,6 +2616,64 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) return (0); } +/* + * __rec_split_crossing_bnd -- + * Save the details for the minimum split size boundary or call for a + * split. + */ +static inline int +__rec_split_crossing_bnd( + WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) +{ + WT_BOUNDARY *bnd; + WT_BTREE *btree; + WT_PAGE_HEADER *dsk; + size_t min_bnd_offset; + + WT_ASSERT(session, WT_CHECK_CROSSING_BND(r, next_len)); + + /* + * If crossing the minimum split size boundary, store the boundary + * details at the current location in the buffer. If we are crossing the + * split boundary at the same time, possible when the next record is + * large enough, just split at this point. + */ + if (WT_CROSSING_MIN_BND(r, next_len) && + !WT_CROSSING_SPLIT_BND(r, next_len)) { + btree = S2BT(session); + bnd = &r->bnd[r->bnd_next]; + dsk = r->disk_image.mem; + min_bnd_offset = (WT_PTRDIFF(r->first_free, dsk) - + bnd->offset) + WT_PAGE_HEADER_BYTE_SIZE(btree); + if (min_bnd_offset == WT_PAGE_HEADER_BYTE_SIZE(btree)) + /* + * This is possible if the first record doesn't fit in + * the minimum split size, we write this record without + * setting up any boundary here. We will get the + * opportunity to setup a boundary before writing out + * the next record. + */ + return (0); + + WT_ASSERT(session, bnd->min_bnd_offset == 0); + + /* All page boundaries reset the dictionary. */ + __rec_dictionary_reset(r); + + bnd->min_bnd_offset = min_bnd_offset; + bnd->min_bnd_entries = r->entries; + bnd->min_bnd_recno = r->recno; + if (dsk->type == WT_PAGE_ROW_INT || + dsk->type == WT_PAGE_ROW_LEAF) + WT_RET(__rec_split_row_promote( + session, r, &bnd->min_bnd_key, dsk->type)); + return (0); + } + + /* We are crossing a split boundary */ + return (__rec_split(session, r, next_len)); +} + /* * __rec_split_raw_worker -- * Handle the raw compression page reconciliation bookkeeping. @@ -2574,7 +2752,7 @@ __rec_split_raw_worker(WT_SESSION_IMPL *session, */ recno = WT_RECNO_OOB; if (dsk->type == WT_PAGE_COL_VAR) - recno = last->recno; + recno = last->max_bnd_recno; entry = max_image_slot = slots = 0; WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { @@ -2801,7 +2979,7 @@ no_slots: */ dst->size = result_len + WT_BLOCK_COMPRESS_SKIP; dsk_dst = dst->mem; - dsk_dst->recno = last->recno; + dsk_dst->recno = last->max_bnd_recno; dsk_dst->mem_size = r->raw_offsets[result_slots] + WT_BLOCK_COMPRESS_SKIP; dsk_dst->u.entries = r->raw_entries[result_slots - 1]; @@ -2821,7 +2999,7 @@ no_slots: WT_RET(__wt_strndup(session, dsk, dsk_dst->mem_size, &last->disk_image)); disk_image = last->disk_image; - disk_image->recno = last->recno; + disk_image->recno = last->max_bnd_recno; disk_image->mem_size = dsk_dst->mem_size; disk_image->u.entries = dsk_dst->u.entries; } @@ -2837,7 +3015,7 @@ no_slots: len = WT_PTRDIFF( r->first_free, (uint8_t *)dsk + dsk_dst->mem_size); dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - (void)memmove(dsk_start, (uint8_t *)r->first_free - len, len); + (void)memmove(dsk_start, r->first_free - len, len); r->entries -= r->raw_entries[result_slots - 1]; r->first_free = dsk_start + len; @@ -2851,14 +3029,14 @@ no_slots: */ switch (dsk->type) { case WT_PAGE_COL_INT: - next->recno = r->raw_recnos[result_slots]; + next->max_bnd_recno = r->raw_recnos[result_slots]; break; case WT_PAGE_COL_VAR: - next->recno = r->raw_recnos[result_slots - 1]; + next->max_bnd_recno = r->raw_recnos[result_slots - 1]; break; case WT_PAGE_ROW_INT: case WT_PAGE_ROW_LEAF: - next->recno = WT_RECNO_OOB; + next->max_bnd_recno = WT_RECNO_OOB; if (!last_block) { /* * Confirm there was uncompressed data remaining @@ -2867,7 +3045,7 @@ no_slots: */ WT_ASSERT(session, len > 0); WT_RET(__rec_split_row_promote_cell( - session, dsk, &next->key)); + session, dsk, &next->max_bnd_key)); } break; } @@ -2879,7 +3057,7 @@ no_slots: */ WT_STAT_DATA_INCR(session, compress_raw_fail); - dsk->recno = last->recno; + dsk->recno = last->max_bnd_recno; dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); dsk->u.entries = r->entries; r->disk_image.size = dsk->mem_size; @@ -2956,35 +3134,9 @@ __rec_split_raw(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) static int __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BOUNDARY *bnd; + WT_BOUNDARY *bnd_cur, *bnd_prev; WT_PAGE_HEADER *dsk; - - /* Adjust the boundary information based on our split status. */ - switch (r->bnd_state) { - case SPLIT_BOUNDARY: - case SPLIT_MAX: - /* - * We never split, the reconciled page fit into a maximum page - * size. Change the first boundary slot to represent the full - * page (the first boundary slot is largely correct, just update - * the number of entries). - */ - r->bnd_next = 0; - break; - case SPLIT_TRACKING_OFF: - /* - * If we have already split, or aren't tracking boundaries, put - * the remaining data in the next boundary slot. - */ - WT_RET(__rec_split_bnd_grow(session, r)); - break; - case SPLIT_TRACKING_RAW: - /* - * We were configured for raw compression, and either we never - * wrote anything, or there's a remaindered block of data. - */ - break; - } + bool grow_bnd; /* * We may arrive here with no entries to write if the page was entirely @@ -3011,20 +3163,66 @@ __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (EBUSY); } - /* Set the boundary reference and increment the count. */ - bnd = &r->bnd[r->bnd_next++]; - bnd->entries = r->entries; - - /* Finalize the header information. */ dsk = r->disk_image.mem; - dsk->recno = bnd->recno; - dsk->u.entries = r->entries; + + /* Set the number of entries for the just finished chunk. */ + bnd_cur = &r->bnd[r->bnd_next]; + bnd_cur->max_bnd_entries = r->entries; + + grow_bnd = true; + /* + * We can reach here even with raw_compression when the last split chunk + * is too small to be sent for raw compression. + */ + if (!r->is_bulk_load && !r->raw_compression) { + if (WT_PTRDIFF(r->first_free, dsk) > r->page_size && + r->bnd_next != 0) { + /* + * We hold two boundaries worth of data in the buffer, + * and this data doesn't fit in a single page. If the + * last chunk is too small, readjust the boundary to a + * pre-computed minimum. + * Write out the penultimate chunk to the disk as a page + */ + WT_RET(__rec_split_write_prev_and_shift_cur( + session, r, true)); + } else + if (r->bnd_next != 0) { + /* + * We have two boundaries, but the data in the + * buffer can fit a single page. Merge the + * boundaries to create a single chunk. + */ + bnd_prev = bnd_cur - 1; + bnd_prev->max_bnd_entries += + bnd_cur->max_bnd_entries; + r->bnd_next--; + grow_bnd = false; + } + } + + /* + * We already have space for an extra boundary if we merged two + * boundaries above, in that case we do not need to grow the boundary + * structure. + */ + if (grow_bnd) + WT_RET(__rec_split_bnd_grow(session, r)); + bnd_cur = &r->bnd[r->bnd_next]; + r->bnd_next++; + + /* + * Current boundary now has all the remaining data/last page now. + * Let's write it to the disk + */ + dsk->recno = bnd_cur->max_bnd_recno; + dsk->u.entries = bnd_cur->max_bnd_entries; dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); r->disk_image.size = dsk->mem_size; /* If this is a checkpoint, we're done, otherwise write the page. */ - return (__rec_is_checkpoint(session, r, bnd) ? - 0 : __rec_split_write(session, r, bnd, &r->disk_image, true)); + return (__rec_is_checkpoint(session, r, bnd_cur) ? + 0 : __rec_split_write(session, r, bnd_cur, &r->disk_image, true)); } /* @@ -3057,98 +3255,6 @@ __rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (__rec_split_finish_std(session, r)); } -/* - * __rec_split_fixup -- - * Fix up after crossing the maximum page boundary. - */ -static int -__rec_split_fixup(WT_SESSION_IMPL *session, WT_RECONCILE *r) -{ - WT_BOUNDARY *bnd; - WT_BTREE *btree; - WT_DECL_ITEM(tmp); - WT_DECL_RET; - WT_PAGE_HEADER *dsk; - size_t i, len; - uint8_t *dsk_start, *p; - - /* - * When we overflow physical limits of the page, we walk the list of - * split chunks we've created and write those pages out, then update - * the caller's information. - */ - btree = S2BT(session); - - /* - * The data isn't laid out on a page boundary or nul padded; copy it to - * a clean, aligned, padded buffer before writing it. - * - * Allocate a scratch buffer to hold the new disk image. Copy the disk - * page's header and block-manager space into the scratch buffer, most - * of the header information remains unchanged between the pages. - */ - WT_RET(__wt_scr_alloc(session, r->disk_image.memsize, &tmp)); - dsk = tmp->mem; - memcpy(dsk, r->disk_image.mem, WT_PAGE_HEADER_BYTE_SIZE(btree)); - - /* - * For each split chunk we've created, update the disk image and copy - * it into place. - */ - dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - for (i = 0, bnd = r->bnd; i < r->bnd_next; ++i, ++bnd) { - /* Copy the page contents to the temporary buffer. */ - len = (bnd + 1)->offset - bnd->offset; - memcpy(dsk_start, - (uint8_t *)r->disk_image.mem + bnd->offset, len); - - /* Finalize the header information and write the page. */ - dsk->recno = bnd->recno; - dsk->u.entries = bnd->entries; - tmp->size = WT_PAGE_HEADER_BYTE_SIZE(btree) + len; - dsk->mem_size = WT_STORE_SIZE(tmp->size); - WT_ERR(__rec_split_write(session, r, bnd, tmp, false)); - } - - /* - * There is probably a remnant in the working buffer that didn't get - * written, copy it down to the beginning of the working buffer. - * - * Confirm the remnant is no larger than a split-sized chunk, including - * header. We know that's the maximum sized remnant because we only have - * remnants if split switches from accumulating to a split boundary to - * accumulating to the end of the page (the other path here is when we - * hit a split boundary, there was room for another split chunk in the - * page, and the next item still wouldn't fit, in which case there is no - * remnant). So: we were accumulating to the end of the page and created - * a remnant. We know the remnant cannot be as large as a split-sized - * chunk, including header, because if there was room for that large a - * remnant, we wouldn't have switched from accumulating to a page end. - */ - p = (uint8_t *)r->disk_image.mem + bnd->offset; - len = WT_PTRDIFF(r->first_free, p); - if (len >= r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree)) - WT_PANIC_ERR(session, EINVAL, - "Reconciliation remnant too large for the split buffer"); - dsk = r->disk_image.mem; - dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - (void)memmove(dsk_start, p, len); - - /* - * Fix up our caller's information, including updating the starting - * record number. - */ - r->entries -= r->total_entries; - r->first_free = dsk_start + len; - WT_ASSERT(session, - r->page_size >= (WT_PAGE_HEADER_BYTE_SIZE(btree) + len)); - r->space_avail = - r->split_size - (WT_PAGE_HEADER_BYTE_SIZE(btree) + len); - -err: __wt_scr_free(session, &tmp); - return (ret); -} - /* * __rec_split_write -- * Write a disk block out for the split helper functions. @@ -3170,11 +3276,17 @@ __rec_split_write(WT_SESSION_IMPL *session, int cmp; uint8_t addr[WT_BTREE_MAX_ADDR_COOKIE]; bool need_image; +#ifdef HAVE_DIAGNOSTIC + bool verify_image; +#endif btree = S2BT(session); dsk = buf->mem; page = r->page; mod = page->modify; +#ifdef HAVE_DIAGNOSTIC + verify_image = true; +#endif /* Set the zero-length value flag in the page header. */ if (dsk->type == WT_PAGE_ROW_LEAF) { @@ -3186,8 +3298,6 @@ __rec_split_write(WT_SESSION_IMPL *session, F_SET(dsk, WT_PAGE_EMPTY_V_NONE); } - bnd->entries = r->entries; - /* Initialize the address (set the page type for the parent). */ switch (dsk->type) { case WT_PAGE_COL_FIX: @@ -3233,7 +3343,8 @@ __rec_split_write(WT_SESSION_IMPL *session, switch (page->type) { case WT_PAGE_COL_FIX: case WT_PAGE_COL_VAR: - if (WT_INSERT_RECNO(supd->ins) >= (bnd + 1)->recno) + if (WT_INSERT_RECNO(supd->ins) >= + (bnd + 1)->max_bnd_recno) goto supd_check_complete; break; case WT_PAGE_ROW_LEAF: @@ -3244,8 +3355,8 @@ __rec_split_write(WT_SESSION_IMPL *session, key->data = WT_INSERT_KEY(supd->ins); key->size = WT_INSERT_KEY_SIZE(supd->ins); } - WT_ERR(__wt_compare(session, - btree->collator, key, &(bnd + 1)->key, &cmp)); + WT_ERR(__wt_compare(session, btree->collator, + key, &(bnd + 1)->max_bnd_key, &cmp)); if (cmp >= 0) goto supd_check_complete; break; @@ -3335,18 +3446,21 @@ supd_check_complete: #ifdef HAVE_VERBOSE /* Output a verbose message if we create a page without many entries */ - if (WT_VERBOSE_ISSET(session, WT_VERB_SPLIT) && r->entries < 6) + if (WT_VERBOSE_ISSET(session, WT_VERB_SPLIT) && + bnd->max_bnd_entries < 6) __wt_verbose(session, WT_VERB_SPLIT, "Reconciliation creating a page with %" PRIu32 " entries, memory footprint %" WT_SIZET_FMT - ", page count %" PRIu32 ", %s, split state: %d", - r->entries, r->page->memory_footprint, r->bnd_next, - F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint", - r->bnd_state); + ", page count %" PRIu32 ", %s", bnd->max_bnd_entries, + r->page->memory_footprint, r->bnd_next, + F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint"); #endif WT_ERR(__wt_bt_write(session, buf, addr, &addr_size, false, F_ISSET(r, WT_CHECKPOINTING), bnd->already_compressed)); +#ifdef HAVE_DIAGNOSTIC + verify_image = false; +#endif WT_ERR(__wt_strndup(session, addr, addr_size, &bnd->addr.addr)); bnd->addr.size = (uint8_t)addr_size; @@ -3373,9 +3487,20 @@ copy_image: */ need_image = F_ISSET(r, WT_EVICT_SCRUB) || (F_ISSET(r, WT_EVICT_UPDATE_RESTORE) && bnd->supd != NULL); - if (need_image && bnd->disk_image == NULL) + if (need_image && bnd->disk_image == NULL) { +#ifdef HAVE_DIAGNOSTIC + /* + * The I/O routines verify all disk images we write, but there + * are paths in reconciliation that don't do I/O. Verify those + * images, too. + */ + WT_ASSERT(session, verify_image == false || + __wt_verify_dsk_image( + session, "[reconcile-image]", buf->data, 0, true) == 0); +#endif WT_ERR(__wt_strndup( session, buf->data, buf->size, &bnd->disk_image)); + } if (!need_image) __wt_free(session, bnd->disk_image); @@ -3531,11 +3656,12 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) uint64_t recno; btree = S2BT(session); + /* * Bulk-load is only permitted on newly created files, not any empty * file -- see the checkpoint code for a discussion. */ - if (!btree->bulk_load_ok) + if (!btree->original) WT_RET_MSG(session, EINVAL, "bulk-load is only possible for newly created trees"); @@ -3552,16 +3678,7 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) r = cbulk->reconcile; r->is_bulk_load = true; - recno = WT_RECNO_OOB; /* -Werror=maybe-uninitialized */ - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: - recno = 1; - break; - case BTREE_ROW: - recno = WT_RECNO_OOB; - break; - } + recno = btree->type == BTREE_ROW ? WT_RECNO_OOB : 1; return (__rec_split_init( session, r, cbulk->leaf, recno, btree->maxleafpage)); @@ -3599,7 +3716,7 @@ __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_RET(__rec_split_finish(session, r)); WT_RET(__rec_write_wrapup(session, r, r->page)); - WT_RET(__rec_write_status(session, r, r->page)); + __rec_write_page_status(session, r); /* Mark the page's parent and the tree dirty. */ parent = r->ref->home; @@ -3636,11 +3753,12 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) cursor->value.data, cursor->value.size, (uint64_t)0)); /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) - WT_RET( - __rec_split_raw(session, r, key->len + val->len)); - else { + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) + WT_RET(__rec_split_raw( + session, r, key->len + val->len)); + } else + if (WT_CROSSING_SPLIT_BND(r, key->len + val->len)) { /* * Turn off prefix compression until a full key written * to the new page, and (unless already working with an @@ -3652,10 +3770,9 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_RET(__rec_cell_build_leaf_key( session, r, NULL, 0, &ovfl_key)); } - - WT_RET(__rec_split(session, r, key->len + val->len)); + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); } - } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -3696,6 +3813,10 @@ __rec_col_fix_bulk_insert_split_check(WT_CURSOR_BULK *cbulk) * split. * * Boundary: split or write the page. + * + * No need to have a minimum split size boundary, all + * pages are filled 100% except the last, allowing it to + * grow in the future. */ __rec_incr(session, r, cbulk->entry, __bitstr_size( @@ -3800,10 +3921,12 @@ __wt_bulk_insert_var( r, cbulk->last.data, cbulk->last.size, cbulk->rle)); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_RET(__rec_split_raw(session, r, val->len)); + } else + if (WT_CROSSING_SPLIT_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (btree->dictionary) @@ -3939,10 +4062,13 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_CHILD_RELEASE_ERR(session, hazard, ref); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_ERR(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_ERR(__rec_split_raw(session, r, val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, val->len)) + WT_ERR(__rec_split_crossing_bnd( + session, r, val->len)); /* Copy the value onto the page. */ __rec_copy_incr(session, r, val); @@ -3984,10 +4110,13 @@ __rec_col_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) addr->addr, addr->size, __rec_vtype(addr), r->recno); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_RET(__rec_split_raw(session, r, val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, val->len)); /* Copy the value onto the page. */ __rec_copy_incr(session, r, val); @@ -4017,7 +4146,7 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) /* Copy the original, disk-image bytes into place. */ memcpy(r->first_free, page->pg_fix_bitf, - __bitstr_size((size_t)page->pg_fix_entries * btree->bitcnt)); + __bitstr_size((size_t)page->entries * btree->bitcnt)); /* Update any changes to the original on-page data items. */ WT_SKIP_FOREACH(ins, WT_COL_UPDATE_SINGLE(page)) { @@ -4029,9 +4158,8 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) } /* Calculate the number of entries per page remainder. */ - entry = page->pg_fix_entries; - nrecs = WT_FIX_BYTES_TO_ENTRIES( - btree, r->space_avail) - page->pg_fix_entries; + entry = page->entries; + nrecs = WT_FIX_BYTES_TO_ENTRIES(btree, r->space_avail) - page->entries; r->recno += entry; /* Walk any append list. */ @@ -4096,6 +4224,10 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) * split. * * Boundary: split or write the page. + * + * No need to have a minimum split size boundary, all + * pages are filled 100% except the last, allowing it to + * grow in the future. */ __rec_incr(session, r, entry, __bitstr_size((size_t)entry * btree->bitcnt)); @@ -4154,7 +4286,7 @@ __rec_col_fix_slvg(WT_SESSION_IMPL *session, session, r, page, pageref->ref_recno, btree->maxleafpage)); /* We may not be taking all of the entries on the original page. */ - page_take = salvage->take == 0 ? page->pg_fix_entries : salvage->take; + page_take = salvage->take == 0 ? page->entries : salvage->take; page_start = salvage->skip == 0 ? 0 : salvage->skip; /* Calculate the number of entries per page. */ @@ -4252,10 +4384,13 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, session, r, value->data, value->size, rle)); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_RET(__rec_split_raw(session, r, val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, val->len)); /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) @@ -4450,8 +4585,8 @@ record_loop: /* * * Write a placeholder. */ - WT_ASSERT(session, - F_ISSET(r, WT_EVICT_UPDATE_RESTORE)); + WT_ASSERT(session, + F_ISSET(r, WT_EVICT_UPDATE_RESTORE)); data = "@"; size = 1; @@ -4918,11 +5053,12 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) r->cell_zero = false; /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) WT_ERR(__rec_split_raw( session, r, key->len + val->len)); - else { + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) { /* * In one path above, we copied address blocks * from the page rather than building the actual @@ -4934,10 +5070,10 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_IKEY_DATA(ikey), ikey->size)); key_onpage_ovfl = false; } - WT_ERR(__rec_split( + + WT_ERR(__rec_split_crossing_bnd( session, r, key->len + val->len)); } - } /* Copy the key and value onto the page. */ __rec_copy_incr(session, r, key); @@ -4987,10 +5123,14 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) addr->addr, addr->size, __rec_vtype(addr), WT_RECNO_OOB); /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, key->len + val->len) : - __rec_split(session, r, key->len + val->len)); + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) + WT_RET(__rec_split_raw( + session, r, key->len + val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); /* Copy the key and value onto the page. */ __rec_copy_incr(session, r, key); @@ -5319,16 +5459,17 @@ build: } /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) WT_ERR(__rec_split_raw( session, r, key->len + val->len)); - else { + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) { /* - * In one path above, we copied address blocks - * from the page rather than building the actual - * key. In that case, we have to build the key - * now because we are about to promote it. + * If we copied address blocks from the page + * rather than building the actual key, we have + * to build the key now because we are about to + * promote it. */ if (key_onpage_ovfl) { WT_ERR(__wt_dsk_cell_data_ref(session, @@ -5347,14 +5488,13 @@ build: if (!ovfl_key) WT_ERR( __rec_cell_build_leaf_key( - session, - r, NULL, 0, &ovfl_key)); + session, r, NULL, 0, + &ovfl_key)); } - WT_ERR(__rec_split( + WT_ERR(__rec_split_crossing_bnd( session, r, key->len + val->len)); } - } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -5417,11 +5557,12 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins), &ovfl_key)); /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) WT_RET(__rec_split_raw( session, r, key->len + val->len)); - else { + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) { /* * Turn off prefix compression until a full key * written to the new page, and (unless already @@ -5433,14 +5574,13 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) if (!ovfl_key) WT_RET( __rec_cell_build_leaf_key( - session, - r, NULL, 0, &ovfl_key)); + session, r, NULL, 0, + &ovfl_key)); } - WT_RET(__rec_split( + WT_RET(__rec_split_crossing_bnd( session, r, key->len + val->len)); } - } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -5467,7 +5607,6 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) static int __rec_split_discard(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_PAGE_MODIFY *mod; WT_MULTI *multi; uint32_t i; @@ -5527,7 +5666,7 @@ __rec_split_discard(WT_SESSION_IMPL *session, WT_PAGE *page) break; } - return (ret); + return (0); } /* @@ -5553,13 +5692,14 @@ __rec_split_dump_keys(WT_SESSION_IMPL *session, WT_PAGE *page, WT_RECONCILE *r) __wt_verbose(session, WT_VERB_SPLIT, "starting key %s", __wt_buf_set_printable( - session, bnd->key.data, bnd->key.size, tkey)); + session, bnd->max_bnd_key.data, + bnd->max_bnd_key.size, tkey)); break; case WT_PAGE_COL_FIX: case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: __wt_verbose(session, WT_VERB_SPLIT, - "starting recno %" PRIu64, bnd->recno); + "starting recno %" PRIu64, bnd->max_bnd_recno); break; WT_ILLEGAL_VALUE_ERR(session); } @@ -5821,10 +5961,10 @@ __rec_split_row(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* We never set the first page's key, grab it from the original page. */ ref = r->ref; if (__wt_ref_is_root(ref)) - WT_RET(__wt_buf_set(session, &r->bnd[0].key, "", 1)); + WT_RET(__wt_buf_set(session, &r->bnd[0].max_bnd_key, "", 1)); else { __wt_ref_key(ref->home, ref, &p, &size); - WT_RET(__wt_buf_set(session, &r->bnd[0].key, p, size)); + WT_RET(__wt_buf_set(session, &r->bnd[0].max_bnd_key, p, size)); } /* Allocate, then initialize the array of replacement blocks. */ @@ -5832,8 +5972,8 @@ __rec_split_row(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) for (multi = mod->mod_multi, bnd = r->bnd, i = 0; i < r->bnd_next; ++multi, ++bnd, ++i) { - WT_RET(__wt_row_ikey_alloc(session, 0, - bnd->key.data, bnd->key.size, &multi->key.ikey)); + WT_RET(__wt_row_ikey_alloc(session, 0, bnd->max_bnd_key.data, + bnd->max_bnd_key.size, &multi->key.ikey)); /* * Copy any disk image. Don't take saved updates without a @@ -5880,7 +6020,7 @@ __rec_split_col(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) for (multi = mod->mod_multi, bnd = r->bnd, i = 0; i < r->bnd_next; ++multi, ++bnd, ++i) { - multi->key.recno = bnd->recno; + multi->key.recno = bnd->max_bnd_recno; /* * Copy any disk image. Don't take saved updates without a @@ -6357,7 +6497,8 @@ __rec_dictionary_lookup( for (dp = __rec_dictionary_skip_search(r->dictionary_head, hash); dp != NULL && dp->hash == hash; dp = dp->next[0]) { WT_RET(__wt_cell_pack_data_match( - dp->cell, &val->cell, val->buf.data, &match)); + (WT_CELL *)((uint8_t *)r->disk_image.mem + dp->offset), + &val->cell, val->buf.data, &match)); if (match) { WT_STAT_DATA_INCR(session, rec_dictionary); *dpp = dp; @@ -6383,7 +6524,7 @@ __rec_dictionary_lookup( * know where on the page it will be written). */ next = r->dictionary[r->dictionary_next++]; - next->cell = NULL; /* Not necessary, just cautious. */ + next->offset = 0; /* Not necessary, just cautious. */ next->hash = hash; __rec_dictionary_skip_insert(r->dictionary_head, next, hash); *dpp = next; diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c new file mode 100644 index 00000000000..26d800aa98e --- /dev/null +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -0,0 +1,180 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __alter_file -- + * Alter a file. + */ +static int +__alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) +{ + WT_DECL_RET; + const char *cfg[4], *filename; + char *config, *newconfig; + + filename = uri; + newconfig = NULL; + if (!WT_PREFIX_SKIP(filename, "file:")) + return (__wt_unexpected_object_type(session, uri, "file:")); + + /* Find the URI */ + WT_RET(__wt_metadata_search(session, uri, &config)); + + WT_ASSERT(session, newcfg[0] != NULL); + /* + * Start with the base configuration because collapse is like + * a projection and if we are reading older metadata, it may not + * have all the components. + */ + cfg[0] = WT_CONFIG_BASE(session, file_meta); + cfg[1] = config; + cfg[2] = newcfg[0]; + cfg[3] = NULL; + WT_ERR(__wt_config_collapse(session, cfg, &newconfig)); + /* + * Only rewrite if there are changes. + */ + if (strcmp(config, newconfig) != 0) + WT_ERR(__wt_metadata_update(session, uri, newconfig)); + else + WT_STAT_CONN_INCR(session, session_table_alter_skip); + +err: __wt_free(session, config); + __wt_free(session, newconfig); + return (ret); +} + +/* + * __alter_colgroup -- + * WT_SESSION::alter for a colgroup. + */ +static int +__alter_colgroup( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_COLGROUP *colgroup; + WT_DECL_RET; + + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); + + /* If we can get the colgroup, perform any potential alterations. */ + if ((ret = __wt_schema_get_colgroup( + session, uri, false, NULL, &colgroup)) == 0) + WT_TRET(__wt_schema_alter(session, colgroup->source, cfg)); + + return (ret); +} + +/* + * __alter_index -- + * WT_SESSION::alter for an index. + */ +static int +__alter_index( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_INDEX *idx; + WT_DECL_RET; + + /* If we can get the index, perform any potential alterations. */ + if ((ret = __wt_schema_get_index( + session, uri, false, NULL, &idx)) == 0) + WT_TRET(__wt_schema_alter(session, idx->source, cfg)); + + return (ret); +} + +/* + * __alter_table -- + * WT_SESSION::alter for a table. + */ +static int +__alter_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_COLGROUP *colgroup; + WT_DECL_RET; + WT_TABLE *table; + const char *name; + u_int i; + + name = uri; + (void)WT_PREFIX_SKIP(name, "table:"); + + WT_RET(__wt_schema_get_table( + session, name, strlen(name), true, &table)); + + /* + * Alter the column groups only if we are using the default + * column group. Otherwise the user should alter each + * index or column group explicitly. + */ + if (table->ncolgroups == 0) + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((colgroup = table->cgroups[i]) == NULL) + continue; + /* + * Alter the column group before updating the metadata + * to avoid the metadata for the table becoming + * inconsistent if we can't get exclusive access. + */ + WT_ERR(__wt_schema_alter( + session, colgroup->source, cfg)); + } +err: __wt_schema_release_table(session, table); + return (ret); +} + +/* + * __wt_schema_alter -- + * Process a WT_SESSION::alter operation for all supported types. + */ +int +__wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DATA_SOURCE *dsrc; + WT_DECL_RET; + + WT_RET(__wt_meta_track_on(session)); + + /* Paranoia: clear any handle from our caller. */ + session->dhandle = NULL; + + if (WT_PREFIX_MATCH(uri, "colgroup:")) + ret = __alter_colgroup(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "file:")) + ret = __alter_file(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "index:")) + ret = __alter_index(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "lsm:")) + ret = __wt_lsm_tree_alter(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "table:")) + ret = __alter_table(session, uri, cfg); + else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) + ret = dsrc->alter == NULL ? + __wt_object_unsupported(session, uri) : + dsrc->alter(dsrc, + &session->iface, uri, (WT_CONFIG_ARG *)cfg); + else + ret = __wt_bad_object_type(session, uri); + + /* + * Map WT_NOTFOUND to ENOENT, based on the assumption WT_NOTFOUND means + * there was no metadata entry. + */ + if (ret == WT_NOTFOUND) + ret = ENOENT; + + /* Bump the schema generation so that stale data is ignored. */ + ++S2C(session)->schema_gen; + + WT_TRET(__wt_meta_track_off(session, true, ret != 0)); + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 020d5e72c13..0677fa711a5 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -35,7 +35,7 @@ __wt_direct_io_size_check(WT_SESSION_IMPL *session, * units of its happy place. */ if (FLD_ISSET(conn->direct_io, - WT_DIRECT_IO_CHECKPOINT | WT_DIRECT_IO_DATA)) { + WT_DIRECT_IO_CHECKPOINT | WT_DIRECT_IO_DATA)) { align = (int64_t)conn->buffer_alignment; if (align != 0 && (cval.val < align || cval.val % align != 0)) WT_RET_MSG(session, EINVAL, @@ -601,7 +601,8 @@ __create_table(WT_SESSION_IMPL *session, if (ncolgroups == 0) { cgsize = strlen("colgroup:") + strlen(tablename) + 1; WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); - snprintf(cgname, cgsize, "colgroup:%s", tablename); + WT_ERR(__wt_snprintf( + cgname, cgsize, "colgroup:%s", tablename)); WT_ERR(__create_colgroup( session, cgname, exclusive, config)); } diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 65c955cf1e9..49801e4e5f9 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -30,7 +30,7 @@ __drop_file( WT_RET(__wt_schema_backup_check(session, filename)); /* Close all btree handles associated with this file. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __wt_conn_dhandle_close_all(session, uri, force)); WT_RET(ret); @@ -75,7 +75,7 @@ __drop_colgroup( /* * __drop_index -- - * WT_SESSION::drop for a colgroup. + * WT_SESSION::drop for an index. */ static int __drop_index( @@ -85,7 +85,7 @@ __drop_index( WT_DECL_RET; WT_TABLE *table; - /* If we can get the colgroup, detach it from the table. */ + /* If we can get the index, detach it from the table. */ if ((ret = __wt_schema_get_index( session, uri, force, &table, &idx)) == 0) { table->idx_complete = false; @@ -136,7 +136,7 @@ __drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) if ((idx = table->indices[i]) == NULL) continue; /* - * Drop the column group before updating the metadata to avoid + * Drop the index before updating the metadata to avoid * the metadata for the table becoming inconsistent if we can't * get exclusive access. */ diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 79e3ef1da7c..74ef5135a4a 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -25,7 +25,7 @@ __schema_add_table(WT_SESSION_IMPL *session, /* Make sure the metadata is open before getting other locks. */ WT_RET(__wt_metadata_cursor(session, NULL)); - WT_WITH_TABLE_LOCK(session, ret, + WT_WITH_TABLE_READ_LOCK(session, ret = __wt_schema_open_table( session, name, namelen, ok_incomplete, &table)); WT_RET(ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index f512482c162..a374f4c2831 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -33,7 +33,7 @@ __rename_file( WT_RET(__wt_schema_backup_check(session, filename)); WT_RET(__wt_schema_backup_check(session, newfile)); /* Close any btree handles in the file. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __wt_conn_dhandle_close_all(session, uri, false)); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index 433224a868e..9de4b916a79 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -26,7 +26,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) conn = S2C(session); if (!conn->hot_backup) return (0); - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); /* * There is a window at the end of a backup where the list has been * cleared from the connection but the flag is still set. It is safe @@ -34,7 +34,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) */ if (!conn->hot_backup || (backup_list = conn->hot_backup_list) == NULL) { - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); return (0); } for (i = 0; backup_list[i] != NULL; ++i) { @@ -43,7 +43,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) break; } } - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c index 52be76bb7a5..62cdd7d367b 100644 --- a/src/third_party/wiredtiger/src/schema/schema_worker.c +++ b/src/third_party/wiredtiger/src/schema/schema_worker.c @@ -49,7 +49,7 @@ __wt_schema_worker(WT_SESSION_IMPL *session, * any open file handles, including checkpoints. */ if (FLD_ISSET(open_flags, WT_DHANDLE_EXCLUSIVE)) { - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __wt_conn_dhandle_close_all( session, uri, false)); WT_ERR(ret); @@ -73,13 +73,8 @@ __wt_schema_worker(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_worker(session, idx->source, file_func, name_func, cfg, open_flags)); } else if (WT_PREFIX_MATCH(uri, "lsm:")) { - /* - * LSM compaction is handled elsewhere, but if we get here - * trying to compact files, don't descend into an LSM tree. - */ - if (file_func != __wt_compact) - WT_ERR(__wt_lsm_tree_worker(session, - uri, file_func, name_func, cfg, open_flags)); + WT_ERR(__wt_lsm_tree_worker(session, + uri, file_func, name_func, cfg, open_flags)); } else if (WT_PREFIX_SKIP(tablename, "table:")) { WT_ERR(__wt_schema_get_table(session, tablename, strlen(tablename), false, &table)); @@ -115,15 +110,12 @@ __wt_schema_worker(WT_SESSION_IMPL *session, } } else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) { wt_session = (WT_SESSION *)session; - if (file_func == __wt_compact && dsrc->compact != NULL) - WT_ERR(dsrc->compact( - dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); - else if (file_func == __wt_salvage && dsrc->salvage != NULL) + if (file_func == __wt_salvage && dsrc->salvage != NULL) WT_ERR(dsrc->salvage( - dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); + dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); else if (file_func == __wt_verify && dsrc->verify != NULL) WT_ERR(dsrc->verify( - dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); + dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); else if (file_func == __wt_checkpoint) ; else if (file_func == __wt_checkpoint_get_handles) diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 3afea383b08..b7daf0e2e02 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -58,7 +58,6 @@ int __wt_session_copy_values(WT_SESSION_IMPL *session) { WT_CURSOR *cursor; - WT_DECL_RET; TAILQ_FOREACH(cursor, &session->cursors, q) if (F_ISSET(cursor, WT_CURSTD_VALUE_INT)) { @@ -80,7 +79,7 @@ __wt_session_copy_values(WT_SESSION_IMPL *session) F_SET(cursor, WT_CURSTD_VALUE_EXT); } - return (ret); + return (0); } /* @@ -129,10 +128,48 @@ __session_clear(WT_SESSION_IMPL *session) * * For these reasons, be careful when clearing the session structure. */ - memset(session, 0, WT_SESSION_CLEAR_SIZE(session)); - session->hazard_size = 0; - session->nhazard = 0; + memset(session, 0, WT_SESSION_CLEAR_SIZE); + WT_INIT_LSN(&session->bg_sync_lsn); + + session->hazard_inuse = 0; + session->nhazard = 0; +} + +/* + * __session_alter -- + * Alter a table setting. + */ +static int +__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, alter, config, cfg); + + /* Disallow objects in the WiredTiger name space. */ + WT_ERR(__wt_str_name_check(session, uri)); + + /* + * We replace the default configuration listing with the current + * configuration. Otherwise the defaults for values that can be + * altered would override settings used by the user in create. + */ + cfg[0] = cfg[1]; + cfg[1] = NULL; + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_alter(session, uri, cfg)))); + +err: if (ret != 0) + WT_STAT_CONN_INCR(session, session_table_alter_fail); + else + WT_STAT_CONN_INCR(session, session_table_alter_success); + API_END_RET_NOTFOUND_MAP(session, ret); } /* @@ -197,9 +234,6 @@ __session_close(WT_SESSION *wt_session, const char *config) /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); - /* Destroy the thread's mutex. */ - WT_TRET(__wt_cond_destroy(session, &session->cond)); - /* The API lock protects opening and closing of sessions. */ __wt_spin_lock(session, &conn->api_lock); @@ -249,18 +283,36 @@ __session_reconfigure(WT_SESSION *wt_session, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, reconfigure, config, cfg); + /* + * Note that this method only checks keys that are passed in by the + * application: we don't want to reset other session settings to their + * default values. + */ + WT_UNUSED(cfg); + if (F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR_MSG(session, EINVAL, "transaction in progress"); - WT_TRET(__wt_session_reset_cursors(session, false)); + WT_ERR(__wt_session_reset_cursors(session, false)); - WT_ERR(__wt_config_gets_def(session, cfg, "isolation", 0, &cval)); - if (cval.len != 0) + ret = __wt_config_getones(session, config, "isolation", &cval); + if (ret == 0 && cval.len != 0) { session->isolation = session->txn.isolation = WT_STRING_MATCH("snapshot", cval.str, cval.len) ? WT_ISO_SNAPSHOT : WT_STRING_MATCH("read-uncommitted", cval.str, cval.len) ? WT_ISO_READ_UNCOMMITTED : WT_ISO_READ_COMMITTED; + } + WT_ERR_NOTFOUND_OK(ret); + + ret = __wt_config_getones(session, config, "ignore_cache_size", &cval); + if (ret == 0) { + if (cval.val) + F_SET(session, WT_SESSION_NO_EVICTION); + else + F_CLR(session, WT_SESSION_NO_EVICTION); + } + WT_ERR_NOTFOUND_OK(ret); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -465,8 +517,8 @@ __wt_session_create( { WT_DECL_RET; - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_create(session, uri, config))); return (ret); } @@ -659,8 +711,8 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_bt_rebalance, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_REBALANCE))); @@ -712,9 +764,9 @@ __session_rename(WT_SESSION *wt_session, WT_ERR(__wt_str_name_check(session, uri)); WT_ERR(__wt_str_name_check(session, newuri)); - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_rename(session, uri, newuri, cfg)))); err: if (ret != 0) @@ -772,45 +824,6 @@ __session_reset(WT_SESSION *wt_session) err: API_END_RET_NOTFOUND_MAP(session, ret); } -/* - * __wt_session_drop -- - * Internal version of WT_SESSION::drop. - */ -int -__wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_DECL_RET; - WT_CONFIG_ITEM cval; - bool checkpoint_wait, lock_wait; - - WT_RET(__wt_config_gets_def(session, cfg, "checkpoint_wait", 1, &cval)); - checkpoint_wait = cval.val != 0; - WT_RET(__wt_config_gets_def(session, cfg, "lock_wait", 1, &cval)); - lock_wait = cval.val != 0 || F_ISSET(session, WT_SESSION_LOCK_NO_WAIT); - - if (!lock_wait) - F_SET(session, WT_SESSION_LOCK_NO_WAIT); - - /* - * Take the checkpoint lock if there is a need to prevent the drop - * operation from failing with EBUSY due to an ongoing checkpoint. - */ - if (checkpoint_wait) - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - ret = __wt_schema_drop(session, uri, cfg)))); - else - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - ret = __wt_schema_drop(session, uri, cfg))); - - if (!lock_wait) - F_CLR(session, WT_SESSION_LOCK_NO_WAIT); - - return (ret); -} - /* * __session_drop -- * WT_SESSION->drop method. @@ -818,8 +831,10 @@ __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) static int __session_drop(WT_SESSION *wt_session, const char *uri, const char *config) { + WT_CONFIG_ITEM cval; WT_DECL_RET; WT_SESSION_IMPL *session; + bool checkpoint_wait, lock_wait; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, drop, config, cfg); @@ -827,7 +842,37 @@ __session_drop(WT_SESSION *wt_session, const char *uri, const char *config) /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); - ret = __wt_session_drop(session, uri, cfg); + WT_ERR(__wt_config_gets_def(session, cfg, "checkpoint_wait", 1, &cval)); + checkpoint_wait = cval.val != 0; + WT_ERR(__wt_config_gets_def(session, cfg, "lock_wait", 1, &cval)); + lock_wait = cval.val != 0; + + /* + * Take the checkpoint lock if there is a need to prevent the drop + * operation from failing with EBUSY due to an ongoing checkpoint. + */ + if (checkpoint_wait) { + if (lock_wait) + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = + __wt_schema_drop(session, uri, cfg)))); + else + WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, + WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, + WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, + ret = + __wt_schema_drop(session, uri, cfg)))); + } else { + if (lock_wait) + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_drop(session, uri, cfg))); + else + WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, + WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, + ret = __wt_schema_drop(session, uri, cfg))); + } err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_drop_fail); @@ -1018,8 +1063,8 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) "configurations"); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_salvage, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE))); @@ -1161,10 +1206,15 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, done: err: /* - * Close any locally-opened start cursor. + * Close any locally-opened start cursor. Reset application cursors, + * they've possibly moved and the application cannot use them. */ if (local_start) WT_TRET(start->close(start)); + else + WT_TRET(start->reset(start)); + if (stop != NULL) + WT_TRET(stop->reset(stop)); return (ret); } @@ -1219,8 +1269,8 @@ __session_truncate(WT_SESSION *wt_session, session, uri, start, stop)); else /* Wait for checkpoints to avoid EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_truncate(session, uri, cfg))); } else WT_ERR(__wt_session_range_truncate(session, uri, start, stop)); @@ -1275,8 +1325,8 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, upgrade, config, cfg); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_upgrade, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE))); @@ -1324,8 +1374,8 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) "configurations"); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY))); @@ -1441,6 +1491,20 @@ __session_transaction_pinned_range(WT_SESSION *wt_session, uint64_t *prange) err: API_END_RET(session, ret); } +/* + * __transaction_sync_run_chk -- + * Check to decide if the transaction sync call should continue running. + */ +static bool +__transaction_sync_run_chk(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + return (FLD_ISSET(conn->flags, WT_CONN_SERVER_LOG)); +} + /* * __session_transaction_sync -- * WT_SESSION->transaction_sync method. @@ -1455,7 +1519,7 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) WT_SESSION_IMPL *session; WT_TXN *txn; struct timespec now, start; - uint64_t timeout_ms, waited_ms; + uint64_t remaining_usec, timeout_ms, waited_ms; bool forever; session = (WT_SESSION_IMPL *)wt_session; @@ -1508,22 +1572,20 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) __wt_epoch(session, &start); /* * Keep checking the LSNs until we find it is stable or we reach - * our timeout. + * our timeout, or there's some other reason to quit. */ while (__wt_log_cmp(&session->bg_sync_lsn, &log->sync_lsn) > 0) { + if (!__transaction_sync_run_chk(session)) + WT_ERR(ETIMEDOUT); + __wt_cond_signal(session, conn->log_file_cond); __wt_epoch(session, &now); waited_ms = WT_TIMEDIFF_MS(now, start); - if (forever || waited_ms < timeout_ms) - /* - * Note, we will wait an increasing amount of time - * each iteration, likely doubling. Also note that - * the function timeout value is in usecs (we are - * computing the wait time in msecs and passing that - * in, unchanged, as the usecs to wait). - */ - __wt_cond_wait(session, log->log_sync_cond, waited_ms); - else + if (forever || waited_ms < timeout_ms) { + remaining_usec = (timeout_ms - waited_ms) * WT_THOUSAND; + __wt_cond_wait(session, log->log_sync_cond, + remaining_usec, __transaction_sync_run_chk); + } else WT_ERR(ETIMEDOUT); } @@ -1588,7 +1650,7 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) WT_ERR_MSG(session, EINVAL, "Checkpoint not permitted in a transaction"); - ret = __wt_txn_checkpoint(session, cfg); + ret = __wt_txn_checkpoint(session, cfg, true); /* * Release common session resources (for example, checkpoint may acquire @@ -1639,7 +1701,7 @@ __session_snapshot(WT_SESSION *wt_session, const char *config) WT_ERR(__wt_txn_named_snapshot_config( session, cfg, &has_create, &has_drop)); - __wt_writelock(session, txn_global->nsnap_rwlock); + __wt_writelock(session, &txn_global->nsnap_rwlock); /* Drop any snapshots to be removed first. */ if (has_drop) @@ -1649,7 +1711,7 @@ __session_snapshot(WT_SESSION *wt_session, const char *config) if (has_create) WT_ERR(__wt_txn_named_snapshot_begin(session, cfg)); -err: __wt_writeunlock(session, txn_global->nsnap_rwlock); +err: __wt_writeunlock(session, &txn_global->nsnap_rwlock); API_END_RET_NOTFOUND_MAP(session, ret); } @@ -1680,6 +1742,7 @@ __open_session(WT_CONNECTION_IMPL *conn, static const WT_SESSION stds = { NULL, NULL, + __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, @@ -1707,6 +1770,7 @@ __open_session(WT_CONNECTION_IMPL *conn, }, stds_readonly = { NULL, NULL, + __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, @@ -1748,7 +1812,7 @@ __open_session(WT_CONNECTION_IMPL *conn, * closes the connection. This is particularly intended to catch * cases where server threads open sessions. */ - WT_ASSERT(session, F_ISSET(conn, WT_CONN_SERVER_RUN)); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_CLOSING)); /* Find the first inactive session slot. */ for (session_ret = conn->sessions, @@ -1776,8 +1840,6 @@ __open_session(WT_CONNECTION_IMPL *conn, session_ret->name = NULL; session_ret->id = i; - WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); - if (WT_SESSION_FIRST_USE(session_ret)) __wt_random_init(&session_ret->rnd); @@ -1810,17 +1872,13 @@ __open_session(WT_CONNECTION_IMPL *conn, * session close because access to it isn't serialized. Allocate the * first time we open this session. */ - if (WT_SESSION_FIRST_USE(session_ret)) - WT_ERR(__wt_calloc_def( - session, conn->hazard_max, &session_ret->hazard)); - - /* - * Set an initial size for the hazard array. It will be grown as - * required up to hazard_max. The hazard_size is reset on close, since - * __wt_hazard_close ensures the array is cleared - so it is safe to - * reset the starting size on each open. - */ - session_ret->hazard_size = 0; + if (WT_SESSION_FIRST_USE(session_ret)) { + WT_ERR(__wt_calloc_def(session, + WT_SESSION_INITIAL_HAZARD_SLOTS, &session_ret->hazard)); + session_ret->hazard_size = WT_SESSION_INITIAL_HAZARD_SLOTS; + session_ret->hazard_inuse = 0; + session_ret->nhazard = 0; + } /* Cache the offset of this session's statistics bucket. */ session_ret->stat_bucket = WT_STATS_SLOT_ID(session); diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 66635007723..72c072e0fb8 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -53,11 +53,14 @@ * blocks, it can't easily know this is the case, and so we'll waste a lot of * effort trying to compact files that can't be compacted. * - * Now, to the actual process. First, we checkpoint the high-level object - * (which is potentially composed of multiple files): there are potentially - * many dirty blocks in the cache, and we want to write them out and then - * discard previous checkpoints so we have as many blocks as possible on the - * file's "available for reuse" list when we start compaction. + * Finally, compaction checkpoints are database-wide, otherwise we can corrupt + * file relationships, for example, an index checkpointed by compaction could + * be out of sync with the primary after a crash. + * + * Now, to the actual process. First, we checkpoint the database: there are + * potentially many dirty blocks in the cache, and we want to write them out + * and then discard previous checkpoints so we have as many blocks as possible + * on the file's "available for reuse" list when we start compaction. * * Then, we compact the high-level object. * @@ -96,29 +99,6 @@ * towards the beginning of the file, and then the file can be truncated. */ -/* - * __compact_uri_analyze -- - * Extract information relevant to deciding what work compact needs to - * do from a URI that is part of a table schema. - * Called via the schema_worker function. - */ -static int -__compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) -{ - /* - * Add references to schema URI objects to the list of objects to be - * compacted. Skip over LSM trees or we will get false positives on - * the "file:" URIs for the chunks. - */ - if (WT_PREFIX_MATCH(uri, "lsm:")) { - session->compact->lsm_count++; - *skipp = true; - } else if (WT_PREFIX_MATCH(uri, "file:")) - session->compact->file_count++; - - return (0); -} - /* * __compact_start -- * Start object compaction. @@ -145,6 +125,29 @@ __compact_end(WT_SESSION_IMPL *session) return (bm->compact_end(bm, session)); } +/* + * __compact_uri_analyze -- + * Extract information relevant to deciding what work compact needs to + * do from a URI that is part of a table schema. + * Called via the schema_worker function. + */ +static int +__compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) +{ + /* + * Add references to schema URI objects to the list of objects to be + * compacted. Skip over LSM trees or we will get false positives on + * the "file:" URIs for the chunks. + */ + if (WT_PREFIX_MATCH(uri, "lsm:")) { + session->compact->lsm_count++; + *skipp = true; + } else if (WT_PREFIX_MATCH(uri, "file:")) + session->compact->file_count++; + + return (0); +} + /* * __compact_handle_append -- * Gather a file handle to be compacted. @@ -157,10 +160,6 @@ __compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); - /* Make sure there is space for the next entry. */ - WT_RET(__wt_realloc_def(session, &session->op_handle_allocated, - session->op_handle_next + 1, &session->op_handle)); - WT_RET(__wt_session_get_btree( session, session->dhandle->name, NULL, NULL, 0)); @@ -170,16 +169,20 @@ __compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) return (ret); } + /* Make sure there is space for the next entry. */ + WT_RET(__wt_realloc_def(session, &session->op_handle_allocated, + session->op_handle_next + 1, &session->op_handle)); + session->op_handle[session->op_handle_next++] = session->dhandle; return (0); } /* - * __session_compact_check_timeout -- + * __wt_session_compact_check_timeout -- * Check if the timeout has been exceeded. */ -static int -__session_compact_check_timeout(WT_SESSION_IMPL *session, struct timespec begin) +int +__wt_session_compact_check_timeout(WT_SESSION_IMPL *session) { struct timespec end; @@ -187,64 +190,116 @@ __session_compact_check_timeout(WT_SESSION_IMPL *session, struct timespec begin) return (0); __wt_epoch(session, &end); - if (session->compact->max_time < WT_TIMEDIFF_SEC(end, begin)) - return (ETIMEDOUT); - return (0); + return (session->compact->max_time > + WT_TIMEDIFF_SEC(end, session->compact->begin) ? 0 : ETIMEDOUT); } /* - * __compact_file -- - * Function to alternate between checkpoints and compaction calls. + * __compact_checkpoint -- + * Perform a checkpoint for compaction. */ static int -__compact_file(WT_SESSION_IMPL *session, const char *cfg[]) +__compact_checkpoint(WT_SESSION_IMPL *session) { - struct timespec start_time; - WT_DATA_HANDLE *dhandle; - WT_DECL_ITEM(t); + WT_TXN_GLOBAL *txn_global; WT_DECL_RET; - int i; + uint64_t txn_gen; + + /* + * Force compaction checkpoints: we don't want to skip it because the + * work we need to have done is done in the underlying block manager. + */ const char *checkpoint_cfg[] = { - WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL, NULL }; + WT_CONFIG_BASE(session, WT_SESSION_checkpoint), "force=1", NULL }; + + /* Checkpoints take a lot of time, check if we've run out. */ + WT_RET(__wt_session_compact_check_timeout(session)); + + if ((ret = __wt_txn_checkpoint(session, checkpoint_cfg, false)) == 0) + return (0); + WT_RET_BUSY_OK(ret); + + /* + * If there's a checkpoint running, wait for it to complete, checking if + * we're out of time. If there's no checkpoint running or the checkpoint + * generation number changes, the checkpoint blocking us has completed. + */ + txn_global = &S2C(session)->txn_global; + for (txn_gen = txn_global->checkpoint_gen;;) { + WT_READ_BARRIER(); + if (!txn_global->checkpoint_running || + txn_gen != txn_global->checkpoint_gen) + break; + + WT_RET(__wt_session_compact_check_timeout(session)); + __wt_sleep(2, 0); + } + + return (0); +} - dhandle = session->dhandle; +/* + * __compact_worker -- + * Function to alternate between checkpoints and compaction calls. + */ +static int +__compact_worker(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + u_int i, loop; + bool didwork; /* - * Force the checkpoint: we don't want to skip it because the work we - * need to have done is done in the underlying block manager. + * Reset the handles' compaction skip flag (we don't bother setting + * or resetting it when we finish compaction, it's simpler to do it + * once, here). */ - WT_ERR(__wt_scr_alloc(session, 128, &t)); - WT_ERR(__wt_buf_fmt( - session, t, "target=(\"%s\"),force=1", dhandle->name)); - checkpoint_cfg[1] = t->data; + for (i = 0; i < session->op_handle_next; ++i) + session->op_handle[i]->compact_skip = false; - __wt_epoch(session, &start_time); + /* + * Perform an initial checkpoint (see this file's leading comment for + * details). + */ + WT_ERR(__compact_checkpoint(session)); /* - * We compact 10% of the file on each pass (but the overall size of the + * We compact 10% of a file on each pass (but the overall size of the * file is decreasing each time, so we're not compacting 10% of the * original file each time). Try 100 times (which is clearly more than - * we need); quit if we make no progress and check for a timeout each - * time through the loop. + * we need); quit if we make no progress. */ - for (i = 0; i < 100; ++i) { - WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); - - session->compact_state = WT_COMPACT_RUNNING; - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_compact(session, cfg)); - WT_ERR(ret); - if (session->compact_state != WT_COMPACT_SUCCESS) + for (loop = 0; loop < 100; ++loop) { + /* Step through the list of files being compacted. */ + for (didwork = false, i = 0; i < session->op_handle_next; ++i) { + /* Skip objects where there's no more work. */ + if (session->op_handle[i]->compact_skip) + continue; + + session->compact_state = WT_COMPACT_RUNNING; + WT_WITH_DHANDLE(session, + session->op_handle[i], ret = __wt_compact(session)); + WT_ERR(ret); + + /* If we did no work, skip this file in the future. */ + if (session->compact_state == WT_COMPACT_SUCCESS) + didwork = true; + else + session->op_handle[i]->compact_skip = true; + } + if (!didwork) break; - WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); - WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); - WT_ERR(__session_compact_check_timeout(session, start_time)); + /* + * Perform two checkpoints (see this file's leading comment for + * details). + */ + WT_ERR(__compact_checkpoint(session)); + WT_ERR(__compact_checkpoint(session)); } err: session->compact_state = WT_COMPACT_NONE; - __wt_scr_free(session, &t); return (ret); } @@ -256,8 +311,9 @@ int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) { - WT_COMPACT compact; + WT_COMPACT_STATE compact; WT_CONFIG_ITEM cval; + WT_DATA_SOURCE *dsrc; WT_DECL_RET; WT_SESSION_IMPL *session; WT_TXN *txn; @@ -270,6 +326,17 @@ __wt_session_compact( if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) goto err; + /* + * Non-LSM object compaction requires checkpoints, which are impossible + * in transactional contexts. Disallow in all contexts (there's no + * reason for LSM to allow this, possible or not), and check now so the + * error message isn't confusing. + */ + txn = &session->txn; + if (F_ISSET(txn, WT_TXN_RUNNING)) + WT_ERR_MSG(session, EINVAL, + "compaction not permitted in a transaction"); + /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); @@ -277,18 +344,28 @@ __wt_session_compact( !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "lsm:") && - !WT_PREFIX_MATCH(uri, "table:")) - WT_ERR(__wt_bad_object_type(session, uri)); + !WT_PREFIX_MATCH(uri, "table:")) { + if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) + ret = dsrc->compact == NULL ? + __wt_object_unsupported(session, uri) : + dsrc->compact( + dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg); + else + ret = __wt_bad_object_type(session, uri); + goto err; + } - /* Setup the structure in the session handle */ - memset(&compact, 0, sizeof(WT_COMPACT)); + /* Setup the session handle's compaction state structure. */ + memset(&compact, 0, sizeof(WT_COMPACT_STATE)); session->compact = &compact; + /* Compaction can be time-limited. */ WT_ERR(__wt_config_gets(session, cfg, "timeout", &cval)); session->compact->max_time = (uint64_t)cval.val; + __wt_epoch(session, &session->compact->begin); /* Find the types of data sources being compacted. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __compact_handle_append, __compact_uri_analyze, cfg, 0)); WT_ERR(ret); @@ -296,23 +373,8 @@ __wt_session_compact( if (session->compact->lsm_count != 0) WT_ERR(__wt_schema_worker( session, uri, NULL, __wt_lsm_compact, cfg, 0)); - if (session->compact->file_count != 0) { - /* - * File compaction requires checkpoints, which will fail in a - * transactional context. Check now so the error message isn't - * confusing. - */ - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, - " File compaction not permitted in a transaction"); - - for (i = 0; i < session->op_handle_next; ++i) { - WT_WITH_DHANDLE(session, session->op_handle[i], - ret = __compact_file(session, cfg)); - WT_ERR(ret); - } - } + if (session->compact->file_count != 0) + WT_ERR(__compact_worker(session)); err: session->compact = NULL; diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 725854c6001..ffeb6137766 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -44,8 +44,7 @@ __session_discard_dhandle( TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); - (void)__wt_atomic_sub32(&dhandle_cache->dhandle->session_ref, 1); - + WT_DHANDLE_RELEASE(dhandle_cache->dhandle); __wt_overwrite_and_free(session, dhandle_cache); } @@ -68,7 +67,7 @@ __session_find_dhandle(WT_SESSION_IMPL *session, retry: TAILQ_FOREACH(dhandle_cache, &session->dhhash[bucket], hashq) { dhandle = dhandle_cache->dhandle; if (WT_DHANDLE_INACTIVE(dhandle) && - !WT_IS_METADATA(session, dhandle)) { + !WT_IS_METADATA(dhandle)) { __session_discard_dhandle(session, dhandle_cache); /* We deleted our entry, retry from the start. */ goto retry; @@ -181,17 +180,17 @@ __wt_session_lock_dhandle( */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && (!want_exclusive || lock_busy)) { - __wt_readlock(session, dhandle->rwlock); + __wt_readlock(session, &dhandle->rwlock); if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) { *is_deadp = 1; - __wt_readunlock(session, dhandle->rwlock); + __wt_readunlock(session, &dhandle->rwlock); return (0); } is_open = F_ISSET(dhandle, WT_DHANDLE_OPEN); if (is_open && !want_exclusive) return (0); - __wt_readunlock(session, dhandle->rwlock); + __wt_readunlock(session, &dhandle->rwlock); } else is_open = false; @@ -201,10 +200,11 @@ __wt_session_lock_dhandle( * with another thread that successfully opens the file, we * don't want to block waiting to get exclusive access. */ - if ((ret = __wt_try_writelock(session, dhandle->rwlock)) == 0) { + if ((ret = + __wt_try_writelock(session, &dhandle->rwlock)) == 0) { if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) { *is_deadp = 1; - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); return (0); } @@ -215,7 +215,7 @@ __wt_session_lock_dhandle( if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && !want_exclusive) { lock_busy = false; - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); continue; } @@ -229,7 +229,8 @@ __wt_session_lock_dhandle( WT_ASSERT(session, !F_ISSET(dhandle, WT_DHANDLE_DEAD)); return (0); } - if (ret != EBUSY || (is_open && want_exclusive)) + if (ret != EBUSY || (is_open && want_exclusive) || + LF_ISSET(WT_DHANDLE_LOCK_ONLY)) return (ret); lock_busy = true; @@ -261,8 +262,8 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) * can get a handle without special flags. */ if (F_ISSET(dhandle, WT_DHANDLE_DISCARD | WT_DHANDLE_DISCARD_FORCE)) { - __session_find_dhandle(session, - dhandle->name, dhandle->checkpoint, &dhandle_cache); + WT_SAVE_DHANDLE(session, __session_find_dhandle(session, + dhandle->name, dhandle->checkpoint, &dhandle_cache)); if (dhandle_cache != NULL) __session_discard_dhandle(session, dhandle_cache); } @@ -270,6 +271,16 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) if (F_ISSET(dhandle, WT_DHANDLE_DISCARD_FORCE)) { ret = __wt_conn_btree_sync_and_close(session, false, true); F_CLR(dhandle, WT_DHANDLE_DISCARD_FORCE); + } else if (F_ISSET(btree, WT_BTREE_BULK)) { + WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && + !F_ISSET(dhandle, WT_DHANDLE_DISCARD)); + /* + * Acquire the schema lock while completing a bulk load. This + * avoids racing with a checkpoint while it gathers a set + * of handles. + */ + WT_WITH_SCHEMA_LOCK(session, ret = + __wt_conn_btree_sync_and_close(session, false, false)); } else if (F_ISSET(dhandle, WT_DHANDLE_DISCARD) || F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) { WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE)); @@ -286,9 +297,9 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) if (locked) { if (write_locked) { F_CLR(dhandle, WT_DHANDLE_EXCLUSIVE); - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); } else - __wt_readunlock(session, dhandle->rwlock); + __wt_readunlock(session, &dhandle->rwlock); } session->dhandle = NULL; @@ -401,7 +412,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) difftime(now, dhandle->timeofdeath) > conn->sweep_idle_time))) { WT_STAT_CONN_INCR(session, dh_session_handles); - WT_ASSERT(session, !WT_IS_METADATA(session, dhandle)); + WT_ASSERT(session, !WT_IS_METADATA(dhandle)); __session_discard_dhandle(session, dhandle_cache); } dhandle_cache = dhandle_cache_next; @@ -411,17 +422,27 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) /* * __session_find_shared_dhandle -- * Search for a data handle in the connection and add it to a session's - * cache. Since the data handle isn't locked, this must be called holding - * the handle list lock, and we must increment the handle's reference - * count before releasing it. + * cache. We must increment the handle's reference count while holding + * the handle list lock. */ static int __session_find_shared_dhandle( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) { - WT_RET(__wt_conn_dhandle_find(session, uri, checkpoint)); - (void)__wt_atomic_add32(&session->dhandle->session_ref, 1); - return (0); + WT_DECL_RET; + + WT_WITH_HANDLE_LIST_READ_LOCK(session, + if ((ret = __wt_conn_dhandle_find(session, uri, checkpoint)) == 0) + WT_DHANDLE_ACQUIRE(session->dhandle)); + + if (ret != WT_NOTFOUND) + return (ret); + + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + if ((ret = __wt_conn_dhandle_alloc(session, uri, checkpoint)) == 0) + WT_DHANDLE_ACQUIRE(session->dhandle)); + + return (ret); } /* @@ -449,16 +470,16 @@ __session_get_dhandle( * We didn't find a match in the session cache, search the shared * handle list and cache the handle we find. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __session_find_shared_dhandle(session, uri, checkpoint)); - WT_RET(ret); + WT_RET(__session_find_shared_dhandle(session, uri, checkpoint)); /* * Fixup the reference count on failure (we incremented the reference * count while holding the handle-list lock). */ - if ((ret = __session_add_dhandle(session)) != 0) - (void)__wt_atomic_sub32(&session->dhandle->session_ref, 1); + if ((ret = __session_add_dhandle(session)) != 0) { + WT_DHANDLE_RELEASE(session->dhandle); + session->dhandle = NULL; + } return (ret); } @@ -504,17 +525,15 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, * reopen handles in the meantime. A combination of the schema * and handle list locks are used to enforce this. */ - if (!F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || - !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)) { + if (!F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)) { dhandle->excl_session = NULL; dhandle->excl_ref = 0; F_CLR(dhandle, WT_DHANDLE_EXCLUSIVE); - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_session_get_btree( - session, uri, checkpoint, cfg, flags))); + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_session_get_btree( + session, uri, checkpoint, cfg, flags)); return (ret); } @@ -531,7 +550,7 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, dhandle->excl_session = NULL; dhandle->excl_ref = 0; F_CLR(dhandle, WT_DHANDLE_EXCLUSIVE); - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); WT_RET(ret); } @@ -552,7 +571,7 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) { - WT_DATA_HANDLE *dhandle, *saved_dhandle; + WT_DATA_HANDLE *saved_dhandle; WT_DECL_RET; WT_ASSERT(session, WT_META_TRACKING(session)); @@ -560,31 +579,33 @@ __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) /* * Get the checkpoint handle exclusive, so no one else can access it - * while we are creating the new checkpoint. + * while we are creating the new checkpoint. Hold the lock until the + * checkpoint completes. */ WT_ERR(__wt_session_get_btree(session, saved_dhandle->name, checkpoint, NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); + if ((ret = __wt_meta_track_handle_lock(session, false)) != 0) { + WT_TRET(__wt_session_release_btree(session)); + goto err; + } /* - * Flush any pages in this checkpoint from the cache (we are about to - * re-write the checkpoint which will mean cached pages no longer have - * valid contents). This is especially noticeable with memory mapped - * files, since changes to the underlying file are visible to the in - * memory pages. + * Get exclusive access to the handle and then flush any pages in this + * checkpoint from the cache (we are about to re-write the checkpoint + * which will mean cached pages no longer have valid contents). This + * is especially noticeable with memory mapped files, since changes to + * the underlying file are visible to the in-memory pages. */ + WT_ERR(__wt_evict_file_exclusive_on(session)); WT_ERR(__wt_cache_op(session, WT_SYNC_DISCARD)); /* * We lock checkpoint handles that we are overwriting, so the handle * must be closed when we release it. */ - dhandle = session->dhandle; - F_SET(dhandle, WT_DHANDLE_DISCARD); - - WT_ERR(__wt_meta_track_handle_lock(session, false)); + F_SET(session->dhandle, WT_DHANDLE_DISCARD); - /* Restore the original btree in the session. */ + /* Restore the original data handle in the session. */ err: session->dhandle = saved_dhandle; - return (ret); } diff --git a/src/third_party/wiredtiger/src/session/session_salvage.c b/src/third_party/wiredtiger/src/session/session_salvage.c index 983b28dd8ea..12ce71cdbb0 100644 --- a/src/third_party/wiredtiger/src/session/session_salvage.c +++ b/src/third_party/wiredtiger/src/session/session_salvage.c @@ -54,6 +54,6 @@ __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_meta_ckptlist_set( session, dhandle->name, ckptbase, NULL)); -err: __wt_meta_ckptlist_free(session, ckptbase); +err: __wt_meta_ckptlist_free(session, &ckptbase); return (ret); } diff --git a/src/third_party/wiredtiger/src/support/cond_auto.c b/src/third_party/wiredtiger/src/support/cond_auto.c index a3ae67f5baa..600e5eab0ff 100644 --- a/src/third_party/wiredtiger/src/support/cond_auto.c +++ b/src/third_party/wiredtiger/src/support/cond_auto.c @@ -1,29 +1,9 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. + * See the file LICENSE for redistribution information. */ #include "wt_internal.h" @@ -38,13 +18,12 @@ * Allocate and initialize an automatically adjusting condition variable. */ int -__wt_cond_auto_alloc( - WT_SESSION_IMPL *session, const char *name, - bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp) +__wt_cond_auto_alloc(WT_SESSION_IMPL *session, + const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) { WT_CONDVAR *cond; - WT_RET(__wt_cond_alloc(session, name, is_signalled, condp)); + WT_RET(__wt_cond_alloc(session, name, condp)); cond = *condp; cond->min_wait = min; @@ -54,34 +33,20 @@ __wt_cond_auto_alloc( return (0); } -/* - * __wt_cond_auto_signal -- - * Signal a condition variable. - */ -void -__wt_cond_auto_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) -{ - - WT_ASSERT(session, cond->min_wait != 0); - __wt_cond_signal(session, cond); -} - /* * __wt_cond_auto_wait_signal -- * Wait on a mutex, optionally timing out. If we get it before the time * out period expires, let the caller know. - * TODO: Can this version of the API be removed, now that we have the - * auto adjusting condition variables? */ void -__wt_cond_auto_wait_signal( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled) +__wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, + bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) { uint64_t delta; /* * Catch cases where this function is called with a condition variable - * that was initialized non-auto. + * that wasn't initialized to do automatic adjustments. */ WT_ASSERT(session, cond->min_wait != 0); @@ -94,7 +59,8 @@ __wt_cond_auto_wait_signal( cond->max_wait, cond->prev_wait + delta); } - __wt_cond_wait_signal(session, cond, cond->prev_wait, signalled); + __wt_cond_wait_signal( + session, cond, cond->prev_wait, run_func, signalled); if (progress || *signalled) WT_STAT_CONN_INCR(session, cond_auto_wait_reset); @@ -108,24 +74,10 @@ __wt_cond_auto_wait_signal( * out period expires, let the caller know. */ void -__wt_cond_auto_wait( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress) +__wt_cond_auto_wait(WT_SESSION_IMPL *session, + WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)) { - bool signalled; - - /* - * Call the signal version so the wait period is reset if the - * condition is woken explicitly. - */ - __wt_cond_auto_wait_signal(session, cond, progress, &signalled); -} + bool notused; -/* - * __wt_cond_auto_destroy -- - * Destroy a condition variable. - */ -int -__wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) -{ - return (__wt_cond_destroy(session, condp)); + __wt_cond_auto_wait_signal(session, cond, progress, run_func, ¬used); } diff --git a/src/third_party/wiredtiger/src/support/crypto.c b/src/third_party/wiredtiger/src/support/crypto.c index ab94ec2c829..cce0d228832 100644 --- a/src/third_party/wiredtiger/src/support/crypto.c +++ b/src/third_party/wiredtiger/src/support/crypto.c @@ -133,5 +133,4 @@ __wt_encrypt_size(WT_SESSION_IMPL *session, return; *sizep = incoming_size + kencryptor->size_const + WT_ENCRYPT_LEN_SIZE; - return; } diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index e48225c2c13..57efde72b23 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -102,9 +102,10 @@ __handler_failure(WT_SESSION_IMPL *session, */ char s[256]; - (void)snprintf(s, sizeof(s), + if (__wt_snprintf(s, sizeof(s), "application %s event handler failed: %s", - which, __wt_strerror(session, error, NULL, 0)); + which, __wt_strerror(session, error, NULL, 0)) != 0) + return; /* * Use the error handler to report the failure, unless it was the error @@ -148,6 +149,23 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) session->event_handler = handler; } +#define WT_ERROR_APPEND(p, remain, ...) do { \ + size_t __len; \ + WT_ERR(__wt_snprintf_len_set(p, remain, &__len, __VA_ARGS__)); \ + if (__len > remain) \ + __len = remain; \ + p += __len; \ + remain -= __len; \ +} while (0) +#define WT_ERROR_APPEND_AP(p, remain, ...) do { \ + size_t __len; \ + WT_ERR(__wt_vsnprintf_len_set(p, remain, &__len, __VA_ARGS__)); \ + if (__len > remain) \ + __len = remain; \ + p += __len; \ + remain -= __len; \ +} while (0) + /* * __wt_eventv -- * Report a message to an event handler. @@ -161,9 +179,9 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, WT_DECL_RET; WT_SESSION *wt_session; struct timespec ts; - size_t len, remain, wlen; + size_t len, remain; const char *err, *prefix; - char *end, *p, tid[128]; + char *p, tid[128]; /* * We're using a stack buffer because we want error messages no matter @@ -174,6 +192,8 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * Buffer placed at the end of the stack in case snprintf overflows. */ char s[2048]; + p = s; + remain = sizeof(s); /* * !!! @@ -185,24 +205,8 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * first session, but if the allocation of the first session fails, for * example, we can end up here without a session.) */ - if (session == NULL) { - if (fprintf(stderr, - "WiredTiger Error%s%s: ", - error == 0 ? "" : ": ", - error == 0 ? "" : - __wt_strerror(session, error, NULL, 0)) < 0) - ret = EIO; - if (vfprintf(stderr, fmt, ap) < 0) - ret = EIO; - if (fprintf(stderr, "\n") < 0) - ret = EIO; - if (fflush(stderr) != 0) - ret = EIO; - return (ret); - } - - p = s; - end = s + sizeof(s); + if (session == NULL) + goto err; /* * We have several prefixes for the error message: a timestamp and the @@ -211,42 +215,24 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * followed by a colon. */ __wt_epoch(session, &ts); - __wt_thread_id(tid, sizeof(tid)); - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, "[%" PRIuMAX ":%" PRIuMAX "][%s]", + WT_ERR(__wt_thread_id(tid, sizeof(tid))); + WT_ERROR_APPEND(p, remain, + "[%" PRIuMAX ":%" PRIuMAX "][%s]", (uintmax_t)ts.tv_sec, (uintmax_t)ts.tv_nsec / WT_THOUSAND, tid); - p = wlen >= remain ? end : p + wlen; - if ((prefix = S2C(session)->error_prefix) != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ", %s", prefix); - p = wlen >= remain ? end : p + wlen; - } + if ((prefix = S2C(session)->error_prefix) != NULL) + WT_ERROR_APPEND(p, remain, ", %s", prefix); prefix = session->dhandle == NULL ? NULL : session->dhandle->name; - if (prefix != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ", %s", prefix); - p = wlen >= remain ? end : p + wlen; - } - if ((prefix = session->name) != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ", %s", prefix); - p = wlen >= remain ? end : p + wlen; - } - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ": "); - p = wlen >= remain ? end : p + wlen; - - if (file_name != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t) - snprintf(p, remain, "%s, %d: ", file_name, line_number); - p = wlen >= remain ? end : p + wlen; - } + if (prefix != NULL) + WT_ERROR_APPEND(p, remain, ", %s", prefix); + if ((prefix = session->name) != NULL) + WT_ERROR_APPEND(p, remain, ", %s", prefix); + WT_ERROR_APPEND(p, remain, ": "); + + if (file_name != NULL) + WT_ERROR_APPEND(p, remain, "%s, %d: ", file_name, line_number); - remain = WT_PTRDIFF(end, p); - wlen = (size_t)vsnprintf(p, remain, fmt, ap); - p = wlen >= remain ? end : p + wlen; + WT_ERROR_APPEND_AP(p, remain, fmt, ap); if (error != 0) { /* @@ -261,10 +247,8 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, */ err = __wt_strerror(session, error, NULL, 0); len = strlen(err); - if (WT_PTRDIFF(p, s) < len || strcmp(p - len, err) != 0) { - remain = WT_PTRDIFF(end, p); - (void)snprintf(p, remain, ": %s", err); - } + if (WT_PTRDIFF(p, s) < len || strcmp(p - len, err) != 0) + WT_ERROR_APPEND(p, remain, ": %s", err); } /* @@ -279,7 +263,7 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * * If an application-specified error message handler fails, complain * using the default error handler. If the default error handler fails, - * there's nothing to do. + * fallback to stderr. */ wt_session = (WT_SESSION *)session; handler = session->event_handler; @@ -293,6 +277,21 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, __handler_failure(session, ret, "error", true); } + if (ret != 0) { +err: if (fprintf(stderr, + "WiredTiger Error%s%s: ", + error == 0 ? "" : ": ", + error == 0 ? "" : + __wt_strerror(session, error, NULL, 0)) < 0) + WT_TRET(EIO); + if (vfprintf(stderr, fmt, ap) < 0) + WT_TRET(EIO); + if (fprintf(stderr, "\n") < 0) + WT_TRET(EIO); + if (fflush(stderr) != 0) + WT_TRET(EIO); + } + return (ret); } @@ -304,6 +303,7 @@ void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -375,7 +375,7 @@ info_msg(WT_SESSION_IMPL *session, const char *fmt, va_list ap) */ char s[2048]; - (void)vsnprintf(s, sizeof(s), fmt, ap); + WT_RET(__wt_vsnprintf(s, sizeof(s), fmt, ap)); wt_session = (WT_SESSION *)session; handler = session->event_handler; @@ -470,6 +470,7 @@ __wt_assert(WT_SESSION_IMPL *session, #ifdef HAVE_DIAGNOSTIC WT_GCC_FUNC_ATTRIBUTE((noreturn)) #endif + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -491,6 +492,7 @@ __wt_assert(WT_SESSION_IMPL *session, int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { F_SET(S2C(session), WT_CONN_PANIC); __wt_err(session, WT_PANIC, "the process must exit and restart"); @@ -515,6 +517,7 @@ __wt_panic(WT_SESSION_IMPL *session) int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { __wt_errx(session, "%s%s%s", name == NULL ? "" : name, name == NULL ? "" : ": ", diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index b1978dbe97a..7e88ad183fe 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -12,6 +12,48 @@ static void __hazard_dump(WT_SESSION_IMPL *); #endif +/* + * hazard_grow -- + * Grow a hazard pointer array. + */ +static int +hazard_grow(WT_SESSION_IMPL *session) +{ + WT_HAZARD *nhazard; + size_t size; + void *ohazard; + + /* + * Allocate a new, larger hazard pointer array and copy the contents of + * the original into place. + */ + size = session->hazard_size; + WT_RET(__wt_calloc_def(session, size * 2, &nhazard)); + memcpy(nhazard, session->hazard, size * sizeof(WT_HAZARD)); + + /* + * Swap the new hazard pointer array into place after initialization + * is complete (initialization must complete before eviction can see + * the new hazard pointer array), then schedule the original to be + * freed. + */ + ohazard = session->hazard; + WT_PUBLISH(session->hazard, nhazard); + + __wt_spin_lock(session, &S2C(session)->api_lock); + __wt_conn_foc_add(session, ohazard); + __wt_spin_unlock(session, &S2C(session)->api_lock); + + /* + * Increase the size of the session's pointer array after swapping it + * into place (the session's reference must be updated before eviction + * can see the new size). + */ + WT_PUBLISH(session->hazard_size, (uint32_t)(size * 2)); + + return (0); +} + /* * __wt_hazard_set -- * Set a hazard pointer. @@ -23,18 +65,62 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #endif ) { - WT_BTREE *btree; - WT_CONNECTION_IMPL *conn; WT_HAZARD *hp; - int restarts = 0; - btree = S2BT(session); - conn = S2C(session); *busyp = false; /* If a file can never be evicted, hazard pointers aren't required. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY)) + return (0); + + /* + * If there isn't a valid page, we're done. This read can race with + * eviction and splits, we re-check it after a barrier to make sure + * we have a valid reference. + */ + if (ref->state != WT_REF_MEM) { + *busyp = true; return (0); + } + + /* If we have filled the current hazard pointer array, grow it. */ + if (session->nhazard >= session->hazard_size) { + WT_ASSERT(session, + session->nhazard == session->hazard_size && + session->hazard_inuse == session->hazard_size); + WT_RET(hazard_grow(session)); + } + + /* + * If there are no available hazard pointer slots, make another one + * visible. + */ + if (session->nhazard >= session->hazard_inuse) { + WT_ASSERT(session, + session->nhazard == session->hazard_inuse && + session->hazard_inuse < session->hazard_size); + hp = &session->hazard[session->hazard_inuse++]; + } else { + WT_ASSERT(session, + session->nhazard < session->hazard_inuse && + session->hazard_inuse <= session->hazard_size); + + /* + * There must be an empty slot in the array, find it. Skip most + * of the active slots by starting after the active count slot; + * there may be a free slot before there, but checking is + * expensive. If we reach the end of the array, continue the + * search from the beginning of the array. + */ + for (hp = session->hazard + session->nhazard;; ++hp) { + if (hp >= session->hazard + session->hazard_inuse) + hp = session->hazard; + if (hp->ref == NULL) + break; + } + } + + WT_ASSERT(session, hp->ref == NULL); /* * Do the dance: @@ -49,83 +135,43 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * pointer before it discards the page (the eviction server sets the * state to WT_REF_LOCKED, then flushes memory and checks the hazard * pointers). - * - * For sessions with many active hazard pointers, skip most of the - * active slots: there may be a free slot in there, but checking is - * expensive. Most hazard pointers are released quickly: optimize - * for that case. */ - for (hp = session->hazard + session->nhazard;; ++hp) { - /* - * If we get to the end of the array, either: - * 1. If we know there are free slots somewhere, and this is - * the first time through, continue the search from the - * start. Don't actually continue the loop because that - * will skip the first slot. - * 2. If we have searched all the way through and we have - * allocated the maximum number of slots, give up. - * 3. Allocate another increment of slots, up to the maximum. - * The slot we are on should now be available. - */ - if (hp >= session->hazard + session->hazard_size) { - if (session->nhazard < session->hazard_size && - restarts++ == 0) - hp = session->hazard; - else if (session->hazard_size >= conn->hazard_max) - break; - else - WT_PUBLISH(session->hazard_size, WT_MIN( - session->hazard_size + WT_HAZARD_INCR, - conn->hazard_max)); - } - - if (hp->page != NULL) - continue; - - hp->page = ref->page; + hp->ref = ref; #ifdef HAVE_DIAGNOSTIC - hp->file = file; - hp->line = line; + hp->file = file; + hp->line = line; #endif - /* Publish the hazard pointer before reading page's state. */ - WT_FULL_BARRIER(); + /* Publish the hazard pointer before reading page's state. */ + WT_FULL_BARRIER(); - /* - * Check if the page state is still valid, where valid means a - * state of WT_REF_MEM and the pointer is unchanged. (The - * pointer can change, it means the page was evicted between - * the time we set our hazard pointer and the publication. It - * would theoretically be possible for the page to be evicted - * and a different page read into the same memory, so the - * pointer hasn't changed but the contents have. That's OK, we - * found this page using the tree's key space, whatever page we - * find here is the page for us to use.) - */ - if (ref->page == hp->page && ref->state == WT_REF_MEM) { - ++session->nhazard; - return (0); - } + /* + * Check if the page state is still valid, where valid means a + * state of WT_REF_MEM. + */ + if (ref->state == WT_REF_MEM) { + ++session->nhazard; /* - * The page isn't available, it's being considered for eviction - * (or being evicted, for all we know). If the eviction server - * sees our hazard pointer before evicting the page, it will - * return the page to use, no harm done, if it doesn't, it will - * go ahead and complete the eviction. - * - * We don't bother publishing this update: the worst case is we - * prevent some random page from being evicted. + * Callers require a barrier here so operations holding + * the hazard pointer see consistent data. */ - hp->page = NULL; - *busyp = true; + WT_READ_BARRIER(); return (0); } -#ifdef HAVE_DIAGNOSTIC - __hazard_dump(session); -#endif - WT_RET_MSG(session, ENOMEM, - "session %p: hazard pointer table full", (void *)session); + /* + * The page isn't available, it's being considered for eviction + * (or being evicted, for all we know). If the eviction server + * sees our hazard pointer before evicting the page, it will + * return the page to use, no harm done, if it doesn't, it will + * go ahead and complete the eviction. + * + * We don't bother publishing this update: the worst case is we + * prevent some random page from being evicted. + */ + hp->ref = NULL; + *busyp = true; + return (0); } /* @@ -133,25 +179,22 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * Clear a hazard pointer. */ int -__wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) +__wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) { - WT_BTREE *btree; WT_HAZARD *hp; - btree = S2BT(session); - /* If a file can never be evicted, hazard pointers aren't required. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY)) return (0); /* * Clear the caller's hazard pointer. * The common pattern is LIFO, so do a reverse search. */ - for (hp = session->hazard + session->hazard_size - 1; + for (hp = session->hazard + session->hazard_inuse - 1; hp >= session->hazard; --hp) - if (hp->page == page) { + if (hp->ref == ref) { /* * We don't publish the hazard pointer clear in the * general case. It's not required for correctness; @@ -160,14 +203,18 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) * generation number was just set, it's unlikely the * page will be selected for eviction. */ - hp->page = NULL; + hp->ref = NULL; /* * If this was the last hazard pointer in the session, * reset the size so that checks can skip this session. + * + * A write-barrier() is necessary before the change to + * the in-use value, the number of active references + * can never be less than the number of in-use slots. */ if (--session->nhazard == 0) - WT_PUBLISH(session->hazard_size, 0); + WT_PUBLISH(session->hazard_inuse, 0); return (0); } @@ -177,7 +224,7 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) */ WT_PANIC_RET(session, EINVAL, "session %p: clear hazard pointer: %p: not found", - (void *)session, (void *)page); + (void *)session, (void *)ref); } /* @@ -196,8 +243,8 @@ __wt_hazard_close(WT_SESSION_IMPL *session) * diagnostic. */ for (found = false, hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) - if (hp->page != NULL) { + hp < session->hazard + session->hazard_inuse; ++hp) + if (hp->ref != NULL) { found = true; break; } @@ -224,9 +271,9 @@ __wt_hazard_close(WT_SESSION_IMPL *session) * can't think of a reason it would be). */ for (hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) - if (hp->page != NULL) { - hp->page = NULL; + hp < session->hazard + session->hazard_inuse; ++hp) + if (hp->ref != NULL) { + hp->ref = NULL; --session->nhazard; } @@ -237,20 +284,95 @@ __wt_hazard_close(WT_SESSION_IMPL *session) (void *)session); } +/* + * hazard_get_reference -- + * Return a consistent reference to a hazard pointer array. + */ +static inline void +hazard_get_reference( + WT_SESSION_IMPL *session, WT_HAZARD **hazardp, uint32_t *hazard_inusep) +{ + /* + * Hazard pointer arrays can be swapped out from under us if they grow. + * First, read the current in-use value. The read must precede the read + * of the hazard pointer itself (so the in-use value is pessimistic + * should the hazard array grow), and additionally ensure we only read + * the in-use value once. Then, read the hazard pointer, also ensuring + * we only read it once. + * + * Use a barrier instead of marking the fields volatile because we don't + * want to slow down the rest of the hazard pointer functions that don't + * need special treatment. + */ + WT_ORDERED_READ(*hazard_inusep, session->hazard_inuse); + WT_ORDERED_READ(*hazardp, session->hazard); +} + +/* + * __wt_hazard_check -- + * Return if there's a hazard pointer to the page in the system. + */ +WT_HAZARD * +__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + WT_SESSION_IMPL *s; + uint32_t i, j, hazard_inuse, max, session_cnt, walk_cnt; + + conn = S2C(session); + + WT_STAT_CONN_INCR(session, cache_hazard_checks); + + /* + * No lock is required because the session array is fixed size, but it + * may contain inactive entries. We must review any active session + * that might contain a hazard pointer, so insert a read barrier after + * reading the active session count. That way, no matter what sessions + * come or go, we'll check the slots for all of the sessions that could + * have been active when we started our check. + */ + WT_ORDERED_READ(session_cnt, conn->session_cnt); + for (s = conn->sessions, + i = j = max = walk_cnt = 0; i < session_cnt; ++s, ++i) { + if (!s->active) + continue; + + hazard_get_reference(s, &hp, &hazard_inuse); + + if (hazard_inuse > max) { + max = hazard_inuse; + WT_STAT_CONN_SET(session, cache_hazard_max, max); + } + + for (j = 0; j < hazard_inuse; ++hp, ++j) { + ++walk_cnt; + if (hp->ref == ref) { + WT_STAT_CONN_INCRV(session, + cache_hazard_walks, walk_cnt); + return (hp); + } + } + } + WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); + return (NULL); +} + /* * __wt_hazard_count -- * Count how many hazard pointers this session has on the given page. */ u_int -__wt_hazard_count(WT_SESSION_IMPL *session, WT_PAGE *page) +__wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) { WT_HAZARD *hp; + uint32_t i, hazard_inuse; u_int count; - for (count = 0, hp = session->hazard + session->hazard_size - 1; - hp >= session->hazard; - --hp) - if (hp->page == page) + hazard_get_reference(session, &hp, &hazard_inuse); + + for (count = 0, i = 0; i < hazard_inuse; ++hp, ++i) + if (hp->ref == ref) ++count; return (count); @@ -267,11 +389,11 @@ __hazard_dump(WT_SESSION_IMPL *session) WT_HAZARD *hp; for (hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) - if (hp->page != NULL) + hp < session->hazard + session->hazard_inuse; ++hp) + if (hp->ref != NULL) __wt_errx(session, "session %p: hazard pointer %p: %s, line %d", (void *)session, - (void *)hp->page, hp->file, hp->line); + (void *)hp->ref, hp->file, hp->line); } #endif diff --git a/src/third_party/wiredtiger/src/support/hex.c b/src/third_party/wiredtiger/src/support/hex.c index 5c48ce8b74a..b54a08dd8f3 100644 --- a/src/third_party/wiredtiger/src/support/hex.c +++ b/src/third_party/wiredtiger/src/support/hex.c @@ -8,8 +8,6 @@ #include "wt_internal.h" -const u_char __wt_hex[] = "0123456789abcdef"; - /* * __fill_hex -- * In-memory conversion of raw bytes to a hexadecimal representation. @@ -25,8 +23,8 @@ __fill_hex(const uint8_t *src, size_t src_max, --dest_max; for (; src_max > 0 && dest_max > 1; src_max -= 1, dest_max -= 2, ++src) { - *dest++ = __wt_hex[(*src & 0xf0) >> 4]; - *dest++ = __wt_hex[*src & 0x0f]; + *dest++ = __wt_hex((*src & 0xf0) >> 4); + *dest++ = __wt_hex(*src & 0x0f); } *dest++ = '\0'; if (lenp != NULL) @@ -90,8 +88,8 @@ __wt_raw_to_esc_hex( *t++ = *p; } else { *t++ = '\\'; - *t++ = __wt_hex[(*p & 0xf0) >> 4]; - *t++ = __wt_hex[*p & 0x0f]; + *t++ = __wt_hex((*p & 0xf0) >> 4); + *t++ = __wt_hex(*p & 0x0f); } *t++ = '\0'; to->size = WT_PTRDIFF(t, to->mem); diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index ea18f556257..7905241be0e 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -27,7 +27,7 @@ */ /* - * Based on "Spinlocks and Read-Write Locks" by Dr. Steven Fuerst: + * Inspired by "Spinlocks and Read-Write Locks" by Dr. Steven Fuerst: * http://locklessinc.com/articles/locks/ * * Dr. Fuerst further credits: @@ -39,99 +39,75 @@ * by John Mellor-Crummey and Michael Scott in their landmark paper "Scalable * Reader-Writer Synchronization for Shared-Memory Multiprocessors". * - * The following is an explanation of this code. First, the underlying lock - * structure. + * The following is an explanation of our interpretation and implementation. + * First, the underlying lock structure. * + * volatile union { + * uint64_t v; // Full 64-bit value * struct { - * uint16_t writers; Now serving for writers - * uint16_t readers; Now serving for readers - * uint16_t next; Next available ticket number - * uint16_t __notused; Padding - * } + * uint8_t current; // Current ticket + * uint8_t next; // Next available ticket + * uint8_t reader; // Read queue ticket + * uint8_t __notused; // Padding + * uint16_t readers_active; // Count of active readers + * uint16_t readers_queued; // Count of queued readers + * } s; + * } u; * * First, imagine a store's 'take a number' ticket algorithm. A customer takes * a unique ticket number and customers are served in ticket order. In the data - * structure, 'writers' is the next writer to be served, 'readers' is the next - * reader to be served, and 'next' is the next available ticket number. + * structure, 'next' is the ticket that will be allocated next, and 'current' + * is the ticket being served. * - * Next, consider exclusive (write) locks. The 'now serving' number for writers - * is 'writers'. To lock, 'take a number' and wait until that number is being - * served; more specifically, atomically copy and increment the current value of - * 'next', and then wait until 'writers' equals that copied number. + * Next, consider exclusive (write) locks. To lock, 'take a number' and wait + * until that number is being served; more specifically, atomically increment + * 'next', and then wait until 'current' equals that allocated ticket. * - * Shared (read) locks are similar. Like writers, readers atomically get the - * next number available. However, instead of waiting for 'writers' to equal - * their number, they wait for 'readers' to equal their number. + * Shared (read) locks are similar, except that readers can share a ticket + * (both with each other and with a single writer). Readers with a given + * ticket execute before the writer with that ticket. In other words, writers + * wait for both their ticket to become current and for all readers to exit + * the lock. * - * This has the effect of queuing lock requests in the order they arrive - * (incidentally avoiding starvation). + * If there are no active writers (indicated by 'current' == 'next'), readers + * can immediately enter the lock by atomically incrementing 'readers_active'. + * When there are writers active, readers form a new queue by first setting + * 'reader' to 'next' (i.e. readers are scheduled after any queued writers, + * avoiding starvation), then atomically incrementing 'readers_queued'. * - * Each lock/unlock pair requires incrementing both 'readers' and 'writers'. - * In the case of a reader, the 'readers' increment happens when the reader - * acquires the lock (to allow read-lock sharing), and the 'writers' increment - * happens when the reader releases the lock. In the case of a writer, both - * 'readers' and 'writers' are incremented when the writer releases the lock. - * - * For example, consider the following read (R) and write (W) lock requests: - * - * writers readers next - * 0 0 0 - * R: ticket 0, readers match OK 0 1 1 - * R: ticket 1, readers match OK 0 2 2 - * R: ticket 2, readers match OK 0 3 3 - * W: ticket 3, writers no match block 0 3 4 - * R: ticket 2, unlock 1 3 4 - * R: ticket 0, unlock 2 3 4 - * R: ticket 1, unlock 3 3 4 - * W: ticket 3, writers match OK 3 3 4 - * - * Note the writer blocks until 'writers' equals its ticket number and it does - * not matter if readers unlock in order or not. - * - * Readers or writers entering the system after the write lock is queued block, - * and the next ticket holder (reader or writer) will unblock when the writer - * unlocks. An example, continuing from the last line of the above example: - * - * writers readers next - * W: ticket 3, writers match OK 3 3 4 - * R: ticket 4, readers no match block 3 3 5 - * R: ticket 5, readers no match block 3 3 6 - * W: ticket 6, writers no match block 3 3 7 - * W: ticket 3, unlock 4 4 7 - * R: ticket 4, readers match OK 4 5 7 - * R: ticket 5, readers match OK 4 6 7 - * - * The 'next' field is a 2-byte value so the available ticket number wraps at - * 64K requests. If a thread's lock request is not granted until the 'next' - * field cycles and the same ticket is taken by another thread, we could grant - * a lock to two separate threads at the same time, and bad things happen: two - * writer threads or a reader thread and a writer thread would run in parallel, - * and lock waiters could be skipped if the unlocks race. This is unlikely, it - * only happens if a lock request is blocked by 64K other requests. The fix is - * to grow the lock structure fields, but the largest atomic instruction we have - * is 8 bytes, the structure has no room to grow. + * The 'next' field is a 1-byte value so the available ticket number wraps + * after 256 requests. If a thread's write lock request would cause the 'next' + * field to catch up with 'current', instead it waits to avoid the same ticket + * being allocated to multiple threads. */ #include "wt_internal.h" /* - * __wt_rwlock_alloc -- - * Allocate and initialize a read/write lock. + * __wt_rwlock_init -- + * Initialize a read/write lock. */ int -__wt_rwlock_alloc( - WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp, const char *name) +__wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - WT_RWLOCK *rwlock; + l->u.v = 0; - __wt_verbose(session, WT_VERB_MUTEX, "rwlock: alloc %s", name); - - WT_RET(__wt_calloc_one(session, &rwlock)); + WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_readers)); + WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_writers)); + return (0); +} - rwlock->name = name; +/* + * __wt_rwlock_destroy -- + * Destroy a read/write lock. + */ +void +__wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l) +{ + l->u.v = 0; - *rwlockp = rwlock; - return (0); + __wt_cond_destroy(session, &l->cond_readers); + __wt_cond_destroy(session, &l->cond_writers); } /* @@ -139,57 +115,40 @@ __wt_rwlock_alloc( * Try to get a shared lock, fail immediately if unavailable. */ int -__wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l, new, old; + WT_RWLOCK new, old; WT_STAT_CONN_INCR(session, rwlock_read); - l = &rwlock->rwlock; - new = old = *l; + old.u.v = l->u.v; - /* - * This read lock can only be granted if the lock was last granted to - * a reader and there are no readers or writers blocked on the lock, - * that is, if this thread's ticket would be the next ticket granted. - * Do the cheap test to see if this can possibly succeed (and confirm - * the lock is in the correct state to grant this read lock). - */ - if (old.s.readers != old.s.next) + /* This read lock can only be granted if there are no active writers. */ + if (old.u.s.current != old.u.s.next) return (EBUSY); /* - * The replacement lock value is a result of allocating a new ticket and - * incrementing the reader value to match it. + * The replacement lock value is a result of adding an active reader. + * Check for overflow: if the maximum number of readers are already + * active, no new readers can enter the lock. */ - new.s.readers = new.s.next = old.s.next + 1; - return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); + new.u.v = old.u.v; + if (++new.u.s.readers_active == 0) + return (EBUSY); + + /* We rely on this atomic operation to provide a barrier. */ + return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); } /* - * __wt_readlock_spin -- - * Spin to get a read lock: only yield the CPU if the lock is held - * exclusive. + * __read_blocked -- + * Check whether the current read lock request should keep waiting. */ -void -__wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +static bool +__read_blocked(WT_SESSION_IMPL *session) { - wt_rwlock_t *l; - - l = &rwlock->rwlock; - - /* - * Try to get the lock in a single operation if it is available to - * readers. This avoids the situation where multiple readers arrive - * concurrently and have to line up in order to enter the lock. For - * read-heavy workloads it can make a significant difference. - */ - while (__wt_try_readlock(session, rwlock) != 0) { - if (l->s.writers_active > 0) - __wt_yield(); - else - WT_PAUSE(); - } + return (session->current_rwticket != + session->current_rwlock->u.s.current); } /* @@ -197,48 +156,97 @@ __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Get a shared lock. */ void -__wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; - uint16_t ticket; + WT_RWLOCK new, old; int pause_cnt; + int16_t writers_active; + uint8_t ticket; WT_STAT_CONN_INCR(session, rwlock_read); WT_DIAGNOSTIC_YIELD; - l = &rwlock->rwlock; + for (;;) { + /* + * Fast path: if there is no active writer, join the current + * group. + */ + for (old.u.v = l->u.v; + old.u.s.current == old.u.s.next; + old.u.v = l->u.v) { + new.u.v = old.u.v; + /* + * Check for overflow: if the maximum number of readers + * are already active, no new readers can enter the + * lock. + */ + if (++new.u.s.readers_active == 0) + goto stall; + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + return; + WT_PAUSE(); + } - /* - * Possibly wrap: if we have more than 64K lockers waiting, the ticket - * value will wrap and two lockers will simultaneously be granted the - * lock. - */ - ticket = __wt_atomic_fetch_add16(&l->s.next, 1); - for (pause_cnt = 0; ticket != l->s.readers;) { /* - * We failed to get the lock; pause before retrying and if we've - * paused enough, yield so we don't burn CPU to no purpose. This - * situation happens if there are more threads than cores in the - * system and we're thrashing on shared resources. + * There is an active writer: join the next group. + * + * Limit how many readers can queue: don't allow more readers + * to queue than there are active writers (calculated as + * `next - current`): otherwise, in write-heavy workloads, + * readers can keep queuing up in front of writers and + * throughput is unstable. + * + * If the maximum number of readers are already queued, wait + * until we can get a valid ticket. */ - if (++pause_cnt < WT_THOUSAND) + writers_active = old.u.s.next - old.u.s.current; + if (old.u.s.readers_queued > writers_active) { +stall: __wt_cond_wait(session, + l->cond_readers, 10 * WT_THOUSAND, NULL); + continue; + } + + /* + * If we are the first reader to queue, set the next read + * group. Note: don't re-read from the lock or we could race + * with a writer unlocking. + */ + new.u.v = old.u.v; + if (new.u.s.readers_queued++ == 0) + new.u.s.reader = new.u.s.next; + ticket = new.u.s.reader; + + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + break; + } + + /* Wait for our group to start. */ + for (pause_cnt = 0; ticket != l->u.s.current; pause_cnt++) { + if (pause_cnt < 1000) WT_PAUSE(); - else + else if (pause_cnt < 1200) __wt_yield(); + else { + session->current_rwlock = l; + session->current_rwticket = ticket; + __wt_cond_wait(session, + l->cond_readers, 10 * WT_THOUSAND, __read_blocked); + } } - /* - * We're the only writer of the readers field, so the update does not - * need to be atomic. - */ - ++l->s.readers; - /* * Applications depend on a barrier here so that operations holding the - * lock see consistent data. + * lock see consistent data. The atomic operation above isn't + * sufficient here because we don't own the lock until our ticket comes + * up and whatever data we are protecting may have changed in the + * meantime. */ WT_READ_BARRIER(); + + /* Sanity check that we (still) have the lock. */ + WT_ASSERT(session, + ticket == l->u.s.current && l->u.s.readers_active > 0); } /* @@ -246,19 +254,24 @@ __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Release a shared lock. */ void -__wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; + WT_RWLOCK new, old; - WT_UNUSED(session); + do { + old.u.v = l->u.v; + WT_ASSERT(session, old.u.s.readers_active > 0); - l = &rwlock->rwlock; + /* + * Decrement the active reader count (other readers are doing + * the same, make sure we don't race). + */ + new.u.v = old.u.v; + --new.u.s.readers_active; + } while (!__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)); - /* - * Increment the writers value (other readers are doing the same, make - * sure we don't race). - */ - (void)__wt_atomic_add16(&l->s.writers, 1); + if (new.u.s.readers_active == 0 && new.u.s.current != new.u.s.next) + __wt_cond_signal(session, l->cond_writers); } /* @@ -266,29 +279,50 @@ __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Try to get an exclusive lock, fail immediately if unavailable. */ int -__wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l, new, old; + WT_RWLOCK new, old; WT_STAT_CONN_INCR(session, rwlock_write); - l = &rwlock->rwlock; - old = new = *l; - /* - * This write lock can only be granted if the lock was last granted to - * a writer and there are no readers or writers blocked on the lock, - * that is, if this thread's ticket would be the next ticket granted. - * Do the cheap test to see if this can possibly succeed (and confirm - * the lock is in the correct state to grant this write lock). + * This write lock can only be granted if no readers or writers blocked + * on the lock, that is, if this thread's ticket would be the next + * ticket granted. Check if this can possibly succeed (and confirm the + * lock is in the correct state to grant this write lock). */ - if (old.s.writers != old.s.next) + old.u.v = l->u.v; + if (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0) return (EBUSY); - /* The replacement lock value is a result of allocating a new ticket. */ - ++new.s.next; - ++new.s.writers_active; - return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); + /* + * We've checked above that there is no writer active (since + * `current == next`), so there should be no readers queued. + */ + WT_ASSERT(session, old.u.s.readers_queued == 0); + + /* + * The replacement lock value is a result of allocating a new ticket. + * + * We rely on this atomic operation to provide a barrier. + */ + new.u.v = old.u.v; + new.u.s.next++; + return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); +} + +/* + * __write_blocked -- + * Check whether the current write lock request should keep waiting. + */ +static bool +__write_blocked(WT_SESSION_IMPL *session) +{ + WT_RWLOCK *l; + + l = session->current_rwlock; + return (session->current_rwticket != l->u.s.current || + l->u.s.readers_active != 0); } /* @@ -296,41 +330,70 @@ __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Wait to get an exclusive lock. */ void -__wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; - uint16_t ticket; + WT_RWLOCK new, old; int pause_cnt; + uint8_t ticket; WT_STAT_CONN_INCR(session, rwlock_write); - l = &rwlock->rwlock; + for (;;) { + old.u.v = l->u.v; + + /* Allocate a ticket. */ + new.u.v = old.u.v; + ticket = new.u.s.next++; - /* - * Possibly wrap: if we have more than 64K lockers waiting, the ticket - * value will wrap and two lockers will simultaneously be granted the - * lock. - */ - ticket = __wt_atomic_fetch_add16(&l->s.next, 1); - (void)__wt_atomic_add16(&l->s.writers_active, 1); - for (pause_cnt = 0; ticket != l->s.writers;) { /* - * We failed to get the lock; pause before retrying and if we've - * paused enough, sleep so we don't burn CPU to no purpose. This - * situation happens if there are more threads than cores in the - * system and we're thrashing on shared resources. + * Check for overflow: if the next ticket is allowed to catch + * up with the current batch, two writers could be granted the + * lock simultaneously. */ - if (++pause_cnt < WT_THOUSAND) + if (new.u.s.current == new.u.s.next) { + __wt_cond_wait(session, + l->cond_writers, 10 * WT_THOUSAND, NULL); + continue; + } + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + break; + } + + /* + * Wait for our group to start and any readers to drain. + * + * We take care here to do an atomic read of the full 64-bit lock + * value. Otherwise, reads are not guaranteed to be ordered and we + * could see no readers active from a different batch and decide that + * we have the lock. + */ + for (pause_cnt = 0, old.u.v = l->u.v; + ticket != old.u.s.current || old.u.s.readers_active != 0; + pause_cnt++, old.u.v = l->u.v) { + if (pause_cnt < 1000) WT_PAUSE(); - else - __wt_sleep(0, 10); + else if (pause_cnt < 1200) + __wt_yield(); + else { + session->current_rwlock = l; + session->current_rwticket = ticket; + __wt_cond_wait(session, + l->cond_writers, 10 * WT_THOUSAND, __write_blocked); + } } /* * Applications depend on a barrier here so that operations holding the - * lock see consistent data. + * lock see consistent data. The atomic operation above isn't + * sufficient here because we don't own the lock until our ticket comes + * up and whatever data we are protecting may have changed in the + * meantime. */ WT_READ_BARRIER(); + + /* Sanity check that we (still) have the lock. */ + WT_ASSERT(session, + ticket == l->u.s.current && l->u.s.readers_active == 0); } /* @@ -338,70 +401,54 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Release an exclusive lock. */ void -__wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l, new; + WT_RWLOCK new, old; - WT_UNUSED(session); - - l = &rwlock->rwlock; - (void)__wt_atomic_sub16(&l->s.writers_active, 1); - - /* - * Ensure that all updates made while the lock was held are visible to - * the next thread to acquire the lock. - */ - WT_WRITE_BARRIER(); + do { + old.u.v = l->u.v; - new = *l; + /* + * We're holding the lock exclusive, there shouldn't be any + * active readers. + */ + WT_ASSERT(session, old.u.s.readers_active == 0); - /* - * We're the only writer of the writers/readers fields, so the update - * does not need to be atomic; we have to update both values at the - * same time though, otherwise we'd potentially race with the thread - * next granted the lock. - */ - ++new.s.writers; - ++new.s.readers; - l->i.wr = new.i.wr; + /* + * Allow the next batch to start. + * + * If there are readers in the next group, swap queued readers + * to active: this could race with new readlock requests, so we + * have to spin. + */ + new.u.v = old.u.v; + if (++new.u.s.current == new.u.s.reader) { + new.u.s.readers_active = new.u.s.readers_queued; + new.u.s.readers_queued = 0; + } + } while (!__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)); + + if (new.u.s.readers_active != 0) + __wt_cond_signal(session, l->cond_readers); + else if (new.u.s.current != new.u.s.next) + __wt_cond_signal(session, l->cond_writers); WT_DIAGNOSTIC_YIELD; } -/* - * __wt_rwlock_destroy -- - * Destroy a read/write lock. - */ -void -__wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp) -{ - WT_RWLOCK *rwlock; - - rwlock = *rwlockp; /* Clear our caller's reference. */ - if (rwlock == NULL) - return; - *rwlockp = NULL; - - __wt_verbose( - session, WT_VERB_MUTEX, "rwlock: destroy %s", rwlock->name); - - __wt_free(session, rwlock); -} - #ifdef HAVE_DIAGNOSTIC /* * __wt_rwlock_islocked -- * Return if a read/write lock is currently locked for reading or writing. */ bool -__wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; + WT_RWLOCK old; WT_UNUSED(session); - l = &rwlock->rwlock; - - return (l->s.writers != l->s.next || l->s.readers != l->s.next); + old.u.v = l->u.v; + return (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0); } #endif diff --git a/src/third_party/wiredtiger/src/support/rand.c b/src/third_party/wiredtiger/src/support/rand.c index 025b18e4ed3..4fae43edc8e 100644 --- a/src/third_party/wiredtiger/src/support/rand.c +++ b/src/third_party/wiredtiger/src/support/rand.c @@ -51,6 +51,7 @@ */ void __wt_random_init(WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_RAND_STATE rnd; @@ -69,6 +70,7 @@ __wt_random_init(WT_RAND_STATE volatile * rnd_state) void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { struct timespec ts; WT_RAND_STATE rnd; @@ -86,6 +88,7 @@ __wt_random_init_seed( */ uint32_t __wt_random(WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_RAND_STATE rnd; uint32_t w, z; @@ -117,3 +120,15 @@ __wt_random(WT_RAND_STATE volatile * rnd_state) return ((z << 16) + (w & 65535)); } + +/* + * __wt_random64 -- + * Return a 64-bit pseudo-random number. + */ +uint64_t +__wt_random64(WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + return (((uint64_t)__wt_random(rnd_state) << 32) + + __wt_random(rnd_state)); +} diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 69987ebc852..485cea90e89 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -69,13 +69,16 @@ int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) { + WT_DECL_RET; va_list ap; size_t len; for (;;) { va_start(ap, fmt); - len = (size_t)vsnprintf(buf->mem, buf->memsize, fmt, ap); + ret = __wt_vsnprintf_len_set( + buf->mem, buf->memsize, &len, fmt, ap); va_end(ap); + WT_RET(ret); /* Check if there was enough space. */ if (len < buf->memsize) { @@ -100,6 +103,7 @@ int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) { + WT_DECL_RET; va_list ap; size_t len, space; char *p; @@ -117,8 +121,9 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) p = (char *)((uint8_t *)buf->mem + buf->size); WT_ASSERT(session, buf->memsize >= buf->size); space = buf->memsize - buf->size; - len = (size_t)vsnprintf(p, space, fmt, ap); + ret = __wt_vsnprintf_len_set(p, space, &len, fmt, ap); va_end(ap); + WT_RET(ret); /* Check if there was enough space. */ if (len < space) { diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 5acd9fc713f..2c2217f8c20 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -64,6 +64,7 @@ static const char * const __stats_dsrc_desc[] = { "cache: pages requested from the cache", "cache: pages written from cache", "cache: pages written requiring in-memory restoration", + "cache: tracked dirty bytes in the cache", "cache: unmodified pages evicted", "cache_walk: Average difference between current eviction generation when the page was last considered", "cache_walk: Average on-disk page image size seen", @@ -225,6 +226,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_pages_requested = 0; stats->cache_write = 0; stats->cache_write_restore = 0; + /* not clearing cache_bytes_dirty */ stats->cache_eviction_clean = 0; /* not clearing cache_state_gen_avg_gap */ /* not clearing cache_state_avg_written_size */ @@ -372,6 +374,7 @@ __wt_stat_dsrc_aggregate_single( to->cache_pages_requested += from->cache_pages_requested; to->cache_write += from->cache_write; to->cache_write_restore += from->cache_write_restore; + to->cache_bytes_dirty += from->cache_bytes_dirty; to->cache_eviction_clean += from->cache_eviction_clean; to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap; to->cache_state_avg_written_size += @@ -535,6 +538,7 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_pages_requested); to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); + to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty); to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); to->cache_state_gen_avg_gap += WT_STAT_READ(from, cache_state_gen_avg_gap); @@ -673,10 +677,15 @@ static const char * const __stats_connection_desc[] = { "cache: eviction server unable to reach eviction goal", "cache: eviction state", "cache: eviction walks abandoned", + "cache: eviction worker thread active", + "cache: eviction worker thread created", "cache: eviction worker thread evicting pages", + "cache: eviction worker thread removed", + "cache: eviction worker thread stable number", "cache: failed eviction of pages that exceeded the in-memory maximum", "cache: files with active eviction walks", "cache: files with new eviction walks started", + "cache: force re-tuning of eviction workers once in a while", "cache: hazard pointer blocked page eviction", "cache: hazard pointer check calls", "cache: hazard pointer check entries walked", @@ -751,9 +760,7 @@ static const char * const __stats_connection_desc[] = { "lock: checkpoint lock acquisitions", "lock: checkpoint lock application thread wait time (usecs)", "lock: checkpoint lock internal thread wait time (usecs)", - "lock: handle-list lock acquisitions", - "lock: handle-list lock application thread wait time (usecs)", - "lock: handle-list lock internal thread wait time (usecs)", + "lock: handle-list lock eviction thread wait time (usecs)", "lock: metadata lock acquisitions", "lock: metadata lock application thread wait time (usecs)", "lock: metadata lock internal thread wait time (usecs)", @@ -765,9 +772,11 @@ static const char * const __stats_connection_desc[] = { "lock: table lock internal thread time waiting for the table lock (usecs)", "log: busy returns attempting to switch slots", "log: consolidated slot closures", + "log: consolidated slot join active slot closed", "log: consolidated slot join races", "log: consolidated slot join transitions", "log: consolidated slot joins", + "log: consolidated slot transitions unable to find free slot", "log: consolidated slot unbuffered writes", "log: log bytes of payload data", "log: log bytes written", @@ -808,6 +817,9 @@ static const char * const __stats_connection_desc[] = { "reconciliation: split objects currently awaiting free", "session: open cursor count", "session: open session count", + "session: table alter failed calls", + "session: table alter successful calls", + "session: table alter unchanged and skipped", "session: table compact failed calls", "session: table compact successful calls", "session: table create failed calls", @@ -951,10 +963,15 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_slow = 0; /* not clearing cache_eviction_state */ stats->cache_eviction_walks_abandoned = 0; + /* not clearing cache_eviction_active_workers */ + stats->cache_eviction_worker_created = 0; stats->cache_eviction_worker_evicting = 0; + stats->cache_eviction_worker_removed = 0; + /* not clearing cache_eviction_stable_state_workers */ stats->cache_eviction_force_fail = 0; /* not clearing cache_eviction_walks_active */ stats->cache_eviction_walks_started = 0; + stats->cache_eviction_force_retune = 0; stats->cache_eviction_hazard = 0; stats->cache_hazard_checks = 0; stats->cache_hazard_walks = 0; @@ -1029,9 +1046,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_checkpoint_count = 0; stats->lock_checkpoint_wait_application = 0; stats->lock_checkpoint_wait_internal = 0; - stats->lock_handle_list_count = 0; - stats->lock_handle_list_wait_application = 0; - stats->lock_handle_list_wait_internal = 0; + stats->lock_handle_list_wait_eviction = 0; stats->lock_metadata_count = 0; stats->lock_metadata_wait_application = 0; stats->lock_metadata_wait_internal = 0; @@ -1043,9 +1058,11 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_table_wait_internal = 0; stats->log_slot_switch_busy = 0; stats->log_slot_closes = 0; + stats->log_slot_active_closed = 0; stats->log_slot_races = 0; stats->log_slot_transitions = 0; stats->log_slot_joins = 0; + stats->log_slot_no_free_slots = 0; stats->log_slot_unbuffered = 0; stats->log_bytes_payload = 0; stats->log_bytes_written = 0; @@ -1086,6 +1103,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing rec_split_stashed_objects */ /* not clearing session_cursor_open */ /* not clearing session_open */ + /* not clearing session_table_alter_fail */ + /* not clearing session_table_alter_success */ + /* not clearing session_table_alter_skip */ /* not clearing session_table_compact_fail */ /* not clearing session_table_compact_success */ /* not clearing session_table_create_fail */ @@ -1222,14 +1242,24 @@ __wt_stat_connection_aggregate( to->cache_eviction_state += WT_STAT_READ(from, cache_eviction_state); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); + to->cache_eviction_active_workers += + WT_STAT_READ(from, cache_eviction_active_workers); + to->cache_eviction_worker_created += + WT_STAT_READ(from, cache_eviction_worker_created); to->cache_eviction_worker_evicting += WT_STAT_READ(from, cache_eviction_worker_evicting); + to->cache_eviction_worker_removed += + WT_STAT_READ(from, cache_eviction_worker_removed); + to->cache_eviction_stable_state_workers += + WT_STAT_READ(from, cache_eviction_stable_state_workers); to->cache_eviction_force_fail += WT_STAT_READ(from, cache_eviction_force_fail); to->cache_eviction_walks_active += WT_STAT_READ(from, cache_eviction_walks_active); to->cache_eviction_walks_started += WT_STAT_READ(from, cache_eviction_walks_started); + to->cache_eviction_force_retune += + WT_STAT_READ(from, cache_eviction_force_retune); to->cache_eviction_hazard += WT_STAT_READ(from, cache_eviction_hazard); to->cache_hazard_checks += WT_STAT_READ(from, cache_hazard_checks); @@ -1325,12 +1355,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_checkpoint_wait_application); to->lock_checkpoint_wait_internal += WT_STAT_READ(from, lock_checkpoint_wait_internal); - to->lock_handle_list_count += - WT_STAT_READ(from, lock_handle_list_count); - to->lock_handle_list_wait_application += - WT_STAT_READ(from, lock_handle_list_wait_application); - to->lock_handle_list_wait_internal += - WT_STAT_READ(from, lock_handle_list_wait_internal); + to->lock_handle_list_wait_eviction += + WT_STAT_READ(from, lock_handle_list_wait_eviction); to->lock_metadata_count += WT_STAT_READ(from, lock_metadata_count); to->lock_metadata_wait_application += WT_STAT_READ(from, lock_metadata_wait_application); @@ -1348,9 +1374,13 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_table_wait_internal); to->log_slot_switch_busy += WT_STAT_READ(from, log_slot_switch_busy); to->log_slot_closes += WT_STAT_READ(from, log_slot_closes); + to->log_slot_active_closed += + WT_STAT_READ(from, log_slot_active_closed); to->log_slot_races += WT_STAT_READ(from, log_slot_races); to->log_slot_transitions += WT_STAT_READ(from, log_slot_transitions); to->log_slot_joins += WT_STAT_READ(from, log_slot_joins); + to->log_slot_no_free_slots += + WT_STAT_READ(from, log_slot_no_free_slots); to->log_slot_unbuffered += WT_STAT_READ(from, log_slot_unbuffered); to->log_bytes_payload += WT_STAT_READ(from, log_bytes_payload); to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); @@ -1397,6 +1427,12 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, rec_split_stashed_objects); to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->session_open += WT_STAT_READ(from, session_open); + to->session_table_alter_fail += + WT_STAT_READ(from, session_table_alter_fail); + to->session_table_alter_success += + WT_STAT_READ(from, session_table_alter_success); + to->session_table_alter_skip += + WT_STAT_READ(from, session_table_alter_skip); to->session_table_compact_fail += WT_STAT_READ(from, session_table_compact_fail); to->session_table_compact_success += diff --git a/src/third_party/wiredtiger/src/support/thread_group.c b/src/third_party/wiredtiger/src/support/thread_group.c index a866d2d01c5..090621fa138 100644 --- a/src/third_party/wiredtiger/src/support/thread_group.c +++ b/src/third_party/wiredtiger/src/support/thread_group.c @@ -50,8 +50,7 @@ __thread_group_grow( { WT_THREAD *thread; - WT_ASSERT(session, - __wt_rwlock_islocked(session, group->lock)); + WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); /* * Any bounds checking is done by the caller so we know that @@ -72,20 +71,19 @@ __thread_group_grow( /* * __thread_group_shrink -- - * Decrease the number of running threads in the group, and free any + * Decrease the number of running threads in the group. Optionally free any * memory associated with slots larger than the new count. */ static int __thread_group_shrink(WT_SESSION_IMPL *session, - WT_THREAD_GROUP *group, uint32_t new_count) + WT_THREAD_GROUP *group, uint32_t new_count, bool free_thread) { WT_DECL_RET; WT_SESSION *wt_session; WT_THREAD *thread; uint32_t current_slot; - WT_ASSERT(session, - __wt_rwlock_islocked(session, group->lock)); + WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); for (current_slot = group->alloc; current_slot > new_count; ) { /* @@ -107,14 +105,15 @@ __thread_group_shrink(WT_SESSION_IMPL *session, WT_TRET(__wt_thread_join(session, thread->tid)); thread->tid = 0; } - - if (thread->session != NULL) { - wt_session = (WT_SESSION *)thread->session; - WT_TRET(wt_session->close(wt_session, NULL)); - thread->session = NULL; + if (free_thread) { + if (thread->session != NULL) { + wt_session = (WT_SESSION *)thread->session; + WT_TRET(wt_session->close(wt_session, NULL)); + thread->session = NULL; + } + __wt_free(session, thread); + group->threads[current_slot] = NULL; } - __wt_free(session, thread); - group->threads[current_slot] = NULL; } /* Update the thread group state to match our changes */ @@ -142,16 +141,19 @@ __thread_group_resize( WT_ASSERT(session, group->current_threads <= group->alloc && - __wt_rwlock_islocked(session, group->lock)); + __wt_rwlock_islocked(session, &group->lock)); if (new_min == group->min && new_max == group->max) return (0); + if (new_min > new_max) + return (EINVAL); + /* - * Coll shrink to reduce the number of thread structures and running + * Call shrink to reduce the number of thread structures and running * threads if required by the change in group size. */ - WT_RET(__thread_group_shrink(session, group, new_max)); + WT_RET(__thread_group_shrink(session, group, new_max, true)); /* * Only reallocate the thread array if it is the largest ever, since @@ -227,9 +229,9 @@ __wt_thread_group_resize( " from max: %" PRIu32 " -> %" PRIu32, (void *)group, group->min, new_min, group->max, new_max); - __wt_writelock(session, group->lock); + __wt_writelock(session, &group->lock); WT_TRET(__thread_group_resize(session, group, new_min, new_max, flags)); - __wt_writeunlock(session, group->lock); + __wt_writeunlock(session, &group->lock); return (ret); } @@ -255,22 +257,22 @@ __wt_thread_group_create( __wt_verbose(session, WT_VERB_THREAD_GROUP, "Creating thread group: %p", (void *)group); - WT_RET(__wt_rwlock_alloc(session, &group->lock, "Thread group")); + WT_RET(__wt_rwlock_init(session, &group->lock)); WT_ERR(__wt_cond_alloc( - session, "Thread group cond", false, &group->wait_cond)); + session, "thread group cond", &group->wait_cond)); cond_alloced = true; - __wt_writelock(session, group->lock); + __wt_writelock(session, &group->lock); group->run_func = run_func; group->name = name; WT_TRET(__thread_group_resize(session, group, min, max, flags)); - __wt_writeunlock(session, group->lock); + __wt_writeunlock(session, &group->lock); /* Cleanup on error to avoid leaking resources */ err: if (ret != 0) { if (cond_alloced) - WT_TRET(__wt_cond_destroy(session, &group->wait_cond)); + __wt_cond_destroy(session, &group->wait_cond); __wt_rwlock_destroy(session, &group->lock); } return (ret); @@ -288,14 +290,14 @@ __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) __wt_verbose(session, WT_VERB_THREAD_GROUP, "Destroying thread group: %p", (void *)group); - WT_ASSERT(session, __wt_rwlock_islocked(session, group->lock)); + WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); /* Shut down all threads and free associated resources. */ - WT_TRET(__thread_group_shrink(session, group, 0)); + WT_TRET(__thread_group_shrink(session, group, 0, true)); __wt_free(session, group->threads); - WT_TRET(__wt_cond_destroy(session, &group->wait_cond)); + __wt_cond_destroy(session, &group->wait_cond); __wt_rwlock_destroy(session, &group->lock); /* @@ -322,15 +324,42 @@ __wt_thread_group_start_one( return (0); if (wait) - __wt_writelock(session, group->lock); - else if (__wt_try_writelock(session, group->lock) != 0) - return (0); + __wt_writelock(session, &group->lock); + else + WT_RET(__wt_try_writelock(session, &group->lock)); /* Recheck the bounds now that we hold the lock */ if (group->current_threads < group->max) WT_TRET(__thread_group_grow( session, group, group->current_threads + 1)); - __wt_writeunlock(session, group->lock); + __wt_writeunlock(session, &group->lock); + + return (ret); +} + +/* + * __wt_thread_group_stop_one -- + * Stop one thread if possible. + */ +int +__wt_thread_group_stop_one( + WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) +{ + WT_DECL_RET; + + if (group->current_threads <= group->min) + return (0); + + if (wait) + __wt_writelock(session, &group->lock); + else + WT_RET(__wt_try_writelock(session, &group->lock)); + + /* Recheck the bounds now that we hold the lock */ + if (group->current_threads > group->min) + WT_TRET(__thread_group_shrink( + session, group, group->current_threads - 1, false)); + __wt_writeunlock(session, &group->lock); return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 1c47f94bb60..ea7faa2e966 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -100,7 +100,7 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) session->txn.isolation == WT_ISO_READ_UNCOMMITTED || !__wt_txn_visible_all(session, txn_state->pinned_id)); - txn_state->pinned_id = WT_TXN_NONE; + txn_state->metadata_pinned = txn_state->pinned_id = WT_TXN_NONE; F_CLR(txn, WT_TXN_HAS_SNAPSHOT); } @@ -108,7 +108,7 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) * __wt_txn_get_snapshot -- * Allocate a snapshot. */ -int +void __wt_txn_get_snapshot(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; @@ -126,7 +126,7 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) n = 0; /* We're going to scan the table: wait for the lock. */ - __wt_readlock_spin(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); current_id = pinned_id = txn_global->current; prev_oldest_id = txn_global->oldest_id; @@ -137,8 +137,10 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) * metadata. We don't have to keep the checkpoint's changes pinned so * don't including it in the published pinned ID. */ - if ((id = txn_global->checkpoint_txnid) != WT_TXN_NONE) + if ((id = txn_global->checkpoint_txnid) != WT_TXN_NONE) { txn->snapshot[n++] = id; + txn_state->metadata_pinned = id; + } /* For pure read-only workloads, avoid scanning. */ if (prev_oldest_id == current_id) { @@ -178,9 +180,8 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) WT_ASSERT(session, prev_oldest_id == txn_global->oldest_id); txn_state->pinned_id = pinned_id; -done: __wt_readunlock(session, txn_global->scan_rwlock); +done: __wt_readunlock(session, &txn_global->scan_rwlock); __txn_sort_snapshot(session, n, current_id); - return (0); } /* @@ -189,14 +190,14 @@ done: __wt_readunlock(session, txn_global->scan_rwlock); */ static void __txn_oldest_scan(WT_SESSION_IMPL *session, - uint64_t *oldest_idp, uint64_t *last_runningp, + uint64_t *oldest_idp, uint64_t *last_runningp, uint64_t *metadata_pinnedp, WT_SESSION_IMPL **oldest_sessionp) { WT_CONNECTION_IMPL *conn; WT_SESSION_IMPL *oldest_session; WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *s; - uint64_t id, last_running, oldest_id, prev_oldest_id; + uint64_t id, last_running, metadata_pinned, oldest_id, prev_oldest_id; uint32_t i, session_cnt; conn = S2C(session); @@ -205,24 +206,24 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, /* The oldest ID cannot change while we are holding the scan lock. */ prev_oldest_id = txn_global->oldest_id; - oldest_id = last_running = txn_global->current; + last_running = oldest_id = txn_global->current; + if ((metadata_pinned = txn_global->checkpoint_txnid) == WT_TXN_NONE) + metadata_pinned = oldest_id; /* Walk the array of concurrent transactions. */ WT_ORDERED_READ(session_cnt, conn->session_cnt); for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - /* - * Update the oldest ID. - * - * Ignore: IDs older than the oldest ID we saw. This can happen - * if we race with a thread that is allocating an ID -- the ID - * will not be used because the thread will keep spinning until - * it gets a valid one. - */ + /* Update the last running transaction ID. */ if ((id = s->id) != WT_TXN_NONE && WT_TXNID_LE(prev_oldest_id, id) && WT_TXNID_LT(id, last_running)) last_running = id; + /* Update the metadata pinned ID. */ + if ((id = s->metadata_pinned) != WT_TXN_NONE && + WT_TXNID_LT(id, metadata_pinned)) + metadata_pinned = id; + /* * !!! * Note: Don't ignore pinned ID values older than the previous @@ -246,9 +247,14 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, WT_TXNID_LT(id, oldest_id)) oldest_id = id; + /* The metadata pinned ID can't move past the oldest ID. */ + if (WT_TXNID_LT(oldest_id, metadata_pinned)) + metadata_pinned = oldest_id; + + *last_runningp = last_running; + *metadata_pinnedp = metadata_pinned; *oldest_idp = oldest_id; *oldest_sessionp = oldest_session; - *last_runningp = last_running; } /* @@ -262,8 +268,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_DECL_RET; WT_SESSION_IMPL *oldest_session; WT_TXN_GLOBAL *txn_global; - uint64_t current_id, last_running, oldest_id; - uint64_t prev_last_running, prev_oldest_id; + uint64_t current_id, last_running, metadata_pinned, oldest_id; + uint64_t prev_last_running, prev_metadata_pinned, prev_oldest_id; bool strict, wait; conn = S2C(session); @@ -271,26 +277,29 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) strict = LF_ISSET(WT_TXN_OLDEST_STRICT); wait = LF_ISSET(WT_TXN_OLDEST_WAIT); - current_id = last_running = txn_global->current; + current_id = last_running = metadata_pinned = txn_global->current; prev_last_running = txn_global->last_running; + prev_metadata_pinned = txn_global->metadata_pinned; prev_oldest_id = txn_global->oldest_id; /* * For pure read-only workloads, or if the update isn't forced and the * oldest ID isn't too far behind, avoid scanning. */ - if (prev_oldest_id == current_id || + if ((prev_oldest_id == current_id && + prev_metadata_pinned == current_id) || (!strict && WT_TXNID_LT(current_id, prev_oldest_id + 100))) return (0); /* First do a read-only scan. */ if (wait) - __wt_readlock_spin(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); else if ((ret = - __wt_try_readlock(session, txn_global->scan_rwlock)) != 0) + __wt_try_readlock(session, &txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); - __txn_oldest_scan(session, &oldest_id, &last_running, &oldest_session); - __wt_readunlock(session, txn_global->scan_rwlock); + __txn_oldest_scan(session, + &oldest_id, &last_running, &metadata_pinned, &oldest_session); + __wt_readunlock(session, &txn_global->scan_rwlock); /* * If the state hasn't changed (or hasn't moved far enough for @@ -299,14 +308,15 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) if ((oldest_id == prev_oldest_id || (!strict && WT_TXNID_LT(oldest_id, prev_oldest_id + 100))) && ((last_running == prev_last_running) || - (!strict && WT_TXNID_LT(last_running, prev_last_running + 100)))) + (!strict && WT_TXNID_LT(last_running, prev_last_running + 100))) && + metadata_pinned == prev_metadata_pinned) return (0); /* It looks like an update is necessary, wait for exclusive access. */ if (wait) - __wt_writelock(session, txn_global->scan_rwlock); + __wt_writelock(session, &txn_global->scan_rwlock); else if ((ret = - __wt_try_writelock(session, txn_global->scan_rwlock)) != 0) + __wt_try_writelock(session, &txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); /* @@ -314,7 +324,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) * scanning. */ if (WT_TXNID_LE(oldest_id, txn_global->oldest_id) && - WT_TXNID_LE(last_running, txn_global->last_running)) + WT_TXNID_LE(last_running, txn_global->last_running) && + WT_TXNID_LE(metadata_pinned, txn_global->metadata_pinned)) goto done; /* @@ -323,7 +334,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) * sure that there isn't a thread that has got a snapshot locally but * not yet published its snap_min. */ - __txn_oldest_scan(session, &oldest_id, &last_running, &oldest_session); + __txn_oldest_scan(session, + &oldest_id, &last_running, &metadata_pinned, &oldest_session); #ifdef HAVE_DIAGNOSTIC { @@ -339,7 +351,9 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) id == WT_TXN_NONE || !WT_TXNID_LT(id, oldest_id)); } #endif - /* Update the oldest ID. */ + /* Update the public IDs. */ + if (WT_TXNID_LT(txn_global->metadata_pinned, metadata_pinned)) + txn_global->metadata_pinned = metadata_pinned; if (WT_TXNID_LT(txn_global->oldest_id, oldest_id)) txn_global->oldest_id = oldest_id; if (WT_TXNID_LT(txn_global->last_running, last_running)) { @@ -361,7 +375,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) #endif } -done: __wt_writeunlock(session, txn_global->scan_rwlock); +done: __wt_writeunlock(session, &txn_global->scan_rwlock); return (ret); } @@ -699,7 +713,7 @@ __wt_txn_stats_update(WT_SESSION_IMPL *session) snapshot_pinned = txn_global->nsnap_oldest_id; WT_STAT_SET(session, stats, txn_pinned_range, - txn_global->current - txn_global->oldest_id); + txn_global->current - txn_global->oldest_id); WT_STAT_SET(session, stats, txn_pinned_snapshot_range, snapshot_pinned == WT_TXN_NONE ? @@ -750,23 +764,20 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) txn_global = &conn->txn_global; txn_global->current = txn_global->last_running = - txn_global->oldest_id = WT_TXN_FIRST; + txn_global->metadata_pinned = txn_global->oldest_id = WT_TXN_FIRST; WT_RET(__wt_spin_init(session, &txn_global->id_lock, "transaction id lock")); - WT_RET(__wt_rwlock_alloc(session, - &txn_global->scan_rwlock, "transaction scan lock")); - WT_RET(__wt_rwlock_alloc(session, - &txn_global->nsnap_rwlock, "named snapshot lock")); + WT_RET(__wt_rwlock_init(session, &txn_global->scan_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); WT_RET(__wt_calloc_def( session, conn->session_size, &txn_global->states)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, txn_global->states); for (i = 0, s = txn_global->states; i < conn->session_size; i++, s++) - s->id = s->pinned_id = WT_TXN_NONE; + s->id = s->metadata_pinned = s->pinned_id = WT_TXN_NONE; return (0); } @@ -792,3 +803,98 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); __wt_free(session, txn_global->states); } + +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) +/* + * __wt_verbose_dump_txn -- + * Output diagnostic information about the global transaction state. + */ +int +__wt_verbose_dump_txn(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_TXN_GLOBAL *txn_global; + WT_TXN *txn; + WT_TXN_STATE *s; + const char *iso_tag; + uint64_t id; + uint32_t i, session_cnt; + + conn = S2C(session); + txn_global = &conn->txn_global; + + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "transaction state dump")); + + WT_RET(__wt_msg(session, "current ID: %" PRIu64, txn_global->current)); + WT_RET(__wt_msg(session, + "last running ID: %" PRIu64, txn_global->last_running)); + WT_RET(__wt_msg(session, "oldest ID: %" PRIu64, txn_global->oldest_id)); + WT_RET(__wt_msg(session, + "oldest named snapshot ID: %" PRIu64, txn_global->nsnap_oldest_id)); + + WT_RET(__wt_msg(session, "checkpoint running? %s", + txn_global->checkpoint_running ? "yes" : "no")); + WT_RET(__wt_msg(session, + "checkpoint generation: %" PRIu64, txn_global->checkpoint_gen)); + WT_RET(__wt_msg(session, + "checkpoint pinned ID: %" PRIu64, txn_global->checkpoint_pinned)); + WT_RET(__wt_msg(session, + "checkpoint txn ID: %" PRIu64, txn_global->checkpoint_txnid)); + + WT_ORDERED_READ(session_cnt, conn->session_cnt); + WT_RET(__wt_msg(session, "session count: %" PRIu32, session_cnt)); + + WT_RET(__wt_msg(session, "Transaction state of active sessions:")); + + /* + * Walk each session transaction state and dump information. Accessing + * the content of session handles is not thread safe, so some + * information may change while traversing if other threads are active + * at the same time, which is OK since this is diagnostic code. + */ + for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { + /* Skip sessions with no active transaction */ + if ((id = s->id) == WT_TXN_NONE && s->pinned_id == WT_TXN_NONE) + continue; + + txn = &conn->sessions[i].txn; + iso_tag = "INVALID"; + switch (txn->isolation) { + case WT_ISO_READ_COMMITTED: + iso_tag = "WT_ISO_READ_COMMITTED"; + break; + case WT_ISO_READ_UNCOMMITTED: + iso_tag = "WT_ISO_READ_UNCOMMITTED"; + break; + case WT_ISO_SNAPSHOT: + iso_tag = "WT_ISO_SNAPSHOT"; + break; + } + + WT_RET(__wt_msg(session, + "ID: %6" PRIu64 + ", mod count: %u" + ", pinned ID: %" PRIu64 + ", snap min: %" PRIu64 + ", snap max: %" PRIu64 + ", metadata pinned ID: %" PRIu64 + ", flags: 0x%08" PRIx32 + ", name: %s" + ", isolation: %s", + id, + txn->mod_count, + s->pinned_id, + txn->snap_min, + txn->snap_max, + s->metadata_pinned, + txn->flags, + conn->sessions[i].name == NULL ? + "EMPTY" : conn->sessions[i].name, + iso_tag)); + } + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + + return (0); +} +#endif diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 1efd0200a3d..f4ccf5eacd0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -8,9 +8,9 @@ #include "wt_internal.h" -static int __checkpoint_lock_tree( - WT_SESSION_IMPL *, bool, bool, const char *[]); -static int __checkpoint_mark_deletes(WT_SESSION_IMPL *, const char *[]); +static int __checkpoint_lock_dirty_tree( + WT_SESSION_IMPL *, bool, bool, bool, const char *[]); +static int __checkpoint_mark_skip(WT_SESSION_IMPL *, WT_CKPT *, bool); static int __checkpoint_presync(WT_SESSION_IMPL *, const char *[]); static int __checkpoint_tree_helper(WT_SESSION_IMPL *, const char *[]); @@ -89,6 +89,33 @@ err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } +/* + * __checkpoint_update_generation -- + * Update the checkpoint generation of the current tree. + * + * This indicates that the tree will not be visited again by the current + * checkpoint. + */ +static void +__checkpoint_update_generation(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + + btree = S2BT(session); + + /* + * Updates to the metadata are made by the checkpoint transaction, so + * the metadata tree's checkpoint generation should never be updated. + */ + if (WT_IS_METADATA(session->dhandle)) + return; + + WT_PUBLISH(btree->checkpoint_gen, + S2C(session)->txn_global.checkpoint_gen); + WT_STAT_DATA_SET(session, + btree_checkpoint_generation, btree->checkpoint_gen); +} + /* * __checkpoint_apply_all -- * Apply an operation to all files involved in a checkpoint. @@ -239,24 +266,82 @@ int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) { WT_BTREE *btree; + WT_CONFIG_ITEM cval; WT_DECL_RET; const char *name; + bool force; - WT_UNUSED(cfg); + btree = S2BT(session); + + /* Find out if we have to force a checkpoint. */ + WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); + force = cval.val != 0; + if (!force) { + WT_RET(__wt_config_gets_def(session, cfg, "name", 0, &cval)); + force = cval.len != 0; + } /* Should not be called with anything other than a file object. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); WT_ASSERT(session, WT_PREFIX_MATCH(session->dhandle->name, "file:")); /* Skip files that are never involved in a checkpoint. */ - if (F_ISSET(S2BT(session), WT_BTREE_NO_CHECKPOINT)) + if (F_ISSET(btree, WT_BTREE_NO_CHECKPOINT)) + return (0); + +#ifdef HAVE_DIAGNOSTIC + /* + * We may have raced between starting the checkpoint transaction and + * some operation completing on the handle that updated the metadata + * (e.g., closing a bulk load cursor). All such operations either have + * exclusive access to the handle or hold the schema lock. We are now + * holding the schema lock and have an open btree handle, so if we + * can't update the metadata, then there has been some state change + * invisible to the checkpoint transaction. + */ + if (!WT_IS_METADATA(session->dhandle)) { + WT_CURSOR *meta_cursor; + bool metadata_race; + + WT_ASSERT(session, !F_ISSET(&session->txn, WT_TXN_ERROR)); + WT_RET(__wt_metadata_cursor(session, &meta_cursor)); + meta_cursor->set_key(meta_cursor, session->dhandle->name); + ret = __wt_curfile_insert_check(meta_cursor); + if (ret == WT_ROLLBACK) { + metadata_race = true; + ret = 0; + } else + metadata_race = false; + WT_TRET(__wt_metadata_cursor_release(session, &meta_cursor)); + WT_RET(ret); + WT_ASSERT(session, !metadata_race); + } +#endif + + /* + * Decide whether the tree needs to be included in the checkpoint and + * if so, acquire the necessary locks. + */ + WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree( + session, true, force, true, cfg)); + WT_RET(ret); + if (F_ISSET(btree, WT_BTREE_SKIP_CKPT)) { + WT_ASSERT(session, btree->ckpt == NULL); + __checkpoint_update_generation(session); return (0); + } - /* Make sure there is space for the next entry. */ + /* + * Make sure there is space for the new entry: do this before getting + * the handle to avoid cleanup if we can't allocate the memory. + */ WT_RET(__wt_realloc_def(session, &session->ckpt_handle_allocated, session->ckpt_handle_next + 1, &session->ckpt_handle)); - /* Not strictly necessary, but cleaner to clear the current handle. */ + /* + * The current tree will be included: get it again because the handle + * we have is only valid for the duration of this function. + */ name = session->dhandle->name; session->dhandle = NULL; @@ -268,48 +353,12 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) * with eviction and we don't want to unfairly penalize (or promote) * eviction in trees due to checkpoints. */ - btree = S2BT(session); btree->evict_walk_saved = btree->evict_walk_period; - WT_SAVE_DHANDLE(session, - ret = __checkpoint_lock_tree(session, true, true, cfg)); - if (ret != 0) { - WT_TRET(__wt_session_release_btree(session)); - return (ret); - } - - /* - * Flag that the handle is part of a checkpoint for the purposes - * of transaction visibility checks. - */ - WT_PUBLISH(btree->include_checkpoint_txn, true); - session->ckpt_handle[session->ckpt_handle_next++] = session->dhandle; return (0); } -/* - * __checkpoint_update_generation -- - * Update the checkpoint generation of the current tree. - * - * This indicates that the tree will not be visited again by the current - * checkpoint. - */ -static void -__checkpoint_update_generation(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - - btree = S2BT(session); - if (!WT_IS_METADATA(session, session->dhandle)) - WT_PUBLISH(btree->include_checkpoint_txn, false); - - WT_PUBLISH(btree->checkpoint_gen, - S2C(session)->txn_global.checkpoint_gen); - WT_STAT_DATA_SET(session, - btree_checkpoint_generation, btree->checkpoint_gen); -} - /* * __checkpoint_reduce_dirty_cache -- * Release clean trees from the list cached for checkpoints. @@ -373,7 +422,6 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) __wt_sleep(0, stepdown_us / 10); __wt_epoch(session, &stop); current_us = WT_TIMEDIFF_US(stop, last); - total_ms = WT_TIMEDIFF_MS(stop, start); bytes_written_total = cache->bytes_written - bytes_written_start; @@ -435,36 +483,6 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) WT_STAT_CONN_SET(session, txn_checkpoint_scrub_time, total_ms); } -/* - * __checkpoint_release_clean_trees -- - * Release clean trees from the list cached for checkpoints. - */ -static int -__checkpoint_release_clean_trees(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - u_int i; - - for (i = 0; i < session->ckpt_handle_next; i++) { - dhandle = session->ckpt_handle[i]; - btree = dhandle->handle; - if (!F_ISSET(btree, WT_BTREE_SKIP_CKPT)) - continue; - __wt_meta_ckptlist_free(session, btree->ckpt); - btree->ckpt = NULL; - WT_WITH_DHANDLE(session, dhandle, - __checkpoint_update_generation(session)); - session->ckpt_handle[i] = NULL; - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_session_release_btree(session)); - WT_RET(ret); - } - - return (0); -} - /* * __checkpoint_stats -- * Update checkpoint timer stats. @@ -526,6 +544,112 @@ __checkpoint_verbose_track(WT_SESSION_IMPL *session, #endif } +/* + * __checkpoint_fail_reset -- + * Reset fields when a failure occurs. + */ +static void +__checkpoint_fail_reset(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + + btree = S2BT(session); + btree->modified = true; + __wt_meta_ckptlist_free(session, &btree->ckpt); +} + +/* + * __checkpoint_prepare -- + * Start the transaction for a checkpoint and gather handles. + */ +static int +__checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; + WT_TXN_STATE *txn_state; + const char *txn_cfg[] = { WT_CONFIG_BASE(session, + WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; + + conn = S2C(session); + txn = &session->txn; + txn_global = &conn->txn_global; + txn_state = WT_SESSION_TXN_STATE(session); + + /* + * Start a snapshot transaction for the checkpoint. + * + * Note: we don't go through the public API calls because they have + * side effects on cursors, which applications can hold open across + * calls to checkpoint. + */ + WT_RET(__wt_txn_begin(session, txn_cfg)); + + WT_DIAGNOSTIC_YIELD; + + /* Ensure a transaction ID is allocated prior to sharing it globally */ + WT_RET(__wt_txn_id_check(session)); + + /* + * Mark the connection as clean. If some data gets modified after + * generating checkpoint transaction id, connection will be reset to + * dirty when reconciliation marks the btree dirty on encountering the + * dirty page. + */ + conn->modified = false; + + /* + * Save the checkpoint session ID. + * + * We never do checkpoints in the default session (with id zero). + */ + WT_ASSERT(session, session->id != 0 && txn_global->checkpoint_id == 0); + txn_global->checkpoint_id = session->id; + + /* + * Remove the checkpoint transaction from the global table. + * + * This allows ordinary visibility checks to move forward because + * checkpoints often take a long time and only write to the metadata. + */ + __wt_writelock(session, &txn_global->scan_rwlock); + txn_global->checkpoint_txnid = txn->id; + txn_global->checkpoint_pinned = WT_MIN(txn->id, txn->snap_min); + + /* + * Sanity check that the oldest ID hasn't moved on before we have + * cleared our entry. + */ + WT_ASSERT(session, + WT_TXNID_LE(txn_global->oldest_id, txn_state->id) && + WT_TXNID_LE(txn_global->oldest_id, txn_state->pinned_id)); + + /* + * Clear our entry from the global transaction session table. Any + * operation that needs to know about the ID for this checkpoint will + * consider the checkpoint ID in the global structure. Most operations + * can safely ignore the checkpoint ID (see the visible all check for + * details). + */ + txn_state->id = txn_state->pinned_id = + txn_state->metadata_pinned = WT_TXN_NONE; + __wt_writeunlock(session, &txn_global->scan_rwlock); + + /* + * Get a list of handles we want to flush; for named checkpoints this + * may pull closed objects into the session cache. + * + * First, gather all handles, then start the checkpoint transaction, + * then release any clean handles. + */ + WT_ASSERT(session, session->ckpt_handle_next == 0); + WT_WITH_TABLE_READ_LOCK(session, ret = __checkpoint_apply_all( + session, cfg, __wt_checkpoint_get_handles, NULL)); + return (ret); +} + /* * __txn_checkpoint -- * Checkpoint a database or a list of objects in the database. @@ -541,25 +665,18 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN *txn; WT_TXN_GLOBAL *txn_global; WT_TXN_ISOLATION saved_isolation; - WT_TXN_STATE *txn_state; void *saved_meta_next; u_int i; uint64_t fsync_duration_usecs; - bool full, idle, logging, tracking; - const char *txn_cfg[] = { WT_CONFIG_BASE(session, - WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; + bool failed, full, idle, logging, tracking; conn = S2C(session); cache = conn->cache; txn = &session->txn; txn_global = &conn->txn_global; - txn_state = WT_SESSION_TXN_STATE(session); saved_isolation = session->isolation; full = idle = logging = tracking = false; - /* Ensure the metadata table is open before taking any locks. */ - WT_RET(__wt_metadata_cursor(session, NULL)); - /* * Do a pass over the configuration arguments and figure out what kind * of checkpoint this is. @@ -625,86 +742,24 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) tracking = true; /* - * Get a list of handles we want to flush; for named checkpoints this - * may pull closed objects into the session cache. - * * We want to skip checkpointing clean handles whenever possible. That * is, when the checkpoint is not named or forced. However, we need to * take care about ordering with respect to the checkpoint transaction. * - * If we skip clean handles before starting the transaction, the + * We can't skip clean handles before starting the transaction or the * checkpoint can miss updates in trees that become dirty as the * checkpoint is starting. If we wait until the transaction has * started before locking a handle, there could be a metadata-changing * operation in between (e.g., salvage) that will cause a write * conflict when the checkpoint goes to write the metadata. * - * First, gather all handles, then start the checkpoint transaction, - * then release any clean handles. + * Hold the schema lock while starting the transaction and gathering + * handles so the set we get is complete and correct. */ - WT_ASSERT(session, session->ckpt_handle_next == 0); - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __checkpoint_apply_all( - session, cfg, __wt_checkpoint_get_handles, NULL)))); + WT_WITH_SCHEMA_LOCK(session, ret = __checkpoint_prepare(session, cfg)); WT_ERR(ret); - /* - * Start a snapshot transaction for the checkpoint. - * - * Note: we don't go through the public API calls because they have - * side effects on cursors, which applications can hold open across - * calls to checkpoint. - */ - WT_ERR(__wt_txn_begin(session, txn_cfg)); - - /* Ensure a transaction ID is allocated prior to sharing it globally */ - WT_ERR(__wt_txn_id_check(session)); - - /* - * Mark the connection as clean. If some data gets modified after - * generating checkpoint transaction id, connection will be reset to - * dirty when reconciliation marks the btree dirty on encountering the - * dirty page. - */ - conn->modified = false; - - /* - * Save the checkpoint session ID. - * - * We never do checkpoints in the default session (with id zero). - */ - WT_ASSERT(session, session->id != 0 && txn_global->checkpoint_id == 0); - txn_global->checkpoint_id = session->id; - - /* - * Remove the checkpoint transaction from the global table. - * - * This allows ordinary visibility checks to move forward because - * checkpoints often take a long time and only write to the metadata. - */ - __wt_writelock(session, txn_global->scan_rwlock); - txn_global->checkpoint_txnid = txn->id; - txn_global->checkpoint_pinned = WT_MIN(txn->id, txn->snap_min); - - /* - * Sanity check that the oldest ID hasn't moved on before we have - * cleared our entry. - */ - WT_ASSERT(session, - WT_TXNID_LE(txn_global->oldest_id, txn_state->id) && - WT_TXNID_LE(txn_global->oldest_id, txn_state->pinned_id)); - - /* - * Clear our entry from the global transaction session table. Any - * operation that needs to know about the ID for this checkpoint will - * consider the checkpoint ID in the global structure. Most operations - * can safely ignore the checkpoint ID (see the visible all check for - * details). - */ - txn_state->id = txn_state->pinned_id = WT_TXN_NONE; - __wt_writeunlock(session, txn_global->scan_rwlock); + WT_ASSERT(session, txn->isolation == WT_ISO_SNAPSHOT); /* * Unblock updates -- we can figure out that any updates to clean pages @@ -713,16 +768,6 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) cache->eviction_scrub_limit = 0.0; WT_STAT_CONN_SET(session, txn_checkpoint_scrub_target, 0); - /* - * Mark old checkpoints that are being deleted and figure out which - * trees we can skip in this checkpoint. - * - * Release clean trees. Any updates made after this point will not - * visible to the checkpoint transaction. - */ - WT_ERR(__checkpoint_apply(session, cfg, __checkpoint_mark_deletes)); - WT_ERR(__checkpoint_release_clean_trees(session)); - /* Tell logging that we have started a database checkpoint. */ if (full && logging) WT_ERR(__wt_txn_checkpoint_log( @@ -785,7 +830,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) /* Disable metadata tracking during the metadata checkpoint. */ saved_meta_next = session->meta_track_next; session->meta_track_next = NULL; - WT_WITH_METADATA_LOCK(session, ret, + WT_WITH_METADATA_LOCK(session, WT_WITH_DHANDLE(session, WT_SESSION_META_DHANDLE(session), ret = __wt_checkpoint(session, cfg))); @@ -829,12 +874,13 @@ err: /* * overwritten the checkpoint, so what ends up on disk is not * consistent. */ - if (ret != 0 && !conn->modified) + failed = ret != 0; + if (failed) conn->modified = true; session->isolation = txn->isolation = WT_ISO_READ_UNCOMMITTED; if (tracking) - WT_TRET(__wt_meta_track_off(session, false, ret != 0)); + WT_TRET(__wt_meta_track_off(session, false, failed)); cache->eviction_scrub_limit = 0.0; WT_STAT_CONN_SET(session, txn_checkpoint_scrub_target, 0); @@ -867,6 +913,13 @@ err: /* for (i = 0; i < session->ckpt_handle_next; ++i) { if (session->ckpt_handle[i] == NULL) continue; + /* + * If the operation failed, mark all trees dirty so they are + * included if a future checkpoint can succeed. + */ + if (failed) + WT_WITH_DHANDLE(session, session->ckpt_handle[i], + __checkpoint_fail_reset(session)); WT_WITH_DHANDLE(session, session->ckpt_handle[i], WT_TRET(__wt_session_release_btree(session))); } @@ -878,14 +931,40 @@ err: /* return (ret); } +/* + * __txn_checkpoint_wrapper -- + * Checkpoint wrapper. + */ +static int +__txn_checkpoint_wrapper(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_DECL_RET; + WT_TXN_GLOBAL *txn_global; + + txn_global = &S2C(session)->txn_global; + + WT_STAT_CONN_SET(session, txn_checkpoint_running, 1); + txn_global->checkpoint_running = true; + WT_FULL_BARRIER(); + + ret = __txn_checkpoint(session, cfg); + + WT_STAT_CONN_SET(session, txn_checkpoint_running, 0); + txn_global->checkpoint_running = false; + WT_FULL_BARRIER(); + + return (ret); +} + /* * __wt_txn_checkpoint -- * Checkpoint a database or a list of objects in the database. */ int -__wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) { WT_DECL_RET; + uint32_t mask; /* * Reset open cursors. Do this explicitly, even though it will happen @@ -895,13 +974,22 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) */ WT_RET(__wt_session_reset_cursors(session, false)); + /* Ensure the metadata table is open before taking any locks. */ + WT_RET(__wt_metadata_cursor(session, NULL)); + /* * Don't highjack the session checkpoint thread for eviction. * * Application threads are not generally available for potentially slow * operations, but checkpoint does enough I/O it may be called upon to * perform slow operations for the block manager. + * + * Application checkpoints wait until the checkpoint lock is available, + * compaction checkpoints don't. */ +#define WT_TXN_SESSION_MASK \ + (WT_SESSION_CAN_WAIT | WT_SESSION_NO_EVICTION) + mask = F_MASK(session, WT_TXN_SESSION_MASK); F_SET(session, WT_SESSION_CAN_WAIT | WT_SESSION_NO_EVICTION); /* @@ -911,14 +999,15 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * calls checkpoint directly, it can be tough to avoid. Serialize here * to ensure we don't get into trouble. */ - WT_STAT_CONN_SET(session, txn_checkpoint_running, 1); - - WT_WITH_CHECKPOINT_LOCK(session, ret, - ret = __txn_checkpoint(session, cfg)); - - WT_STAT_CONN_SET(session, txn_checkpoint_running, 0); + if (waiting) + WT_WITH_CHECKPOINT_LOCK(session, + ret = __txn_checkpoint_wrapper(session, cfg)); + else + WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, + ret = __txn_checkpoint_wrapper(session, cfg)); - F_CLR(session, WT_SESSION_CAN_WAIT | WT_SESSION_NO_EVICTION); + F_CLR(session, WT_TXN_SESSION_MASK); + F_SET(session, mask); return (ret); } @@ -1015,12 +1104,13 @@ __drop_to(WT_CKPT *ckptbase, const char *name, size_t len) } /* - * __checkpoint_lock_tree -- - * Acquire the locks required to checkpoint a tree. + * __checkpoint_lock_dirty_tree -- + * Decide whether the tree needs to be included in the checkpoint and if + * so, acquire the necessary locks. */ static int -__checkpoint_lock_tree(WT_SESSION_IMPL *session, - bool is_checkpoint, bool need_tracking, const char *cfg[]) +__checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, + bool is_checkpoint, bool force, bool need_tracking, const char *cfg[]) { WT_BTREE *btree; WT_CKPT *ckpt, *ckptbase; @@ -1057,7 +1147,7 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, * - On connection close when we know there can't be any races. */ WT_ASSERT(session, !need_tracking || - WT_IS_METADATA(session, dhandle) || WT_META_TRACKING(session)); + WT_IS_METADATA(dhandle) || WT_META_TRACKING(session)); /* Get the list of checkpoints for this file. */ WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, &ckptbase)); @@ -1127,7 +1217,7 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, * Hold the lock until we're done (blocking hot backups from starting), * we don't want to race with a future hot backup. */ - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); hot_backup_locked = true; if (conn->hot_backup) WT_CKPT_FOREACH(ckptbase, ckpt) { @@ -1144,6 +1234,14 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, ckpt->name); } + /* + * Mark old checkpoints that are being deleted and figure out which + * trees we can skip in this checkpoint. + */ + WT_ERR(__checkpoint_mark_skip(session, ckptbase, force)); + if (F_ISSET(btree, WT_BTREE_SKIP_CKPT)) + goto err; + /* * Lock the checkpoints that will be deleted. * @@ -1177,64 +1275,47 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, } /* - * There are special files: those being bulk-loaded, salvaged, upgraded - * or verified during the checkpoint. We have to do something for those - * objects because a checkpoint is an external name the application can - * reference and the name must exist no matter what's happening during - * the checkpoint. For bulk-loaded files, we could block until the load - * completes, checkpoint the partial load, or magic up an empty-file - * checkpoint. The first is too slow, the second is insane, so do the - * third. - * Salvage, upgrade and verify don't currently require any work, all - * three hold the schema lock, blocking checkpoints. If we ever want to - * fix that (and I bet we eventually will, at least for verify), we can - * copy the last checkpoint the file has. That works if we guarantee - * salvage, upgrade and verify act on objects with previous checkpoints - * (true if handles are closed/re-opened between object creation and a - * subsequent salvage, upgrade or verify operation). Presumably, - * salvage and upgrade will discard all previous checkpoints when they - * complete, which is fine with us. This change will require reference - * counting checkpoints, and once that's done, we should use checkpoint - * copy instead of forcing checkpoints on clean objects to associate - * names with checkpoints. + * There are special tree: those being bulk-loaded, salvaged, upgraded + * or verified during the checkpoint. They should never be part of a + * checkpoint: we will fail to lock them because the operations have + * exclusive access to the handles. Named checkpoints will fail in that + * case, ordinary checkpoints will skip files that cannot be opened + * normally. */ WT_ASSERT(session, !is_checkpoint || !F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); - WT_ASSERT(session, btree->ckpt == NULL); + WT_ASSERT(session, btree->ckpt == NULL && + !F_ISSET(btree, WT_BTREE_SKIP_CKPT)); btree->ckpt = ckptbase; return (0); err: if (hot_backup_locked) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); - __wt_meta_ckptlist_free(session, ckptbase); + __wt_meta_ckptlist_free(session, &ckptbase); __wt_free(session, name_alloc); return (ret); } /* - * __checkpoint_mark_deletes -- - * Figure out what old checkpoints will be deleted, and whether the - * checkpoint can be skipped entirely. + * __checkpoint_mark_skip -- + * Figure out whether the checkpoint can be skipped for a tree. */ static int -__checkpoint_mark_deletes( - WT_SESSION_IMPL *session, const char *cfg[]) +__checkpoint_mark_skip( + WT_SESSION_IMPL *session, WT_CKPT *ckptbase, bool force) { WT_BTREE *btree; - WT_CKPT *ckpt, *ckptbase; - WT_CONFIG_ITEM cval; + WT_CKPT *ckpt; const char *name; int deleted; - bool force; btree = S2BT(session); - ckptbase = btree->ckpt; /* * Check for clean objects not requiring a checkpoint. @@ -1260,12 +1341,7 @@ __checkpoint_mark_deletes( * to open the checkpoint in a cursor after taking any checkpoint, which * means it must exist. */ - force = false; F_CLR(btree, WT_BTREE_SKIP_CKPT); - if (!btree->modified && cfg != NULL) { - WT_RET(__wt_config_gets(session, cfg, "force", &cval)); - force = cval.val != 0; - } if (!btree->modified && !force) { deleted = 0; WT_CKPT_FOREACH(ckptbase, ckpt) @@ -1309,7 +1385,6 @@ __checkpoint_tree( WT_DATA_HANDLE *dhandle; WT_DECL_RET; WT_LSN ckptlsn; - int was_modified; bool fake_ckpt; WT_UNUSED(cfg); @@ -1320,7 +1395,6 @@ __checkpoint_tree( conn = S2C(session); dhandle = session->dhandle; fake_ckpt = false; - was_modified = btree->modified; /* * Set the checkpoint LSN to the maximum LSN so that if logging is @@ -1345,7 +1419,7 @@ __checkpoint_tree( * delete a physical checkpoint, and that will end in tears. */ if (is_checkpoint) - if (btree->bulk_load_ok) { + if (btree->original) { fake_ckpt = true; goto fake; } @@ -1421,7 +1495,7 @@ fake: /* * sync the file here or we could roll forward the metadata in * recovery and open a checkpoint that isn't yet durable. */ - if (WT_IS_METADATA(session, dhandle) || + if (WT_IS_METADATA(dhandle) || !F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR(__wt_checkpoint_sync(session, NULL)); @@ -1451,14 +1525,12 @@ err: /* * If the checkpoint didn't complete successfully, make sure the * tree is marked dirty. */ - if (ret != 0 && !btree->modified && was_modified) { + if (ret != 0) { btree->modified = true; - if (!S2C(session)->modified) - S2C(session)->modified = true; + S2C(session)->modified = true; } - __wt_meta_ckptlist_free(session, ckptbase); - btree->ckpt = NULL; + __wt_meta_ckptlist_free(session, &btree->ckpt); return (ret); } @@ -1477,7 +1549,8 @@ __checkpoint_presync(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); btree = S2BT(session); - WT_ASSERT(session, !btree->include_checkpoint_txn); + WT_ASSERT(session, btree->checkpoint_gen == + S2C(session)->txn_global.checkpoint_gen); btree->evict_walk_period = btree->evict_walk_saved; return (0); } @@ -1526,21 +1599,24 @@ __checkpoint_tree_helper(WT_SESSION_IMPL *session, const char *cfg[]) int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { + WT_CONFIG_ITEM cval; WT_DECL_RET; + bool force; /* Should not be called with a checkpoint handle. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); /* We must hold the metadata lock if checkpointing the metadata. */ - WT_ASSERT(session, !WT_IS_METADATA(session, session->dhandle) || + WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || F_ISSET(session, WT_SESSION_LOCKED_METADATA)); - WT_SAVE_DHANDLE(session, - ret = __checkpoint_lock_tree(session, true, true, cfg)); - WT_RET(ret); - WT_SAVE_DHANDLE(session, - ret = __checkpoint_mark_deletes(session, cfg)); + WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); + force = cval.val != 0; + WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree( + session, true, force, true, cfg)); WT_RET(ret); + if (F_ISSET(S2BT(session), WT_BTREE_SKIP_CKPT)) + return (0); return (__checkpoint_tree(session, true, cfg)); } @@ -1615,15 +1691,10 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (need_tracking) WT_RET(__wt_meta_track_on(session)); - WT_SAVE_DHANDLE(session, - ret = __checkpoint_lock_tree(session, false, need_tracking, NULL)); + WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree( + session, false, false, need_tracking, NULL)); WT_ASSERT(session, ret == 0); - if (ret == 0) { - WT_SAVE_DHANDLE(session, - ret = __checkpoint_mark_deletes(session, NULL)); - WT_ASSERT(session, ret == 0); - } - if (ret == 0) + if (ret == 0 && !F_ISSET(btree, WT_BTREE_SKIP_CKPT)) ret = __checkpoint_tree(session, false, NULL); if (need_tracking) diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index f1b78879d76..2931dc1ce82 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -269,7 +269,7 @@ __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, WT_ITEM ckpt_snapshot_unused; uint32_t ckpt_file, ckpt_offset; u_int ckpt_nsnapshot_unused; - const char *fmt = WT_UNCHECKED_STRING(IIIU); + const char *fmt = WT_UNCHECKED_STRING(IIIu); if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, &ckpt_file, &ckpt_offset, @@ -297,7 +297,7 @@ __wt_txn_checkpoint_log( uint8_t *end, *p; size_t recsize; uint32_t i, rectype = WT_LOGREC_CHECKPOINT; - const char *fmt = WT_UNCHECKED_STRING(IIIIU); + const char *fmt = WT_UNCHECKED_STRING(IIIIu); txn = &session->txn; ckpt_lsn = &txn->ckpt_lsn; @@ -368,14 +368,16 @@ __wt_txn_checkpoint_log( /* * If this full checkpoint completed successfully and there is - * no hot backup in progress, tell the logging subsystem the - * checkpoint LSN so that it can archive. Do not update the - * logging checkpoint LSN if this is during a clean connection - * close, only during a full checkpoint. A clean close may not - * update any metadata LSN and we do not want to archive in - * that case. + * no hot backup in progress and this is not recovery, tell + * the logging subsystem the checkpoint LSN so that it can + * archive. Do not update the logging checkpoint LSN if this + * is during a clean connection close, only during a full + * checkpoint. A clean close may not update any metadata LSN + * and we do not want to archive in that case. */ - if (!S2C(session)->hot_backup && txn->full_ckpt) + if (!S2C(session)->hot_backup && + !F_ISSET(S2C(session), WT_CONN_RECOVERING) && + txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); /* FALLTHROUGH */ @@ -551,6 +553,7 @@ __txn_printlog(WT_SESSION_IMPL *session, */ int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; WT_TXN_PRINTLOG_ARGS args; diff --git a/src/third_party/wiredtiger/src/txn/txn_nsnap.c b/src/third_party/wiredtiger/src/txn/txn_nsnap.c index 7ba0cc8700e..659570dbcd9 100644 --- a/src/third_party/wiredtiger/src/txn/txn_nsnap.c +++ b/src/third_party/wiredtiger/src/txn/txn_nsnap.c @@ -28,7 +28,6 @@ __nsnap_destroy(WT_SESSION_IMPL *session, WT_NAMED_SNAPSHOT *nsnap) static int __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) { - WT_DECL_RET; WT_NAMED_SNAPSHOT *found; WT_TXN_GLOBAL *txn_global; @@ -56,7 +55,7 @@ __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) __nsnap_destroy(session, found); WT_STAT_CONN_INCR(session, txn_snapshots_dropped); - return (ret); + return (0); } /* @@ -67,7 +66,6 @@ __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) static int __nsnap_drop_to(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, bool inclusive) { - WT_DECL_RET; WT_NAMED_SNAPSHOT *last, *nsnap, *prev; WT_TXN_GLOBAL *txn_global; uint64_t new_nsnap_oldest; @@ -134,7 +132,7 @@ __nsnap_drop_to(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, bool inclusive) new_nsnap_oldest == WT_TXN_NONE || !__wt_txn_visible_all(session, new_nsnap_oldest)); - return (ret); + return (0); } /* @@ -152,26 +150,45 @@ __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) const char *txn_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; - bool started_txn; + bool include_updates, started_txn; started_txn = false; nsnap_new = NULL; txn_global = &S2C(session)->txn_global; txn = &session->txn; + WT_RET(__wt_config_gets_def(session, cfg, "include_updates", 0, &cval)); + include_updates = cval.val != 0; + WT_RET(__wt_config_gets_def(session, cfg, "name", 0, &cval)); WT_ASSERT(session, cval.len != 0); if (!F_ISSET(txn, WT_TXN_RUNNING)) { + if (include_updates) + WT_RET_MSG(session, EINVAL, "A transaction must be " + "running to include updates in a named snapshot"); + WT_RET(__wt_txn_begin(session, txn_cfg)); started_txn = true; } - F_SET(txn, WT_TXN_READONLY); + if (!include_updates) + F_SET(txn, WT_TXN_READONLY); /* Save a copy of the transaction's snapshot. */ WT_ERR(__wt_calloc_one(session, &nsnap_new)); nsnap = nsnap_new; WT_ERR(__wt_strndup(session, cval.str, cval.len, &nsnap->name)); + + /* + * To include updates from a writing transaction, make sure a + * transaction ID has been allocated. + */ + if (include_updates) { + WT_ERR(__wt_txn_id_check(session)); + WT_ASSERT(session, txn->id != WT_TXN_NONE); + nsnap->id = txn->id; + } else + nsnap->id = WT_TXN_NONE; nsnap->pinned_id = WT_SESSION_TXN_STATE(session)->pinned_id; nsnap->snap_min = txn->snap_min; nsnap->snap_max = txn->snap_max; @@ -194,9 +211,9 @@ __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) if (TAILQ_EMPTY(&txn_global->nsnaph)) { WT_ASSERT(session, txn_global->nsnap_oldest_id == WT_TXN_NONE && !__wt_txn_visible_all(session, nsnap_new->pinned_id)); - __wt_readlock(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); txn_global->nsnap_oldest_id = nsnap_new->pinned_id; - __wt_readunlock(session, txn_global->scan_rwlock); + __wt_readunlock(session, &txn_global->scan_rwlock); } TAILQ_INSERT_TAIL(&txn_global->nsnaph, nsnap_new, q); WT_STAT_CONN_INCR(session, txn_snapshots_created); @@ -209,8 +226,7 @@ err: if (started_txn) { WT_TRET(__wt_txn_rollback(session, NULL)); WT_DIAGNOSTIC_YIELD; WT_ASSERT(session, !__wt_txn_visible_all(session, pinned_id)); - } else if (ret == 0) - F_SET(txn, WT_TXN_NAMED_SNAPSHOT); + } if (nsnap_new != NULL) __nsnap_destroy(session, nsnap_new); @@ -281,16 +297,16 @@ __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) if (session->ncursors > 0) WT_RET(__wt_session_copy_values(session)); - __wt_readlock(session, txn_global->nsnap_rwlock); + __wt_readlock(session, &txn_global->nsnap_rwlock); TAILQ_FOREACH(nsnap, &txn_global->nsnaph, q) if (WT_STRING_MATCH(nsnap->name, nameval->str, nameval->len)) { /* * Acquire the scan lock so the oldest ID can't move * forward without seeing our pinned ID. */ - __wt_readlock(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); txn_state->pinned_id = nsnap->pinned_id; - __wt_readunlock(session, txn_global->scan_rwlock); + __wt_readunlock(session, &txn_global->scan_rwlock); WT_ASSERT(session, !__wt_txn_visible_all( session, txn_state->pinned_id) && @@ -303,10 +319,15 @@ __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) memcpy(txn->snapshot, nsnap->snapshot, nsnap->snapshot_count * sizeof(*nsnap->snapshot)); + if (nsnap->id != WT_TXN_NONE) { + WT_ASSERT(session, txn->id == WT_TXN_NONE); + txn->id = nsnap->id; + F_SET(txn, WT_TXN_READONLY); + } F_SET(txn, WT_TXN_HAS_SNAPSHOT); break; } - __wt_readunlock(session, txn_global->nsnap_rwlock); + __wt_readunlock(session, &txn_global->nsnap_rwlock); if (nsnap == NULL) WT_RET_MSG(session, EINVAL, diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 65811aa3bf4..30932195b1e 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -93,7 +93,7 @@ __recovery_cursor(WT_SESSION_IMPL *session, WT_RECOVERY *r, "%s op %" PRIu32 " to file %" PRIu32 " at LSN %" PRIu32 \ "/%" PRIu32, \ cursor == NULL ? "Skipping" : "Applying", \ - optype, fileid, lsnp->l.file, lsnp->l.offset); \ + optype, fileid, (lsnp)->l.file, (lsnp)->l.offset); \ if (cursor == NULL) \ break @@ -248,8 +248,6 @@ static int __txn_commit_apply( WT_RECOVERY *r, WT_LSN *lsnp, const uint8_t **pp, const uint8_t *end) { - WT_UNUSED(lsnp); - /* The logging subsystem zero-pads records. */ while (*pp < end && **pp) WT_RET(__txn_op_apply(r, lsnp, pp, end)); @@ -503,7 +501,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * Pass WT_LOGSCAN_RECOVER so that old logs get truncated. */ r.metadata_only = false; - __wt_verbose(session, WT_VERB_RECOVERY, + __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, "Main recovery loop: starting at %" PRIu32 "/%" PRIu32, r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset); WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec)); diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index c2cf6c22aa4..93a96d44219 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -26,6 +26,7 @@ extern int __wt_optopt; /* character checked for validity */ extern int __wt_optreset; /* reset getopt */ extern char *__wt_optarg; /* argument associated with option */ +int util_alter(WT_SESSION *, int, char *[]); int util_backup(WT_SESSION *, int, char *[]); int util_cerr(WT_CURSOR *, const char *, int); int util_compact(WT_SESSION *, int, char *[]); @@ -39,7 +40,6 @@ int util_flush(WT_SESSION *, const char *); int util_list(WT_SESSION *, int, char *[]); int util_load(WT_SESSION *, int, char *[]); int util_loadtext(WT_SESSION *, int, char *[]); -char *util_name(WT_SESSION *, const char *, const char *); int util_printlog(WT_SESSION *, int, char *[]); int util_read(WT_SESSION *, int, char *[]); int util_read_line(WT_SESSION *, ULINE *, bool, bool *); @@ -48,6 +48,8 @@ int util_rename(WT_SESSION *, int, char *[]); int util_salvage(WT_SESSION *, int, char *[]); int util_stat(WT_SESSION *, int, char *[]); int util_str2recno(WT_SESSION *, const char *p, uint64_t *recnop); +int util_truncate(WT_SESSION *, int, char *[]); int util_upgrade(WT_SESSION *, int, char *[]); +char *util_uri(WT_SESSION *, const char *, const char *); int util_verify(WT_SESSION *, int, char *[]); int util_write(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_alter.c b/src/third_party/wiredtiger/src/utilities/util_alter.c new file mode 100644 index 00000000000..ef01a1ed826 --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_alter.c @@ -0,0 +1,53 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_alter(WT_SESSION *session, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char **configp; + + while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + + /* The remaining arguments are uri/string pairs. */ + if (argc % 2 != 0) + return (usage()); + + for (configp = argv; + configp != NULL && *configp != NULL; configp += 2) + if ((ret = session->alter( + session, configp[0], configp[1])) != 0) { + (void)util_err(session, ret, + "session.alter: %s, %s", configp[0], configp[1]); + return (1); + } + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "alter uri configuration ...\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_backup.c b/src/third_party/wiredtiger/src/utilities/util_backup.c index 5dc9671fb45..f1b31f7621a 100644 --- a/src/third_party/wiredtiger/src/utilities/util_backup.c +++ b/src/third_party/wiredtiger/src/utilities/util_backup.c @@ -109,9 +109,14 @@ copy(WT_SESSION *session, const char *directory, const char *name) /* Build the target pathname. */ len = strlen(directory) + strlen(name) + 2; - if ((to = malloc(len)) == NULL) - goto memerr; - (void)snprintf(to, len, "%s/%s", directory, name); + if ((to = malloc(len)) == NULL) { + fprintf(stderr, "%s: %s\n", progname, strerror(errno)); + return (1); + } + if ((ret = __wt_snprintf(to, len, "%s/%s", directory, name)) != 0) { + fprintf(stderr, "%s: %s\n", progname, strerror(ret)); + goto err; + } if (verbose && printf("Backing up %s/%s to %s\n", home, name, to) < 0) { fprintf(stderr, "%s: %s\n", progname, strerror(EIO)); @@ -126,11 +131,7 @@ copy(WT_SESSION *session, const char *directory, const char *name) fprintf(stderr, "%s/%s to %s: backup copy: %s\n", home, name, to, session->strerror(session, ret)); - if (0) { -memerr: fprintf(stderr, "%s: %s\n", progname, strerror(errno)); - } err: free(to); - return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_compact.c b/src/third_party/wiredtiger/src/utilities/util_compact.c index c114eb207fa..e469b4dce6e 100644 --- a/src/third_party/wiredtiger/src/utilities/util_compact.c +++ b/src/third_party/wiredtiger/src/utilities/util_compact.c @@ -30,21 +30,13 @@ util_compact(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - if ((ret = session->compact(session, uri, NULL)) != 0) { - fprintf(stderr, "%s: compact(%s): %s\n", - progname, uri, session->strerror(session, ret)); - goto err; - } - - if (0) { -err: ret = 1; - } + if ((ret = session->compact(session, uri, NULL)) != 0) + (void)util_err(session, ret, "session.compact: %s", uri); free(uri); - return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_create.c b/src/third_party/wiredtiger/src/utilities/util_create.c index 4e609736f2d..7c22a67792b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_create.c +++ b/src/third_party/wiredtiger/src/utilities/util_create.c @@ -15,9 +15,9 @@ util_create(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - const char *config, *uri; + char *config, *uri; - config = NULL; + config = uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "c:")) != EOF) switch (ch) { case 'c': /* command-line configuration */ @@ -35,12 +35,14 @@ util_create(WT_SESSION *session, int argc, char *argv[]) if (argc != 1) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); if ((ret = session->create(session, uri, config)) != 0) - return (util_err(session, ret, "%s: session.create", uri)); - return (0); + (void)util_err(session, ret, "session.create: %s", uri); + + free(uri); + return (ret); } static int diff --git a/src/third_party/wiredtiger/src/utilities/util_drop.c b/src/third_party/wiredtiger/src/utilities/util_drop.c index ba41445dfb6..456005d445d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_drop.c +++ b/src/third_party/wiredtiger/src/utilities/util_drop.c @@ -15,8 +15,9 @@ util_drop(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - char *name; + char *uri; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -30,12 +31,13 @@ util_drop(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the uri. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - ret = session->drop(session, name, "force"); + if ((ret = session->drop(session, uri, "force")) != 0) + (void)util_err(session, ret, "session.drop: %s", uri); - free(name); + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 7dde13ee837..955148b7d46 100644 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -6,10 +6,14 @@ * See the file LICENSE for redistribution information. */ +#include #include "util.h" #include "util_dump.h" -static int dump_config(WT_SESSION *, const char *, bool, bool); +#define STRING_MATCH_CONFIG(s, item) \ + (strncmp(s, (item).str, (item).len) == 0 && (s)[(item).len] == '\0') + +static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool); static int dump_json_begin(WT_SESSION *); static int dump_json_end(WT_SESSION *); static int dump_json_separator(WT_SESSION *); @@ -17,7 +21,8 @@ static int dump_json_table_end(WT_SESSION *); static int dump_prefix(WT_SESSION *, bool, bool); static int dump_record(WT_CURSOR *, bool, bool); static int dump_suffix(WT_SESSION *, bool); -static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *, bool); +static int dump_table_config( + WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, bool); static int dump_table_parts_config( WT_SESSION *, WT_CURSOR *, const char *, const char *, bool); static int dup_json_string(const char *, char **); @@ -32,10 +37,11 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) size_t len; int ch, i; bool hex, json, reverse; - char *checkpoint, *config, *name; + char *checkpoint, *config, *p, *simpleuri, *uri; hex = json = reverse = false; - checkpoint = config = name = NULL; + checkpoint = config = simpleuri = uri = NULL; + cursor = NULL; while ((ch = __wt_getopt(progname, argc, argv, "c:f:jrx")) != EOF) switch (ch) { case 'c': @@ -75,21 +81,19 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) return (usage()); if (json && - ((ret = dump_json_begin(session)) != 0 || - (ret = dump_prefix(session, hex, json)) != 0)) + (dump_json_begin(session) != 0 || + dump_prefix(session, hex, json) != 0)) goto err; for (i = 0; i < argc; i++) { if (json && i > 0) - if ((ret = dump_json_separator(session)) != 0) + if (dump_json_separator(session) != 0) goto err; - free(name); - name = NULL; - - if ((name = util_name(session, argv[i], "table")) == NULL) - goto err; + free(uri); + free(simpleuri); + uri = simpleuri = NULL; - if (dump_config(session, name, hex, json) != 0) + if ((uri = util_uri(session, argv[i], "table")) == NULL) goto err; len = @@ -109,18 +113,34 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) (void)strcat(config, json ? "dump=json" : (hex ? "dump=hex" : "dump=print")); if ((ret = session->open_cursor( - session, name, NULL, config, &cursor)) != 0) { + session, uri, NULL, config, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(%s) failed: %s\n", - progname, name, session->strerror(session, ret)); + progname, uri, session->strerror(session, ret)); + goto err; + } + + if ((simpleuri = strdup(uri)) == NULL) { + (void)util_err(session, errno, NULL); goto err; } + if ((p = strchr(simpleuri, '(')) != NULL) + *p = '\0'; + if (dump_config(session, simpleuri, cursor, hex, json) != 0) + goto err; - if ((ret = dump_record(cursor, reverse, json)) != 0) + if (dump_record(cursor, reverse, json) != 0) goto err; - if (json && (ret = dump_json_table_end(session)) != 0) + if (json && dump_json_table_end(session) != 0) goto err; + + ret = cursor->close(cursor); + cursor = NULL; + if (ret != 0) { + (void)util_err(session, ret, NULL); + goto err; + } } - if (json && ((ret = dump_json_end(session)) != 0)) + if (json && dump_json_end(session) != 0) goto err; if (0) { @@ -128,8 +148,12 @@ err: ret = 1; } free(config); - free(name); - + free(uri); + free(simpleuri); + if (cursor != NULL && (ret = cursor->close(cursor)) != 0) { + (void)util_err(session, ret, NULL); + ret = 1; + } return (ret); } @@ -138,15 +162,16 @@ err: ret = 1; * Dump the config for the uri. */ static int -dump_config(WT_SESSION *session, const char *uri, bool hex, bool json) +dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex, + bool json) { - WT_CURSOR *cursor; + WT_CURSOR *mcursor; WT_DECL_RET; int tret; /* Open a metadata cursor. */ if ((ret = session->open_cursor( - session, "metadata:create", NULL, NULL, &cursor)) != 0) { + session, "metadata:create", NULL, NULL, &mcursor)) != 0) { fprintf(stderr, "%s: %s: session.open_cursor: %s\n", progname, "metadata:create", session->strerror(session, ret)); return (1); @@ -156,10 +181,11 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json) * want to output a header if the user entered the wrong name. This is * where we find out a table doesn't exist, use a simple error message. */ - cursor->set_key(cursor, uri); - if ((ret = cursor->search(cursor)) == 0) { + mcursor->set_key(mcursor, uri); + if ((ret = mcursor->search(mcursor)) == 0) { if ((!json && dump_prefix(session, hex, json) != 0) || - dump_table_config(session, cursor, uri, json) != 0 || + dump_table_config(session, mcursor, cursor, + uri, json) != 0 || dump_suffix(session, json) != 0) ret = 1; } else if (ret == WT_NOTFOUND) @@ -167,8 +193,8 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json) else ret = util_err(session, ret, "%s", uri); - if ((tret = cursor->close(cursor)) != 0) { - tret = util_cerr(cursor, "close", tret); + if ((tret = mcursor->close(mcursor)) != 0) { + tret = util_cerr(mcursor, "close", tret); if (ret == 0) ret = tret; } @@ -224,17 +250,127 @@ dump_json_table_end(WT_SESSION *session) return (0); } +/* + * dump_add_config + * Add a formatted config string to an output buffer. + */ +static int +dump_add_config(WT_SESSION *session, char **bufp, size_t *leftp, + const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) +{ + WT_DECL_RET; + size_t n; + va_list ap; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_set(*bufp, *leftp, &n, fmt, ap); + va_end(ap); + if (ret != 0) + return (util_err(session, ret, NULL)); + *bufp += n; + *leftp -= (size_t)n; + return (0); +} + +/* + * dump_projection -- + * Create a new config containing projection information. + */ +static int +dump_projection(WT_SESSION *session, const char *config, WT_CURSOR *cursor, + char **newconfigp) +{ + WT_DECL_RET; + WT_CONFIG_ITEM key, value; + WT_CONFIG_PARSER *parser; + WT_EXTENSION_API *wt_api; + size_t len, vallen; + int nkeys; + char *newconfig; + const char *keyformat, *p; + + len = strlen(config) + strlen(cursor->value_format) + + strlen(cursor->uri) + 20; + if ((newconfig = malloc(len)) == NULL) + return util_err(session, errno, NULL); + *newconfigp = newconfig; + wt_api = session->connection->get_extension_api(session->connection); + if ((ret = wt_api->config_parser_open(wt_api, session, config, + strlen(config), &parser)) != 0) + return (util_err( + session, ret, "WT_EXTENSION_API.config_parser_open")); + keyformat = cursor->key_format; + for (nkeys = 0; *keyformat; keyformat++) + if (!__wt_isdigit((u_char)*keyformat)) + nkeys++; + + /* + * Copy the configuration, replacing some fields to match the + * projection. + */ + while ((ret = parser->next(parser, &key, &value)) == 0) { + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s=", (int)key.len, key.str)); + if (STRING_MATCH_CONFIG("value_format", key)) + WT_RET(dump_add_config(session, &newconfig, &len, + "%s", cursor->value_format)); + else if (STRING_MATCH_CONFIG("columns", key)) { + /* copy names of keys */ + p = value.str; + vallen = value.len; + while (vallen > 0) { + if ((*p == ',' || *p == ')') && --nkeys == 0) + break; + p++; + vallen--; + } + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s", (int)(p - value.str), value.str)); + + /* copy names of projected values */ + p = strchr(cursor->uri, '('); + assert(p != NULL); + assert(p[strlen(p) - 1] == ')'); + p++; + if (*p != ')') + WT_RET(dump_add_config(session, &newconfig, + &len, "%s", ",")); + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s),", (int)(strlen(p) - 1), p)); + } else if (value.type == WT_CONFIG_ITEM_STRING && + value.len != 0) + WT_RET(dump_add_config(session, &newconfig, &len, + "\"%.*s\",", (int)value.len, value.str)); + else + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s,", (int)value.len, value.str)); + } + if (ret != WT_NOTFOUND) + return (util_err(session, ret, "WT_CONFIG_PARSER.next")); + + assert(len > 0); + if ((ret = parser->close(parser)) != 0) + return (util_err( + session, ret, "WT_CONFIG_PARSER.close")); + + return (0); +} + /* * dump_table_config -- * Dump the config for a table. */ static int dump_table_config( - WT_SESSION *session, WT_CURSOR *cursor, const char *uri, bool json) + WT_SESSION *session, WT_CURSOR *mcursor, WT_CURSOR *cursor, + const char *uri, bool json) { WT_DECL_RET; + char *proj_config; const char *name, *v; + proj_config = NULL; /* Get the table name. */ if ((name = strchr(uri, ':')) == NULL) { fprintf(stderr, "%s: %s: corrupted uri\n", progname, uri); @@ -246,20 +382,25 @@ dump_table_config( * Dump out the config information: first, dump the uri entry itself, * it overrides all subsequent configurations. */ - cursor->set_key(cursor, uri); - if ((ret = cursor->search(cursor)) != 0) - return (util_cerr(cursor, "search", ret)); - if ((ret = cursor->get_value(cursor, &v)) != 0) - return (util_cerr(cursor, "get_value", ret)); - - WT_RET(print_config(session, uri, v, json, true)); + mcursor->set_key(mcursor, uri); + if ((ret = mcursor->search(mcursor)) != 0) + return (util_cerr(mcursor, "search", ret)); + if ((ret = mcursor->get_value(mcursor, &v)) != 0) + return (util_cerr(mcursor, "get_value", ret)); + + if (strchr(cursor->uri, '(') != NULL) { + WT_ERR(dump_projection(session, v, cursor, &proj_config)); + v = proj_config; + } + WT_ERR(print_config(session, uri, v, json, true)); - WT_RET(dump_table_parts_config( - session, cursor, name, "colgroup:", json)); - WT_RET(dump_table_parts_config( - session, cursor, name, "index:", json)); + WT_ERR(dump_table_parts_config( + session, mcursor, name, "colgroup:", json)); + WT_ERR(dump_table_parts_config( + session, mcursor, name, "index:", json)); - return (0); +err: free(proj_config); + return (ret); } /* @@ -295,9 +436,11 @@ dump_table_parts_config(WT_SESSION *session, WT_CURSOR *cursor, len = strlen(entry) + strlen(name) + 1; if ((uriprefix = malloc(len)) == NULL) - return util_err(session, errno, NULL); - - snprintf(uriprefix, len, "%s%s", entry, name); + return (util_err(session, errno, NULL)); + if ((ret = __wt_snprintf(uriprefix, len, "%s%s", entry, name)) != 0) { + free(uriprefix); + return (util_err(session, ret, NULL)); + } /* * Search the file looking for column group and index key/value pairs: @@ -364,17 +507,18 @@ dump_prefix(WT_SESSION *session, bool hex, bool json) (void)wiredtiger_version(&vmajor, &vminor, &vpatch); + if (json && printf( + " \"%s\" : \"%d (%d.%d.%d)\",\n", + DUMP_JSON_VERSION_MARKER, DUMP_JSON_CURRENT_VERSION, + vmajor, vminor, vpatch) < 0) + return (util_err(session, EIO, NULL)); + if (!json && (printf( "WiredTiger Dump (WiredTiger Version %d.%d.%d)\n", vmajor, vminor, vpatch) < 0 || printf("Format=%s\n", hex ? "hex" : "print") < 0 || printf("Header\n") < 0)) return (util_err(session, EIO, NULL)); - else if (json && printf( - " \"%s\" : \"%d (%d.%d.%d)\",\n", - DUMP_JSON_VERSION_MARKER, DUMP_JSON_CURRENT_VERSION, - vmajor, vminor, vpatch) < 0) - return (util_err(session, EIO, NULL)); return (0); } diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index e91dbfce05b..f19ba4d1f97 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -19,10 +19,10 @@ util_list(WT_SESSION *session, int argc, char *argv[]) WT_DECL_RET; int ch; bool cflag, vflag; - char *name; + char *uri; cflag = vflag = false; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "cv")) != EOF) switch (ch) { case 'c': @@ -42,17 +42,16 @@ util_list(WT_SESSION *session, int argc, char *argv[]) case 0: break; case 1: - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); break; default: return (usage()); } - ret = list_print(session, name, cflag, vflag); - - free(name); + ret = list_print(session, uri, cflag, vflag); + free(uri); return (ret); } @@ -99,7 +98,7 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) * List the high-level objects in the database. */ static int -list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) +list_print(WT_SESSION *session, const char *uri, bool cflag, bool vflag) { WT_CURSOR *cursor; WT_DECL_RET; @@ -120,7 +119,7 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) ret, "%s: WT_SESSION.open_cursor", WT_METADATA_URI)); } - found = name == NULL; + found = uri == NULL; while ((ret = cursor->next(cursor)) == 0) { /* Get the key. */ if ((ret = cursor->get_key(cursor, &key)) != 0) @@ -129,8 +128,8 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) /* * If a name is specified, only show objects that match. */ - if (name != NULL) { - if (!WT_PREFIX_MATCH(key, name)) + if (uri != NULL) { + if (!WT_PREFIX_MATCH(key, uri)) continue; found = true; } @@ -161,7 +160,7 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) if (ret != WT_NOTFOUND) return (util_cerr(cursor, "next", ret)); if (!found) { - fprintf(stderr, "%s: %s: not found\n", progname, name); + fprintf(stderr, "%s: %s: not found\n", progname, uri); return (1); } diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index ac18df80851..d2f00402217 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -80,8 +80,8 @@ util_load(WT_SESSION *session, int argc, char *argv[]) if (no_overwrite) flags |= LOAD_JSON_NO_OVERWRITE; return (util_load_json(session, filename, flags)); - } else - return (load_dump(session)); + } + return (load_dump(session)); } /* @@ -120,13 +120,15 @@ load_dump(WT_SESSION *session) goto err; /* Open the insert cursor. */ - (void)snprintf(config, sizeof(config), + if ((ret = __wt_snprintf(config, sizeof(config), "dump=%s%s%s", hex ? "hex" : "print", - append ? ",append" : "", no_overwrite ? ",overwrite=false" : ""); + append ? ",append" : "", + no_overwrite ? ",overwrite=false" : "")) != 0) + return (util_err(session, ret, NULL)); if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) { - ret = util_err(session, ret, "%s: session.open", uri); + ret = util_err(session, ret, "%s: session.open_cursor", uri); goto err; } @@ -472,6 +474,7 @@ config_update(WT_SESSION *session, char **list) static int config_rename(WT_SESSION *session, char **urip, const char *name) { + WT_DECL_RET; size_t len; char *buf, *p; @@ -490,7 +493,9 @@ config_rename(WT_SESSION *session, char **urip, const char *name) } *p = '\0'; p = strchr(p + 1, ':'); - snprintf(buf, len, "%s:%s%s", *urip, name, p == NULL ? "" : p); + if ((ret = __wt_snprintf( + buf, len, "%s:%s%s", *urip, name, p == NULL ? "" : p)) != 0) + return (util_err(session, ret, NULL)); *urip = buf; return (0); diff --git a/src/third_party/wiredtiger/src/utilities/util_load_json.c b/src/third_party/wiredtiger/src/utilities/util_load_json.c index 020a4ed9ba9..c693e2b7651 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load_json.c +++ b/src/third_party/wiredtiger/src/utilities/util_load_json.c @@ -145,6 +145,7 @@ static int json_kvraw_append(WT_SESSION *session, JSON_INPUT_STATE *ins, const char *str, size_t len) { + WT_DECL_RET; size_t needsize; char *tmp; @@ -152,11 +153,15 @@ json_kvraw_append(WT_SESSION *session, needsize = strlen(ins->kvraw) + len + 2; if ((tmp = malloc(needsize)) == NULL) return (util_err(session, errno, NULL)); - snprintf(tmp, needsize, "%s %.*s", ins->kvraw, (int)len, str); + WT_ERR(__wt_snprintf( + tmp, needsize, "%s %.*s", ins->kvraw, (int)len, str)); free(ins->kvraw); ins->kvraw = tmp; } return (0); + +err: free(tmp); + return (util_err(session, ret, NULL)); } /* @@ -181,7 +186,7 @@ json_strdup(WT_SESSION *session, JSON_INPUT_STATE *ins, char **resultp) goto err; } resultlen += 1; - if ((result = (char *)malloc((size_t)resultlen)) == NULL) { + if ((result = malloc((size_t)resultlen)) == NULL) { ret = util_err(session, errno, NULL); goto err; } @@ -236,13 +241,16 @@ json_data(WT_SESSION *session, goto err; uri = clp->list[0]; - (void)snprintf(config, sizeof(config), + if ((ret = __wt_snprintf(config, sizeof(config), "dump=json%s%s", LF_ISSET(LOAD_JSON_APPEND) ? ",append" : "", - LF_ISSET(LOAD_JSON_NO_OVERWRITE) ? ",overwrite=false" : ""); + LF_ISSET(LOAD_JSON_NO_OVERWRITE) ? ",overwrite=false" : "")) != 0) { + ret = util_err(session, ret, NULL); + goto err; + } if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) { - ret = util_err(session, ret, "%s: session.open", uri); + ret = util_err(session, ret, "%s: session.open_cursor", uri); goto err; } keyformat = cursor->key_format; @@ -256,7 +264,7 @@ json_data(WT_SESSION *session, nfield = 0; JSON_EXPECT(session, ins, '{'); if (ins->kvraw == NULL) { - if ((ins->kvraw = (char *)malloc(1)) == NULL) { + if ((ins->kvraw = malloc(1)) == NULL) { ret = util_err(session, errno, NULL); goto err; } @@ -358,8 +366,11 @@ json_top_level(WT_SESSION *session, JSON_INPUT_STATE *ins, uint32_t flags) while (json_peek(session, ins) == 's') { JSON_EXPECT(session, ins, 's'); tableuri = realloc(tableuri, ins->toklen); - snprintf(tableuri, ins->toklen, "%.*s", - (int)(ins->toklen - 2), ins->tokstart + 1); + if ((ret = __wt_snprintf(tableuri, ins->toklen, + "%.*s", (int)(ins->toklen - 2), ins->tokstart + 1)) != 0) { + ret = util_err(session, ret, NULL); + goto err; + } JSON_EXPECT(session, ins, ':'); if (!hasversion) { if (strcmp(tableuri, DUMP_JSON_VERSION_MARKER) != 0) { diff --git a/src/third_party/wiredtiger/src/utilities/util_loadtext.c b/src/third_party/wiredtiger/src/utilities/util_loadtext.c index f9c5b6e9a1f..7602d43f8c9 100644 --- a/src/third_party/wiredtiger/src/utilities/util_loadtext.c +++ b/src/third_party/wiredtiger/src/utilities/util_loadtext.c @@ -15,9 +15,11 @@ static int usage(void); int util_loadtext(WT_SESSION *session, int argc, char *argv[]) { + WT_DECL_RET; int ch; - const char *uri; + char *uri; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "f:")) != EOF) switch (ch) { case 'f': /* input file */ @@ -35,10 +37,13 @@ util_loadtext(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the uri. */ if (argc != 1) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - return (text(session, uri)); + ret = text(session, uri); + + free(uri); + return (ret); } /* @@ -61,7 +66,7 @@ text(WT_SESSION *session, const char *uri) */ if ((ret = session->open_cursor( session, uri, NULL, "append,overwrite", &cursor)) != 0) - return (util_err(session, ret, "%s: session.open", uri)); + return (util_err(session, ret, "%s: session.open_cursor", uri)); /* * We're about to load strings, make sure the formats match. diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 2054b94e3ce..c6f225bb667 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -20,7 +20,43 @@ static const char *command; /* Command name */ #define REC_LOGOFF "log=(enabled=false)" #define REC_RECOVER "log=(recover=on)" -static int usage(void); +static void +usage(void) +{ + fprintf(stderr, + "WiredTiger Data Engine (version %d.%d)\n", + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); + fprintf(stderr, + "global options:\n" + "\t" "-C\t" "wiredtiger_open configuration\n" + "\t" "-h\t" "database directory\n" + "\t" "-L\t" "turn logging off for debug-mode\n" + "\t" "-R\t" "run recovery if configured\n" + "\t" "-V\t" "display library version and exit\n" + "\t" "-v\t" "verbose\n"); + fprintf(stderr, + "commands:\n" + "\t" "alter\t alter an object\n" + "\t" "backup\t database backup\n" + "\t" "compact\t compact an object\n" + "\t" "copyright copyright information\n" + "\t" "create\t create an object\n" + "\t" "drop\t drop an object\n" + "\t" "dump\t dump an object\n" + "\t" "list\t list database objects\n" + "\t" "load\t load an object\n" + "\t" "loadtext load an object from a text file\n" + "\t" "printlog display the database log\n" + "\t" "read\t read values from an object\n" + "\t" "rebalance rebalance an object\n" + "\t" "rename\t rename an object\n" + "\t" "salvage\t salvage a file\n" + "\t" "stat\t display statistics for an object\n" + "\t" "truncate truncate an object, removing all content\n" + "\t" "upgrade\t upgrade an object\n" + "\t" "verify\t verify an object\n" + "\t" "write\t write values to an object\n"); +} int main(int argc, char *argv[]) @@ -73,8 +109,9 @@ main(int argc, char *argv[]) cmd_config = __wt_optarg; break; case 'E': /* secret key */ + free(secretkey); /* lint: set more than once */ if ((secretkey = strdup(__wt_optarg)) == NULL) { - ret = util_err(NULL, errno, NULL); + (void)util_err(NULL, errno, NULL); goto err; } memset(__wt_optarg, 0, strlen(__wt_optarg)); @@ -92,24 +129,27 @@ main(int argc, char *argv[]) break; case 'V': /* version */ printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); - return (EXIT_SUCCESS); + goto done; case 'v': /* verbose */ verbose = true; break; case '?': default: - return (usage()); + usage(); + goto err; } if (logoff && recover) { fprintf(stderr, "Only one of -L and -R is allowed.\n"); - return (EXIT_FAILURE); + goto err; } argc -= __wt_optind; argv += __wt_optind; /* The next argument is the command name. */ - if (argc < 1) - return (usage()); + if (argc < 1) { + usage(); + goto err; + } command = argv[0]; /* Reset getopt. */ @@ -117,6 +157,10 @@ main(int argc, char *argv[]) func = NULL; switch (command[0]) { + case 'a': + if (strcmp(command, "alter") == 0) + func = util_alter; + break; case 'b': if (strcmp(command, "backup") == 0) func = util_backup; @@ -126,7 +170,7 @@ main(int argc, char *argv[]) func = util_compact; else if (strcmp(command, "copyright") == 0) { util_copyright(); - return (EXIT_SUCCESS); + goto done; } else if (strcmp(command, "create") == 0) { func = util_create; config = "create"; @@ -171,6 +215,10 @@ main(int argc, char *argv[]) config = "statistics=(all)"; } break; + case 't' : + if (strcmp(command, "truncate") == 0) + func = util_truncate; + break; case 'u': if (strcmp(command, "upgrade") == 0) func = util_upgrade; @@ -186,8 +234,10 @@ main(int argc, char *argv[]) default: break; } - if (func == NULL) - return (usage()); + if (func == NULL) { + usage(); + goto err; + } /* Build the configuration string. */ len = 10; /* some slop */ @@ -204,30 +254,39 @@ main(int argc, char *argv[]) } len += strlen(rec_config); if ((p = malloc(len)) == NULL) { - ret = util_err(NULL, errno, NULL); + (void)util_err(NULL, errno, NULL); goto err; } - (void)snprintf(p, len, "%s,%s,%s%s%s%s", + if ((ret = __wt_snprintf(p, len, "%s,%s,%s%s%s%s", config == NULL ? "" : config, - cmd_config == NULL ? "" : cmd_config, rec_config, p1, p2, p3); + cmd_config == NULL ? "" : cmd_config, + rec_config, p1, p2, p3)) != 0) { + (void)util_err(NULL, ret, NULL); + goto err; + } config = p; /* Open the database and a session. */ if ((ret = wiredtiger_open(home, verbose ? verbose_handler : NULL, config, &conn)) != 0) { - ret = util_err(NULL, ret, NULL); + (void)util_err(NULL, ret, NULL); goto err; } if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { - ret = util_err(NULL, ret, NULL); + (void)util_err(NULL, ret, NULL); goto err; } /* Call the function. */ ret = func(session, argc, argv); + if (0) { +err: ret = 1; + } +done: + /* Close the database. */ -err: if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) + if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) ret = tret; free(p); @@ -236,51 +295,14 @@ err: if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); } -static int -usage(void) -{ - fprintf(stderr, - "WiredTiger Data Engine (version %d.%d)\n", - WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); - fprintf(stderr, - "global options:\n" - "\t" "-C\t" "wiredtiger_open configuration\n" - "\t" "-h\t" "database directory\n" - "\t" "-L\t" "turn logging off for debug-mode\n" - "\t" "-R\t" "run recovery if configured\n" - "\t" "-V\t" "display library version and exit\n" - "\t" "-v\t" "verbose\n"); - fprintf(stderr, - "commands:\n" - "\t" "backup\t database backup\n" - "\t" "compact\t compact an object\n" - "\t" "copyright copyright information\n" - "\t" "create\t create an object\n" - "\t" "drop\t drop an object\n" - "\t" "dump\t dump an object\n" - "\t" "list\t list database objects\n" - "\t" "load\t load an object\n" - "\t" "loadtext load an object from a text file\n" - "\t" "printlog display the database log\n" - "\t" "read\t read values from an object\n" - "\t" "rebalance rebalance an object\n" - "\t" "rename\t rename an object\n" - "\t" "salvage\t salvage a file\n" - "\t" "stat\t display statistics for an object\n" - "\t" "upgrade\t upgrade an object\n" - "\t" "verify\t verify an object\n" - "\t" "write\t write values to an object\n"); - - return (EXIT_FAILURE); -} - /* - * util_name -- + * util_uri -- * Build a name. */ char * -util_name(WT_SESSION *session, const char *s, const char *type) +util_uri(WT_SESSION *session, const char *s, const char *type) { + WT_DECL_RET; size_t len; char *name; @@ -304,8 +326,12 @@ util_name(WT_SESSION *session, const char *s, const char *type) * the default type for the operation. */ if (strchr(s, ':') != NULL) - strcpy(name, s); + WT_ERR(__wt_snprintf(name, len, "%s", s)); else - snprintf(name, len, "%s:%s", type, s); + WT_ERR(__wt_snprintf(name, len, "%s:%s", type, s)); return (name); + +err: free(name); + (void)util_err(session, ret, NULL); + return (NULL); } diff --git a/src/third_party/wiredtiger/src/utilities/util_misc.c b/src/third_party/wiredtiger/src/utilities/util_misc.c index 0905bfa97be..e26185a0096 100644 --- a/src/third_party/wiredtiger/src/utilities/util_misc.c +++ b/src/third_party/wiredtiger/src/utilities/util_misc.c @@ -140,7 +140,10 @@ util_flush(WT_SESSION *session, const char *uri) if ((buf = malloc(len)) == NULL) return (util_err(session, errno, NULL)); - (void)snprintf(buf, len, "target=(\"%s\")", uri); + if ((ret = __wt_snprintf(buf, len, "target=(\"%s\")", uri)) != 0) { + free(buf); + return (util_err(session, ret, NULL)); + } ret = session->checkpoint(session, buf); free(buf); diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index e7fa2134934..5f3ed43905b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -14,8 +14,8 @@ int util_printlog(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; - int ch; uint32_t flags; + int ch; flags = 0; while ((ch = __wt_getopt(progname, argc, argv, "f:x")) != EOF) @@ -41,17 +41,9 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) if (argc != 0) return (usage()); - ret = __wt_txn_printlog(session, flags); - - if (ret != 0) { - fprintf(stderr, "%s: printlog failed: %s\n", - progname, session->strerror(session, ret)); - goto err; - } + if ((ret = __wt_txn_printlog(session, flags)) != 0) + (void)util_err(session, ret, "printlog"); - if (0) { -err: ret = 1; - } return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_read.c b/src/third_party/wiredtiger/src/utilities/util_read.c index 2e766377aa9..393949b6a1c 100644 --- a/src/third_party/wiredtiger/src/utilities/util_read.c +++ b/src/third_party/wiredtiger/src/utilities/util_read.c @@ -18,8 +18,9 @@ util_read(WT_SESSION *session, int argc, char *argv[]) uint64_t recno; int ch; bool rkey, rval; - const char *uri, *value; + char *uri, *value; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -32,13 +33,19 @@ util_read(WT_SESSION *session, int argc, char *argv[]) /* The remaining arguments are a uri followed by a list of keys. */ if (argc < 2) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - /* Open the object. */ - if ((ret = session->open_cursor( - session, uri, NULL, NULL, &cursor)) != 0) - return (util_err(session, ret, "%s: session.open", uri)); + /* + * Open the object; free allocated memory immediately to simplify + * future error handling. + */ + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + (void)util_err(session, ret, "%s: session.open_cursor", uri); + free(uri); + if (ret != 0) + return (ret); /* * A simple search only makes sense if the key format is a string or a diff --git a/src/third_party/wiredtiger/src/utilities/util_rebalance.c b/src/third_party/wiredtiger/src/utilities/util_rebalance.c index 45f161487e5..c188ea17d22 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rebalance.c +++ b/src/third_party/wiredtiger/src/utilities/util_rebalance.c @@ -15,9 +15,9 @@ util_rebalance(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - char *name; + char *uri; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -30,25 +30,21 @@ util_rebalance(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - if ((ret = session->rebalance(session, name, NULL)) != 0) { - fprintf(stderr, "%s: rebalance(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + if ((ret = session->rebalance(session, uri, NULL)) != 0) + (void)util_err(session, ret, "session.rebalance: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; - } - - free(name); - + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_rename.c b/src/third_party/wiredtiger/src/utilities/util_rename.c index aee299c6e63..bb2d40cd103 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rename.c +++ b/src/third_party/wiredtiger/src/utilities/util_rename.c @@ -30,22 +30,15 @@ util_rename(WT_SESSION *session, int argc, char *argv[]) /* The remaining arguments are the object uri and new name. */ if (argc != 2) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); newuri = argv[1]; - if ((ret = session->rename(session, uri, newuri, NULL)) != 0) { - fprintf(stderr, "%s: rename %s to %s: %s\n", - progname, uri, newuri, session->strerror(session, ret)); - goto err; - } - - if (0) { -err: ret = 1; - } + if ((ret = session->rename(session, uri, newuri, NULL)) != 0) + (void)util_err( + session, ret, "session.rename: %s, %s", uri, newuri); free(uri); - return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_salvage.c b/src/third_party/wiredtiger/src/utilities/util_salvage.c index 679d1074457..6cc2278b846 100644 --- a/src/third_party/wiredtiger/src/utilities/util_salvage.c +++ b/src/third_party/wiredtiger/src/utilities/util_salvage.c @@ -16,10 +16,10 @@ util_salvage(WT_SESSION *session, int argc, char *argv[]) WT_DECL_RET; int ch; const char *force; - char *name; + char *uri; force = NULL; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "F")) != EOF) switch (ch) { case 'F': @@ -35,25 +35,21 @@ util_salvage(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the file name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "file")) == NULL) + if ((uri = util_uri(session, *argv, "file")) == NULL) return (1); - if ((ret = session->salvage(session, name, force)) != 0) { - fprintf(stderr, "%s: salvage(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + if ((ret = session->salvage(session, uri, force)) != 0) + (void)util_err(session, ret, "session.salvage: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; - } - - free(name); - + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_stat.c b/src/third_party/wiredtiger/src/utilities/util_stat.c index 4376f559ceb..0692afe2819 100644 --- a/src/third_party/wiredtiger/src/utilities/util_stat.c +++ b/src/third_party/wiredtiger/src/utilities/util_stat.c @@ -55,7 +55,7 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) objname = (char *)""; break; case 1: - if ((objname = util_name(session, *argv, "table")) == NULL) + if ((objname = util_uri(session, *argv, "table")) == NULL) return (1); objname_free = true; break; @@ -68,7 +68,10 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) fprintf(stderr, "%s: %s\n", progname, strerror(errno)); goto err; } - snprintf(uri, urilen, "statistics:%s", objname); + if ((ret = __wt_snprintf(uri, urilen, "statistics:%s", objname)) != 0) { + fprintf(stderr, "%s: %s\n", progname, strerror(ret)); + goto err; + } if ((ret = session->open_cursor(session, uri, NULL, config, &cursor)) != 0) { @@ -82,8 +85,8 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, NULL)) == 0) if (printf("%s=%s\n", desc, pval) < 0) { - ret = errno; - break; + (void)util_err(session, errno, "printf"); + goto err; } if (ret == WT_NOTFOUND) ret = 0; diff --git a/src/third_party/wiredtiger/src/utilities/util_truncate.c b/src/third_party/wiredtiger/src/utilities/util_truncate.c new file mode 100644 index 00000000000..35de02345c8 --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_truncate.c @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_truncate(WT_SESSION *session, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char *uri; + + uri = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + + /* The remaining argument is the uri. */ + if (argc != 1) + return (usage()); + if ((uri = util_uri(session, *argv, "table")) == NULL) + return (1); + + if ((ret = session->truncate(session, uri, NULL, NULL, NULL)) != 0) + (void)util_err(session, ret, "session.truncate: %s", uri); + + free(uri); + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "truncate uri\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_upgrade.c b/src/third_party/wiredtiger/src/utilities/util_upgrade.c index 63b23f28c16..f89bd46e133 100644 --- a/src/third_party/wiredtiger/src/utilities/util_upgrade.c +++ b/src/third_party/wiredtiger/src/utilities/util_upgrade.c @@ -15,9 +15,9 @@ util_upgrade(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - char *name; + char *uri; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -30,25 +30,21 @@ util_upgrade(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - if ((ret = session->upgrade(session, name, NULL)) != 0) { - fprintf(stderr, "%s: upgrade(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + if ((ret = session->upgrade(session, uri, NULL)) != 0) + (void)util_err(session, ret, "session.upgrade: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; - } - - free(name); - + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c index 82bdd780cd3..ace1be7a5de 100644 --- a/src/third_party/wiredtiger/src/utilities/util_verify.c +++ b/src/third_party/wiredtiger/src/utilities/util_verify.c @@ -17,10 +17,10 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) size_t size; int ch; bool dump_address, dump_blocks, dump_layout, dump_pages; - char *config, *dump_offsets, *name; + char *config, *dump_offsets, *uri; dump_address = dump_blocks = dump_layout = dump_pages = false; - config = dump_offsets = name = NULL; + config = dump_offsets = uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "d:")) != EOF) switch (ch) { case 'd': @@ -55,7 +55,7 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); /* Build the configuration string as necessary. */ @@ -69,10 +69,10 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) strlen("dump_offsets[],") + (dump_offsets == NULL ? 0 : strlen(dump_offsets)) + 20; if ((config = malloc(size)) == NULL) { - (void)util_err(session, errno, NULL); + ret = util_err(session, errno, NULL); goto err; } - snprintf(config, size, + if ((ret = __wt_snprintf(config, size, "%s%s%s%s%s%s%s", dump_address ? "dump_address," : "", dump_blocks ? "dump_blocks," : "", @@ -80,25 +80,24 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) dump_offsets != NULL ? "dump_offsets=[" : "", dump_offsets != NULL ? dump_offsets : "", dump_offsets != NULL ? "]," : "", - dump_pages ? "dump_pages," : ""); - } - if ((ret = session->verify(session, name, config)) != 0) { - fprintf(stderr, "%s: verify(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + dump_pages ? "dump_pages," : "")) != 0) { + (void)util_err(session, ret, NULL); + goto err; + } } - - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; + if ((ret = session->verify(session, uri, config)) != 0) + (void)util_err(session, ret, "session.verify: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - free(config); - free(name); - +err: free(config); + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_write.c b/src/third_party/wiredtiger/src/utilities/util_write.c index 7d9bce02b36..1d3e6937f8d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_write.c +++ b/src/third_party/wiredtiger/src/utilities/util_write.c @@ -18,10 +18,10 @@ util_write(WT_SESSION *session, int argc, char *argv[]) uint64_t recno; int ch; bool append, overwrite, rkey; - const char *uri; - char config[100]; + char *uri, config[100]; append = overwrite = false; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "ao")) != EOF) switch (ch) { case 'a': @@ -47,15 +47,25 @@ util_write(WT_SESSION *session, int argc, char *argv[]) } else if (argc < 3 || ((argc - 1) % 2 != 0)) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - /* Open the object. */ - (void)snprintf(config, sizeof(config), "%s,%s", - append ? "append=true" : "", overwrite ? "overwrite=true" : ""); - if ((ret = session->open_cursor( - session, uri, NULL, config, &cursor)) != 0) - return (util_err(session, ret, "%s: session.open", uri)); + /* + * Open the object; free allocated memory immediately to simplify + * future error handling. + */ + if ((ret = __wt_snprintf(config, sizeof(config), "%s,%s", + append ? "append=true" : "", + overwrite ? "overwrite=true" : "")) != 0) { + free(uri); + return (util_err(session, ret, NULL)); + } + if ((ret = + session->open_cursor(session, uri, NULL, config, &cursor)) != 0) + (void)util_err(session, ret, "%s: session.open_cursor", uri); + free(uri); + if (ret != 0) + return (ret); /* * A simple search only makes sense if the key format is a string or a diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index 67249ff887e..b6299bbbadc 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -29,8 +29,6 @@ #include "test_util.h" static struct { - char *progname; /* Program name */ - WT_CONNECTION *wt_conn; /* WT_CONNECTION handle */ WT_SESSION *wt_session; /* WT_SESSION handle */ @@ -61,10 +59,7 @@ main(int argc, char *argv[]) { int ch; - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; + (void)testutil_set_progname(argv); /* Set default configuration values. */ g.c_cache = 10; @@ -75,7 +70,7 @@ main(int argc, char *argv[]) g.c_srand = 3233456; /* Set values from the command line. */ - while ((ch = __wt_getopt(g.progname, argc, argv, "c:f:k:o:s:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "c:f:k:o:s:")) != EOF) switch (ch) { case 'c': /* Cache size */ g.c_cache = (u_int)atoi(__wt_optarg); @@ -126,9 +121,9 @@ setup(void) * Open configuration -- put command line configuration options at the * end so they can override "standard" configuration. */ - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", - g.progname, g.c_cache, g.config_open == NULL ? "" : g.config_open); + progname, g.c_cache, g.config_open == NULL ? "" : g.config_open)); testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); @@ -246,7 +241,7 @@ populate_entries(void) void usage(void) { - fprintf(stderr, "usage: %s [-cfkos]\n", g.progname); + fprintf(stderr, "usage: %s [-cfkos]\n", progname); fprintf(stderr, "%s", "\t-c cache size\n" "\t-f number of bits per item\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index ef49a9492ce..84d2765843a 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -74,7 +74,7 @@ checkpointer(void *arg) WT_UNUSED(arg); - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); printf("checkpointer thread starting: tid: %s\n", tid); (void)real_checkpointer(); @@ -107,8 +107,9 @@ real_checkpointer(void) "WiredTigerCheckpoint", strlen("WiredTigerCheckpoint")) == 0) checkpoint_config = NULL; else { + testutil_check(__wt_snprintf( + _buf, sizeof(_buf), "name=%s", g.checkpoint_name)); checkpoint_config = _buf; - snprintf(checkpoint_config, 128, "name=%s", g.checkpoint_name); } while (g.running) { /* Execute a checkpoint */ @@ -147,7 +148,8 @@ verify_checkpoint(WT_SESSION *session) ret = t_ret = 0; key_count = 0; - snprintf(ckpt, 128, "checkpoint=%s", g.checkpoint_name); + testutil_check(__wt_snprintf( + ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); cursors = calloc((size_t)g.ntables, sizeof(*cursors)); if (cursors == NULL) return (log_print_err("verify_checkpoint", ENOMEM, 1)); @@ -159,7 +161,8 @@ verify_checkpoint(WT_SESSION *session) */ if (g.cookies[i].type == LSM) continue; - snprintf(next_uri, 128, "table:__wt%04d", i); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", i)); if ((ret = session->open_cursor( session, next_uri, NULL, ckpt, &cursors[i])) != 0) { (void)log_print_err( @@ -296,7 +299,8 @@ diagnose_key_error( session = cursor1->session; key1_orig = key2_orig = 0; - snprintf(ckpt, 128, "checkpoint=%s", g.checkpoint_name); + testutil_check(__wt_snprintf( + ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); /* Save the failed keys. */ if (cursor1->get_key(cursor1, &key1_orig) != 0 || @@ -338,7 +342,8 @@ diagnose_key_error( * Now try opening new cursors on the checkpoints and see if we * get the same missing key via searching. */ - snprintf(next_uri, 128, "table:__wt%04d", index1); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index1)); if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -350,7 +355,8 @@ diagnose_key_error( if (c->close(c) != 0) return (1); - snprintf(next_uri, 128, "table:__wt%04d", index2); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index2)); if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -367,7 +373,8 @@ live_check: * Now try opening cursors on the live checkpoint to see if we get the * same missing key via searching. */ - snprintf(next_uri, 128, "table:__wt%04d", index1); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index1)); if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -376,7 +383,8 @@ live_check: if (c->close(c) != 0) return (1); - snprintf(next_uri, 128, "table:__wt%04d", index2); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index2)); if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) return (1); c->set_key(c, key2_orig); diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 123d4e00df5..39b1f428c2c 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -6,8 +6,8 @@ set -e echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m -# We are done if short tests are requested -test -z "$TESTUTIL_DISABLE_LONG_TESTS" || exit 0 +# We are done unless long tests are enabled. +test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 echo "checkpoint: 6 column-store tables" $TEST_WRAPPER ./t -T 6 -t c diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 4998019ad8e..e7e1a0b81a5 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -50,10 +50,7 @@ main(int argc, char *argv[]) char *working_dir; const char *config_open; - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; + (void)testutil_set_progname(argv); config_open = NULL; ret = 0; @@ -68,7 +65,7 @@ main(int argc, char *argv[]) runs = 1; while ((ch = __wt_getopt( - g.progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) + progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) switch (ch) { case 'c': g.checkpoint_name = __wt_optarg; @@ -132,7 +129,7 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); - printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid()); + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); @@ -202,11 +199,11 @@ wt_connect(const char *config_open) testutil_make_work_dir(g.home); - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(fast),error_prefix=\"%s\",cache_size=1GB%s%s", - g.progname, + progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( g.home, &event_handler, config, &g.conn)) != 0) @@ -297,10 +294,10 @@ log_print_err(const char *m, int e, int fatal) g.running = 0; g.status = e; } - fprintf(stderr, "%s: %s: %s\n", g.progname, m, wiredtiger_strerror(e)); + fprintf(stderr, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); if (g.logfp != NULL) fprintf(g.logfp, "%s: %s: %s\n", - g.progname, m, wiredtiger_strerror(e)); + progname, m, wiredtiger_strerror(e)); return (e); } @@ -333,7 +330,7 @@ usage(void) "usage: %s " "[-S] [-C wiredtiger-config] [-k keys] [-l log]\n\t" "[-n ops] [-c checkpoint] [-r runs] [-t f|r|v] [-W workers]\n", - g.progname); + progname); fprintf(stderr, "%s", "\t-C specify wiredtiger_open configuration arguments\n" "\t-c checkpoint name to used named checkpoints\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 0d0d02447d5..347bd2c6e89 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -58,7 +58,6 @@ typedef struct { u_int nkeys; /* Keys to load */ u_int nops; /* Operations per thread */ FILE *logfp; /* Message log file. */ - char *progname; /* Program name */ int nworkers; /* Number workers configured */ int ntables; /* Number tables configured */ int ntables_created; /* Number tables opened */ diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index e4fe7bd1b29..82d1b8685c4 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -39,14 +39,12 @@ static int create_table(WT_SESSION *session, COOKIE *cookie) { int ret; - char *p, *end, config[128]; + char config[128]; - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), - "key_format=%s,value_format=S", cookie->type == COL ? "r" : "q"); - if (cookie->type == LSM) - (void)snprintf(p, (size_t)(end - p), ",type=lsm"); + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=%s,value_format=S,%s", + cookie->type == COL ? "r" : "q", + cookie->type == LSM ? ",type=lsm" : "")); if ((ret = session->create(session, cookie->uri, config)) != 0) if (ret != EEXIST) @@ -88,8 +86,9 @@ start_workers(table_type type) (table_type)((i % MAX_TABLE_TYPE) + 1); else g.cookies[i].type = type; - (void)snprintf(g.cookies[i].uri, 128, - "%s%04d", URI_BASE, g.cookies[i].id); + testutil_check(__wt_snprintf( + g.cookies[i].uri, sizeof(g.cookies[i].uri), + "%s%04d", URI_BASE, g.cookies[i].id)); /* Should probably be atomic to avoid races. */ if ((ret = create_table(session, &g.cookies[i])) != 0) @@ -132,7 +131,8 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) char valuebuf[64]; cursor->set_key(cursor, keyno); - (void)snprintf(valuebuf, sizeof(valuebuf), "%037u", new_val); + testutil_check(__wt_snprintf( + valuebuf, sizeof(valuebuf), "%037u", new_val)); cursor->set_value(cursor, valuebuf); if ((ret = cursor->insert(cursor)) != 0) { if (ret == WT_ROLLBACK) @@ -153,7 +153,7 @@ worker(void *arg) WT_UNUSED(arg); - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); printf("worker thread starting: tid: %s\n", tid); (void)real_worker(); diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index a96492c1e71..f2b4fcacdc8 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -4,8 +4,13 @@ LDADD = $(top_builddir)/test/utility/libtest_util.la \ $(top_builddir)/libwiredtiger.la AM_LDFLAGS = -static +noinst_PROGRAMS= + +test_scope_SOURCES = scope/main.c +noinst_PROGRAMS += test_scope + test_wt1965_col_efficiency_SOURCES = wt1965_col_efficiency/main.c -noinst_PROGRAMS = test_wt1965_col_efficiency +noinst_PROGRAMS += test_wt1965_col_efficiency test_wt2403_lsm_workload_SOURCES = wt2403_lsm_workload/main.c noinst_PROGRAMS += test_wt2403_lsm_workload @@ -37,9 +42,24 @@ noinst_PROGRAMS += test_wt2834_join_bloom_fix test_wt2853_perf_SOURCES = wt2853_perf/main.c noinst_PROGRAMS += test_wt2853_perf +test_wt2909_checkpoint_integrity_SOURCES = wt2909_checkpoint_integrity/main.c +noinst_PROGRAMS += test_wt2909_checkpoint_integrity + test_wt2999_join_extractor_SOURCES = wt2999_join_extractor/main.c noinst_PROGRAMS += test_wt2999_join_extractor +test_wt3120_filesys_SOURCES = wt3120_filesys/main.c +noinst_PROGRAMS += test_wt3120_filesys + +test_wt3135_search_near_collator_SOURCES = wt3135_search_near_collator/main.c +noinst_PROGRAMS += test_wt3135_search_near_collator + +test_wt3184_dup_index_collator_SOURCES = wt3184_dup_index_collator/main.c +noinst_PROGRAMS += test_wt3184_dup_index_collator + +test_rwlock_SOURCES = rwlock/main.c +noinst_PROGRAMS += test_rwlock + # Run this during a "make check" smoke test. TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c new file mode 100644 index 00000000000..04813182478 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -0,0 +1,184 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: HELP-4355 + * Test rwlock collapse under load. + */ +#define MAX_THREADS 1000 +#define READS_PER_WRITE 10000 +//#define READS_PER_WRITE 1000000 +//#define READS_PER_WRITE 100 + +#define CHECK_CORRECTNESS 1 +//#define USE_POSIX 1 + +static WT_RWLOCK rwlock; +static pthread_rwlock_t p_rwlock; +static bool running; +static uint64_t shared_counter; + +void *thread_rwlock(void *); +void *thread_dump(void *); + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + struct timespec te, ts; + pthread_t dump_id, id[MAX_THREADS]; + int i; + + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->nthreads = 100; + opts->nops = 1000000; /* per thread */ + testutil_check(testutil_parse_opts(argc, argv, opts)); + running = true; + + testutil_make_work_dir(opts->home); + testutil_check(wiredtiger_open(opts->home, NULL, + "create,session_max=1000,statistics=(fast)", &opts->conn)); + + testutil_check(__wt_rwlock_init(NULL, &rwlock)); + testutil_check(pthread_rwlock_init(&p_rwlock, NULL)); + + testutil_check(pthread_create( + &dump_id, NULL, thread_dump, (void *)opts)); + + __wt_epoch(NULL, &ts); + for (i = 0; i < (int)opts->nthreads; ++i) + testutil_check(pthread_create( + &id[i], NULL, thread_rwlock, (void *)opts)); + + while (--i >= 0) + testutil_check(pthread_join(id[i], NULL)); + __wt_epoch(NULL, &te); + printf("%.2lf\n", WT_TIMEDIFF_MS(te, ts) / 1000.0); + + running = false; + testutil_check(pthread_join(dump_id, NULL)); + + testutil_check(pthread_rwlock_destroy(&p_rwlock)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} + +/* + * Acquire a rwlock, every Nth operation, acquire exclusive. + */ +void * +thread_rwlock(void *arg) +{ + TEST_OPTS *opts; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session; + uint64_t i, counter; + bool writelock; + + opts = (TEST_OPTS *)arg; + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); + session = (WT_SESSION_IMPL *)wt_session; + + printf("Running rwlock thread\n"); + for (i = 1; i <= opts->nops; ++i) { + writelock = (i % READS_PER_WRITE == 0); + +#ifdef USE_POSIX + if (writelock) + testutil_check(pthread_rwlock_wrlock(&p_rwlock)); + else + testutil_check(pthread_rwlock_rdlock(&p_rwlock)); +#else + if (writelock) + __wt_writelock(session, &rwlock); + else + __wt_readlock(session, &rwlock); +#endif + + /* + * Do a tiny amount of work inside the lock so the compiler + * can't optimize everything away. + */ + (void)__wt_atomic_add64(&counter, 1); + +#ifdef CHECK_CORRECTNESS + if (writelock) + counter = ++shared_counter; + else + counter = shared_counter; + + __wt_yield(); + + testutil_assert(counter == shared_counter); +#endif + +#ifdef USE_POSIX + testutil_check(pthread_rwlock_unlock(&p_rwlock)); +#else + if (writelock) + __wt_writeunlock(session, &rwlock); + else + __wt_readunlock(session, &rwlock); +#endif + + if (i % 10000 == 0) { + printf("%s", session->id == 20 ? ".\n" : "."); + fflush(stdout); + } + } + + opts->running = false; + + return (NULL); +} + +void * +thread_dump(void *arg) { + WT_UNUSED(arg); + + while (running) { + sleep(1); + printf("\n" + "rwlock { current %" PRIu8 ", next %" PRIu8 + ", reader %" PRIu8 ", readers_active %" PRIu16 + ", readers_queued %" PRIu16 " }\n", + rwlock.u.s.current, + rwlock.u.s.next, + rwlock.u.s.reader, + rwlock.u.s.readers_active, + rwlock.u.s.readers_queued); + } + + return (NULL); +} diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c new file mode 100644 index 00000000000..15dabd97c40 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/scope/main.c @@ -0,0 +1,288 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#define KEY "key" +#define VALUE "value" + +static int ignore_errors; + +static int +handle_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + (void)(handler); + + /* Skip the error messages we're expecting to see. */ + if (ignore_errors > 0 && + (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { + --ignore_errors; + return (0); + } + + (void)fprintf(stderr, "%s: %s\n", + message, session->strerror(session, error)); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + handle_error, + NULL, + NULL, + NULL +}; + +static void +cursor_scope_ops(WT_SESSION *session, const char *uri) +{ + struct { + const char *op; + enum { INSERT, SEARCH, SEARCH_NEAR, + REMOVE, REMOVE_POS, RESERVE, UPDATE } func; + const char *config; + } *op, ops[] = { + /* + * The ops order is fixed and shouldn't change, that is, insert + * has to happen first so search, update and remove operations + * are possible, and remove has to be last. + */ + { "insert", INSERT, NULL, }, + { "search", SEARCH, NULL, }, + { "search", SEARCH_NEAR, NULL, }, +#if 0 + { "reserve", RESERVE, NULL, }, +#endif + { "update", UPDATE, NULL, }, + { "remove", REMOVE, NULL, }, + { "remove", REMOVE_POS, NULL, }, + { NULL, INSERT, NULL } + }; + WT_CURSOR *cursor; + uint64_t keyr; + const char *key, *value; + char keybuf[100], valuebuf[100]; + int exact; + bool recno; + + /* Reserve requires a running transaction. */ + testutil_check(session->begin_transaction(session, NULL)); + + cursor = NULL; + for (op = ops; op->op != NULL; op++) { + key = value = NULL; + + /* Open a cursor. */ + if (cursor != NULL) + testutil_check(cursor->close(cursor)); + testutil_check(session->open_cursor( + session, uri, NULL, op->config, &cursor)); + recno = strcmp(cursor->key_format, "r") == 0; + + /* + * Set up application buffers so we can detect overwrites + * or failure to copy application information into library + * memory. + */ + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + strcpy(keybuf, KEY); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + cursor->set_value(cursor, valuebuf); + + /* + * The application must keep key and value memory valid until + * the next operation that positions the cursor, modifies the + * data, or resets or closes the cursor. + * + * Modifying either the key or value buffers is not permitted. + */ + switch (op->func) { + case INSERT: + testutil_check(cursor->insert(cursor)); + break; + case SEARCH: + testutil_check(cursor->search(cursor)); + break; + case SEARCH_NEAR: + testutil_check(cursor->search_near(cursor, &exact)); + break; + case REMOVE_POS: + /* + * Remove has two modes, one where the remove is based + * on a cursor position, the other where it's based on + * a set key. The results are different, so test them + * separately. + */ + testutil_check(cursor->search(cursor)); + /* FALLTHROUGH */ + case REMOVE: + testutil_check(cursor->remove(cursor)); + break; + case RESERVE: +#if 0 + testutil_check(cursor->reserve(cursor)); +#endif + break; + case UPDATE: + testutil_check(cursor->update(cursor)); + break; + } + + /* + * The cursor should no longer reference application memory, + * and application buffers can be safely overwritten. + */ + memset(keybuf, 'K', sizeof(keybuf)); + memset(valuebuf, 'V', sizeof(valuebuf)); + + /* + * Check that get_key/get_value behave as expected after the + * operation. + */ + switch (op->func) { + case INSERT: + case REMOVE: + /* + * Insert and remove configured with a search key do + * not position the cursor and have no key or value. + * + * There should be two error messages, ignore them. + */ + ignore_errors = 2; + if (recno) + testutil_assert( + cursor->get_key(cursor, &keyr) != 0); + else + testutil_assert( + cursor->get_key(cursor, &key) != 0); + testutil_assert(cursor->get_value(cursor, &value) != 0); + testutil_assert(ignore_errors == 0); + break; + case REMOVE_POS: + /* + * Remove configured with a cursor position has a key, + * but no value. + * + * There should be one error message, ignore it. + */ + if (recno) { + testutil_assert( + cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert( + cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + ignore_errors = 1; + testutil_assert(cursor->get_value(cursor, &value) != 0); + testutil_assert(ignore_errors == 0); + break; + case RESERVE: + case SEARCH: + case SEARCH_NEAR: + case UPDATE: + /* + * Reserve, search, search-near and update position the + * cursor and have both a key and value. + * + * Any key/value should not reference application + * memory. + */ + if (recno) { + testutil_assert( + cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert( + cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + testutil_assert(cursor->get_value(cursor, &value) == 0); + testutil_assert(value != valuebuf); + testutil_assert(strcmp(value, VALUE) == 0); + break; + } + + /* + * We have more than one remove operation, add the key back + * in. + */ + if (op->func == REMOVE || op->func == REMOVE_POS) { + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + cursor->set_key(cursor, KEY); + } + cursor->set_value(cursor, VALUE); + testutil_check(cursor->insert(cursor)); + } + } +} + +static void +run(WT_CONNECTION *conn, const char *uri, const char *config) +{ + WT_SESSION *session; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, config)); + cursor_scope_ops(session, uri); + testutil_check(session->close(session, NULL)); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check( + wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); + + run(opts->conn, "file:file.SS", "key_format=S,value_format=S"); + run(opts->conn, "file:file.rS", "key_format=r,value_format=S"); + run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S"); + run(opts->conn, "lsm:lsm.rS", "key_format=r,value_format=S"); + run(opts->conn, "table:table.SS", "key_format=S,value_format=S"); + run(opts->conn, "table:table.rS", "key_format=r,value_format=S"); + + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c index a7235d81b31..e5b73d5e642 100644 --- a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c @@ -132,7 +132,8 @@ main(int argc, char *argv[]) testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); - sprintf(table_format, "key_format=r,value_format="); + testutil_check(__wt_snprintf( + table_format, sizeof(table_format), "key_format=r,value_format=")); for (i = 0; i < NR_FIELDS; i++) strcat(table_format, "Q"); diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index 4b352b26051..9876582fffa 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -68,8 +68,8 @@ page_init(uint64_t n) else { if (recno % 3 == 0) ++vrecno; - snprintf(buf, - sizeof(buf), "%" PRIu64 " VALUE ------", vrecno); + testutil_check(__wt_snprintf(buf, + sizeof(buf), "%" PRIu64 " VALUE ------", vrecno)); cursor->set_value(cursor, buf); } testutil_check(cursor->insert(cursor)); @@ -101,9 +101,10 @@ main(int argc, char *argv[]) uint64_t i, id; char buf[100]; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; opts->n_append_threads = N_APPEND_THREADS; @@ -111,19 +112,19 @@ main(int argc, char *argv[]) testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); - snprintf(buf, sizeof(buf), + testutil_check(__wt_snprintf(buf, sizeof(buf), "create," "cache_size=%s," "eviction=(threads_max=5)," "statistics=(fast)", - opts->table_type == TABLE_FIX ? "500MB" : "2GB"); + opts->table_type == TABLE_FIX ? "500MB" : "2GB")); testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); - snprintf(buf, sizeof(buf), + testutil_check(__wt_snprintf(buf, sizeof(buf), "key_format=r,value_format=%s," "allocation_size=4K,leaf_page_max=64K", - opts->table_type == TABLE_FIX ? "8t" : "S"); + opts->table_type == TABLE_FIX ? "8t" : "S")); testutil_check(session->create(session, opts->uri, buf)); testutil_check(session->close(session, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 239a3f300d0..617490fec4d 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -92,10 +92,11 @@ main(int argc, char *argv[]) TEST_OPTS *opts, _opts; const char *tablename; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); memset(sharedopts, 0, sizeof(*sharedopts)); @@ -105,14 +106,18 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename); - snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename); - snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename); - snprintf(sharedopts->joinuri, sizeof(sharedopts->joinuri), - "join:%s", opts->uri); + testutil_check(__wt_snprintf( + sharedopts->posturi, sizeof(sharedopts->posturi), + "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + sharedopts->baluri, sizeof(sharedopts->baluri), + "index:%s:bal", tablename)); + testutil_check(__wt_snprintf( + sharedopts->flaguri, sizeof(sharedopts->flaguri), + "index:%s:flag", tablename)); + testutil_check(__wt_snprintf( + sharedopts->joinuri, sizeof(sharedopts->joinuri), + "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=1G", &opts->conn)); @@ -349,19 +354,21 @@ static void *thread_join(void *arg) balcur->set_key(balcur, 0); testutil_check(balcur->search(balcur)); if (sharedopts->bloom) - sprintf(cfg, "compare=lt,strategy=bloom,count=%d", - N_RECORDS); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=lt,strategy=bloom,count=%d", N_RECORDS)); else - sprintf(cfg, "compare=lt"); + testutil_check(__wt_snprintf( + cfg, sizeof(cfg), "compare=lt")); testutil_check(session->join(session, joincur, balcur, cfg)); flagcur->set_key(flagcur, 0); testutil_check(flagcur->search(flagcur)); if (sharedopts->bloom) - sprintf(cfg, "compare=eq,strategy=bloom,count=%d", - N_RECORDS); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=eq,strategy=bloom,count=%d", N_RECORDS)); else - sprintf(cfg, "compare=eq"); + testutil_check(__wt_snprintf( + cfg, sizeof(cfg), "compare=eq")); testutil_check(session->join(session, joincur, flagcur, cfg)); /* Expect no values returned */ diff --git a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c index 1368e7c8c09..656cea04145 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c @@ -102,9 +102,12 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(index1uri, sizeof(index1uri), "index:%s:index1", tablename); - snprintf(index2uri, sizeof(index2uri), "index:%s:index2", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + index1uri, sizeof(index1uri), "index:%s:index1", tablename)); + testutil_check(__wt_snprintf( + index2uri, sizeof(index2uri), "index:%s:index2", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "statistics=(all),create", &opts->conn)); @@ -150,7 +153,8 @@ main(int argc, char *argv[]) cursor2->set_key(cursor2, half + 1); testutil_check(cursor2->search(cursor2)); - sprintf(bloom_cfg, "compare=lt,strategy=bloom,count=%d", half); + testutil_check(__wt_snprintf(bloom_cfg, sizeof(bloom_cfg), + "compare=lt,strategy=bloom,count=%d", half)); testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &jcursor)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index ae18760a829..ba17d485e07 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -49,9 +49,10 @@ main(int argc, char *argv[]) uint64_t current_value; int i; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); opts->nthreads = 10; opts->nrecords = 1000; diff --git a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c index 0ec1c765d99..be3eff6136c 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c @@ -82,9 +82,12 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(countryuri, sizeof(countryuri), "index:%s:country", tablename); - snprintf(yearuri, sizeof(yearuri), "index:%s:year", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + countryuri, sizeof(countryuri), "index:%s:country", tablename)); + testutil_check(__wt_snprintf( + yearuri, sizeof(yearuri), "index:%s:year", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=200M", &opts->conn)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index 7c80496f1b6..e128df29f41 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -59,11 +59,11 @@ main(int argc, char *argv[]) char flaguri[256]; char joinuri[256]; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); @@ -83,10 +83,14 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(posturi, sizeof(posturi), "index:%s:post", tablename); - snprintf(balanceuri, sizeof(balanceuri), "index:%s:balance", tablename); - snprintf(flaguri, sizeof(flaguri), "index:%s:flag", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + posturi, sizeof(posturi), "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + balanceuri, sizeof(balanceuri), "index:%s:balance", tablename)); + testutil_check(__wt_snprintf( + flaguri, sizeof(flaguri), "index:%s:flag", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(session->create(session, posturi, "columns=(post)")); testutil_check(session->create(session, balanceuri, @@ -126,14 +130,14 @@ main(int argc, char *argv[]) balancecur->set_key(balancecur, 0); testutil_check(balancecur->search(balancecur)); - sprintf(cfg, "compare=lt,strategy=bloom,count=%d", - N_RECORDS / 100); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=lt,strategy=bloom,count=%d", N_RECORDS / 100)); testutil_check(session->join(session, joincur, balancecur, cfg)); flagcur->set_key(flagcur, 0); testutil_check(flagcur->search(flagcur)); - sprintf(cfg, "compare=eq,strategy=bloom,count=%d", - N_RECORDS / 100); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=eq,strategy=bloom,count=%d", N_RECORDS / 100)); testutil_check(session->join(session, joincur, flagcur, cfg)); /* Expect no values returned */ diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 6cec9634cd1..46ba71372e5 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -82,11 +82,11 @@ main(int argc, char *argv[]) int i, nfail; const char *tablename; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; - - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); memset(sharedopts, 0, sizeof(*sharedopts)); memset(insert_args, 0, sizeof(insert_args)); @@ -114,12 +114,15 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename); - snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename); - snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename); + testutil_check(__wt_snprintf( + sharedopts->posturi, sizeof(sharedopts->posturi), + "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + sharedopts->baluri, sizeof(sharedopts->baluri), + "index:%s:bal", tablename)); + testutil_check(__wt_snprintf( + sharedopts->flaguri, sizeof(sharedopts->flaguri), + "index:%s:flag", tablename)); testutil_check(session->create(session, sharedopts->posturi, "columns=(post)")); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c new file mode 100644 index 00000000000..ce7bd72fa3f --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -0,0 +1,669 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#include +#include +#include + +/* + * JIRA ticket reference: WT-2909 + * Test case description: + * + * This test attempts to check the integrity of checkpoints by injecting + * failures (by means of a custom file system) and then trying to recover. To + * insulate the top level program from various crashes that may occur when + * injecting failures, the "populate" code runs in another process, and is + * expected to sometimes fail. Then the top level program runs recovery (with + * the normal file system) and checks the results. Any failure at the top level + * indicates a checkpoint integrity problem. + * + * Each subtest uses the same kind of schema and data, the only variance is + * when the faults are injected. At the moment, this test only injects during + * checkpoints, and only injects write failures. It varies in the number of + * successful writes that occur before an injected failure (during a checkpoint + * operation), this can be indicated with "-o N". When N is not specified, the + * test attempts to find the optimal range of N for testing. Clearly when N is + * large, then the checkpoint may be successfully written, and the data + * represented by the checkpoint will be fully present. When N is small, + * nothing of interest is written and no data is present. To find the sweet + * spot where interesting failures occur, the test does a binary search to find + * the approximate N that divides the "small" and "large" cases. This is not + * strictly deterministic, a given N may give different results on different + * runs. But approximate optimal N can be determined, allowing a series of + * additional tests clustered around this N. + * + * The data is stored in two tables, one having indices. Both tables have + * the same keys and are updated with the same key in a single transaction. + * + * Failure mode: + * If one table is out of step with the other, that is detected as a failure at + * the top level. If an index is missing values (or has extra values), that is + * likewise a failure at the top level. If the tables or the home directory + * cannot be opened, that is a top level error. The tables must be present + * as an initial checkpoint is done without any injected fault. + */ + +/* + * This program does not run on Windows. The non-portable aspects at minimum + * are fork/exec the use of environment variables (used by fail_fs), and file + * name and build locations of dynamically loaded libraries. + */ +#define BIG_SIZE (1024 * 10) +#define BIG_CONTENTS "" +#define MAX_ARGS 20 +#define MAX_OP_RANGE 1000 +#define STDERR_FILE "stderr.txt" +#define STDOUT_FILE "stdout.txt" +#define TESTS_PER_OP_VALUE 3 +#define VERBOSE_PRINT 10000 + +static int check_results(TEST_OPTS *, uint64_t *); +static void check_values(WT_CURSOR *, int, int, int, char *); +static int create_big_string(char **); +static void cursor_count_items(WT_CURSOR *, uint64_t *); +static void disable_failures(void); +static void enable_failures(uint64_t, uint64_t); +static void generate_key(uint64_t, int *); +static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, + char **); +static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, + uint64_t *); +static void run_check_subtest_range(TEST_OPTS *, const char *, bool); +static int run_process(TEST_OPTS *, const char *, char *[], int *); +static void subtest_main(int, char *[], bool); +static void subtest_populate(TEST_OPTS *, bool); + +extern int __wt_optind; + +#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + +/* + * check_results -- + * Check all the tables and verify the results. + */ +static int +check_results(TEST_OPTS *opts, uint64_t *foundp) +{ + WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur; + WT_SESSION *session; + uint64_t count, idxcount, nrecords; + uint32_t rndint; + int key, key_got, ret, v0, v1, v2; + char *bigref, *big; + + testutil_check(create_big_string(&bigref)); + nrecords = opts->nrecords; + testutil_check(wiredtiger_open(opts->home, NULL, + "create,log=(enabled)", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, "table:subtest", NULL, + NULL, &maincur)); + testutil_check(session->open_cursor(session, "table:subtest2", NULL, + NULL, &maincur2)); + testutil_check(session->open_cursor(session, "index:subtest:v0", NULL, + NULL, &v0cur)); + testutil_check(session->open_cursor(session, "index:subtest:v1", NULL, + NULL, &v1cur)); + testutil_check(session->open_cursor(session, "index:subtest:v2", NULL, + NULL, &v2cur)); + + count = 0; + while ((ret = maincur->next(maincur)) == 0) { + testutil_check(maincur2->next(maincur2)); + testutil_check(maincur2->get_key(maincur2, &key_got)); + testutil_check(maincur2->get_value(maincur2, &rndint)); + + generate_key(count, &key); + generate_value(rndint, count, bigref, &v0, &v1, &v2, &big); + testutil_assert(key == key_got); + + /* Check the key/values in main table. */ + testutil_check(maincur->get_key(maincur, &key_got)); + testutil_assert(key == key_got); + check_values(maincur, v0, v1, v2, big); + + /* Check the values in the indices. */ + v0cur->set_key(v0cur, v0); + testutil_check(v0cur->search(v0cur)); + check_values(v0cur, v0, v1, v2, big); + v1cur->set_key(v1cur, v1); + testutil_check(v1cur->search(v1cur)); + check_values(v1cur, v0, v1, v2, big); + v2cur->set_key(v2cur, v2); + testutil_check(v2cur->search(v2cur)); + check_values(v2cur, v0, v1, v2, big); + + count++; + if (count % VERBOSE_PRINT == 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, + nrecords); + } + if (count % VERBOSE_PRINT != 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords); + + /* + * Always expect at least one entry, as populate does a + * checkpoint after the first insert. + */ + testutil_assert(count > 0); + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(maincur2->next(maincur2) == WT_NOTFOUND); + cursor_count_items(v0cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v1cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v2cur, &idxcount); + testutil_assert(count == idxcount); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + + free(bigref); + *foundp = count; + return (0); +} + +/* + * check_values -- + * Check that the values in the cursor match the given values. + */ +static void +check_values(WT_CURSOR *cursor, int v0, int v1, int v2, char *big) +{ + int v0_got, v1_got, v2_got; + char *big_got; + + testutil_check(cursor->get_value(cursor, &v0_got, &v1_got, &v2_got, + &big_got)); + testutil_assert(v0 == v0_got); + testutil_assert(v1 == v1_got); + testutil_assert(v2 == v2_got); + testutil_assert(strcmp(big, big_got) == 0); +} + +/* + * create_big_string -- + * Create and fill the "reference" big array. + */ +static int create_big_string(char **bigp) +{ + size_t i, mod; + char *big; + + if ((big = malloc(BIG_SIZE + 1)) == NULL) + return (ENOMEM); + mod = strlen(BIG_CONTENTS); + for (i = 0; i < BIG_SIZE; i++) { + big[i] = BIG_CONTENTS[i % mod]; + } + big[BIG_SIZE] = '\0'; + *bigp = big; + return (0); +} + +/* + * cursor_count_items -- + * Count the number of items in the table by traversing + * through the cursor. + */ +static void +cursor_count_items(WT_CURSOR *cursor, uint64_t *countp) +{ + int ret; + + *countp = 0; + + testutil_check(cursor->reset(cursor)); + while ((ret = cursor->next(cursor)) == 0) + (*countp)++; + testutil_assert(ret == WT_NOTFOUND); +} + +/* + * disable_failures -- + * Disable failures in the fail file system. + */ +static void +disable_failures(void) +{ + testutil_check(setenv("WT_FAIL_FS_ENABLE", "0", 1)); +} + +/* + * enable_failures -- + * Enable failures in the fail file system. + */ +static void +enable_failures(uint64_t allow_writes, uint64_t allow_reads) +{ + char value[100]; + + testutil_check(setenv("WT_FAIL_FS_ENABLE", "1", 1)); + testutil_check(__wt_snprintf( + value, sizeof(value), "%" PRIu64, allow_writes)); + testutil_check(setenv("WT_FAIL_FS_WRITE_ALLOW", value, 1)); + testutil_check(__wt_snprintf( + value, sizeof(value), "%" PRIu64, allow_reads)); + testutil_check(setenv("WT_FAIL_FS_READ_ALLOW", value, 1)); +} + +/* + * generate_key -- + * Generate a key used by the "subtest" and "subtest2" tables. + */ +static void +generate_key(uint64_t i, int *keyp) +{ + *keyp = (int)i; +} + +/* + * generate_value -- + * Generate values for the "subtest" table. + */ +static void +generate_value(uint32_t rndint, uint64_t i, char *bigref, + int *v0p, int *v1p, int *v2p, char **bigp) +{ + *v0p = (int)(i * 7); + *v1p = (int)(i * 10007); + *v2p = (int)(i * 100000007); + *bigp = &bigref[rndint % BIG_SIZE]; +} + +/* + * run_check_subtest -- + * Run the subtest with the given parameters and check the results. + */ +static void +run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, + bool close_test, uint64_t *nresultsp) +{ + int estatus, narg; + char rarg[20], sarg[20], *subtest_args[MAX_ARGS]; + + narg = 0; + if (debugger != NULL) { + subtest_args[narg++] = (char *)debugger; + subtest_args[narg++] = (char *)"--"; + } + + subtest_args[narg++] = (char *)opts->progname; + /* "subtest" must appear before arguments */ + if (close_test) + subtest_args[narg++] = (char *)"subtest_close"; + else + subtest_args[narg++] = (char *)"subtest"; + subtest_args[narg++] = (char *)"-h"; + subtest_args[narg++] = opts->home; + subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */ + subtest_args[narg++] = (char *)"-p"; + subtest_args[narg++] = (char *)"-o"; + testutil_check(__wt_snprintf(sarg, sizeof(sarg), "%" PRIu64, nops)); + subtest_args[narg++] = sarg; /* number of operations */ + subtest_args[narg++] = (char *)"-n"; + testutil_check(__wt_snprintf( + rarg, sizeof(rarg), "%" PRIu64, opts->nrecords)); + subtest_args[narg++] = rarg; /* number of records */ + subtest_args[narg++] = NULL; + testutil_assert(narg <= MAX_ARGS); + if (opts->verbose) + printf("running a separate process with %" PRIu64 + " operations until fail...\n", nops); + testutil_clean_work_dir(opts->home); + testutil_check(run_process( + opts, debugger != NULL ? debugger : opts->progname, + subtest_args, &estatus)); + if (opts->verbose) + printf("process exited %d\n", estatus); + + /* + * Verify results in parent process. + */ + testutil_check(check_results(opts, nresultsp)); +} + +/* + * run_check_subtest_range -- + * + * Run successive tests via binary search that determines the approximate + * crossover point between when data is recoverable or not. Once that is + * determined, run the subtest in a range near that crossover point. + * + * The theory is that running at the crossover point will tend to trigger + * "interesting" failures at the borderline when the checkpoint is about to, + * or has, succeeded. If any of those failures creates a WT home directory + * that cannot be recovered, the top level test will fail. + */ +static void +run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) +{ + uint64_t cutoff, high, low, mid, nops, nresults; + int i; + bool got_failure, got_success; + + if (opts->verbose) + printf("Determining best range of operations until failure, " + "with close_test %s.\n", + (close_test ? "enabled" : "disabled")); + + run_check_subtest(opts, debugger, 1, close_test, &cutoff); + low = 0; + high = MAX_OP_RANGE; + mid = (low + high) / 2; + while (mid != low) { + run_check_subtest(opts, debugger, mid, close_test, + &nresults); + if (nresults > cutoff) + high = mid; + else + low = mid; + mid = (low + high) / 2; + } + /* + * mid is the number of ops that is the crossover point. + * Run some tests near that point to try to trigger weird + * failures. If mid is too low or too high, it indicates + * there is a fundamental problem with the test. + */ + testutil_assert(mid > 1 && mid < MAX_OP_RANGE - 1); + if (opts->verbose) + printf("Retesting around %" PRIu64 " operations.\n", + mid); + + got_failure = false; + got_success = false; + for (nops = mid - 10; nops < mid + 10; nops++) { + for (i = 0; i < TESTS_PER_OP_VALUE; i++) { + run_check_subtest(opts, debugger, nops, + close_test, &nresults); + if (nresults > cutoff) + got_failure = true; + else + got_success = true; + } + } + /* + * Check that it really ran with a crossover point. + */ + testutil_assert(got_failure); + testutil_assert(got_success); +} + +/* + * run_process -- + * Run a program with arguments, wait until it completes. + */ +static int +run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) +{ + int pid; + char **arg; + + if (opts->verbose) { + printf("running: "); + for (arg = argv; *arg != NULL; arg++) + printf("%s ", *arg); + printf("\n"); + } + if ((pid = fork()) == 0) { + (void)execv(prog, argv); + testutil_die(errno, "%s", prog); + } else if (pid < 0) + return (errno); + + (void)waitpid(pid, status, 0); + return (0); +} + +/* + * subtest_main -- + * The main program for the subtest + */ +static void +subtest_main(int argc, char *argv[], bool close_test) +{ + TEST_OPTS *opts, _opts; + WT_SESSION *session; + char config[1024], filename[1024]; + struct rlimit rlim; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + memset(&rlim, 0, sizeof(rlim)); + + /* No core files during fault injection tests. */ + testutil_check(setrlimit(RLIMIT_CORE, &rlim)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + /* Redirect stderr, stdout. */ + testutil_check(__wt_snprintf( + filename, sizeof(filename), "%s/%s", opts->home, STDERR_FILE)); + testutil_assert(freopen(filename, "a", stderr) != NULL); + testutil_check(__wt_snprintf( + filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); + testutil_assert(freopen(filename, "a", stdout) != NULL); + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_size=250M,log=(enabled)," + "transaction_sync=(enabled,method=none),extensions=(" + WT_FAIL_FS_LIB + "=(early_load,config={environment=true,verbose=true})]")); + + testutil_check(wiredtiger_open(opts->home, NULL, config, &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->create(session, "table:subtest", + "key_format=i,value_format=iiiS," + "columns=(id,v0,v1,v2,big)")); + + testutil_check(session->create(session, "table:subtest2", + "key_format=i,value_format=i")); + + testutil_check(session->create(session, "index:subtest:v0", + "columns=(v0)")); + testutil_check(session->create(session, "index:subtest:v1", + "columns=(v1)")); + testutil_check(session->create(session, "index:subtest:v2", + "columns=(v2)")); + + testutil_check(session->close(session, NULL)); + + subtest_populate(opts, close_test); + + testutil_cleanup(opts); +} + +/* + * This macro is used as a substitute for testutil_check, except that it is + * aware of when a failure may be expected due to the effects of the fail_fs. + * This macro is used only in subtest_populate(), it uses local variables. + */ +#define CHECK(expr) { \ + int _ret; \ + _ret = expr; \ + if (_ret != 0) { \ + if (!failmode || \ + (_ret != WT_RUN_RECOVERY && _ret != EIO)) { \ + fprintf(stderr, " BAD RETURN %d for \"%s\"\n", \ + _ret, #expr); \ + testutil_check(_ret); \ + } else \ + failed = true; \ + } \ +} + +/* + * subtest_populate -- + * Populate the tables. + */ +static void +subtest_populate(TEST_OPTS *opts, bool close_test) +{ + WT_CURSOR *maincur, *maincur2; + WT_RAND_STATE rnd; + WT_SESSION *session; + uint64_t i, nrecords; + uint32_t rndint; + int key, v0, v1, v2; + char *big, *bigref; + bool failed, failmode; + + failmode = failed = false; + __wt_random_init_seed(NULL, &rnd); + CHECK(create_big_string(&bigref)); + nrecords = opts->nrecords; + + CHECK(opts->conn->open_session( + opts->conn, NULL, NULL, &session)); + + CHECK(session->open_cursor(session, "table:subtest", NULL, + NULL, &maincur)); + + CHECK(session->open_cursor(session, "table:subtest2", NULL, + NULL, &maincur2)); + + for (i = 0; i < nrecords && !failed; i++) { + rndint = __wt_random(&rnd); + generate_key(i, &key); + generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); + CHECK(session->begin_transaction(session, NULL)); + maincur->set_key(maincur, key); + maincur->set_value(maincur, v0, v1, v2, big); + CHECK(maincur->insert(maincur)); + + maincur2->set_key(maincur2, key); + maincur2->set_value(maincur2, rndint); + CHECK(maincur2->insert(maincur2)); + CHECK(session->commit_transaction(session, NULL)); + + if (i == 0) + /* + * Force an initial checkpoint, that helps to + * distinguish a clear failure from just not running + * long enough. + */ + CHECK(session->checkpoint(session, NULL)); + + if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) + printf(" %" PRIu64 "/%" PRIu64 "\n", + (i + 1), nrecords); + /* Attempt to isolate the failures to checkpointing. */ + if (i == (nrecords/100)) { + enable_failures(opts->nops, 1000000); + failmode = true; /* CHECK should expect failures. */ + CHECK(session->checkpoint(session, NULL)); + failmode = false; + disable_failures(); + if (failed && opts->verbose) + printf("checkpoint failed (expected).\n"); + } + } + + /* + * Closing handles after an extreme fail is likely to cause + * cascading failures (or crashes), so recommended practice is + * to immediately exit. We're interested in testing both with + * and without the recommended practice. + */ + if (failed) { + if (!close_test) { + fprintf(stderr, "exit early.\n"); + exit(0); + } else + fprintf(stderr, "closing after failure.\n"); + } + + free(bigref); + CHECK(maincur->close(maincur)); + CHECK(maincur2->close(maincur2)); + CHECK(session->close(session, NULL)); +} + +/* + * main -- + * The main program for the test. When invoked with "subtest" + * argument, run the subtest. Otherwise, run a separate process + * for each needed subtest, and check the results. + */ +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + uint64_t nresults; + const char *debugger; + + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + debugger = NULL; + + testutil_check(testutil_parse_opts(argc, argv, opts)); + argc -= __wt_optind; + argv += __wt_optind; + if (opts->nrecords == 0) + opts->nrecords = 50000; + + while (argc > 0) { + if (strcmp(argv[0], "subtest") == 0) { + subtest_main(argc, argv, false); + return (0); + } else if (strcmp(argv[0], "subtest_close") == 0) { + subtest_main(argc, argv, true); + return (0); + } else if (strcmp(argv[0], "gdb") == 0) + debugger = "/usr/bin/gdb"; + else + testutil_assert(false); + argc--; + argv++; + } + if (opts->verbose) { + printf("Number of operations until failure: %" PRIu64 + " (change with -o N)\n", opts->nops); + printf("Number of records: %" PRIu64 + " (change with -n N)\n", opts->nrecords); + } + if (opts->nops == 0) { + run_check_subtest_range(opts, debugger, false); + run_check_subtest_range(opts, debugger, true); + } else + run_check_subtest(opts, debugger, opts->nops, + opts->nrecords, &nresults); + + testutil_clean_work_dir(opts->home); + testutil_cleanup(opts); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c new file mode 100644 index 00000000000..2fae85017d4 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c @@ -0,0 +1,99 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3120 + * Test case description: A simple file system extension built into + * a shared library. + * Failure mode: Loading the file system and closing the connection + * is enough to evoke the failure. This test does slightly more + * than that. + */ + +#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor; + WT_SESSION *session; + char *kstr, *vstr; + char buf[1024]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "create,extensions=(" WT_FAIL_FS_LIB "=(early_load=true))")); + testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, + "key_format=S,value_format=S")); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor)); + cursor->set_key(cursor, "a"); + cursor->set_value(cursor, "0"); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, "b"); + cursor->set_value(cursor, "1"); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + + /* Force to disk and re-open. */ + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); + + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor)); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "a") == 0); + testutil_assert(strcmp(vstr, "0") == 0); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "b") == 0); + testutil_assert(strcmp(vstr, "1") == 0); + testutil_assert(cursor->next(cursor) == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + printf("Success\n"); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c new file mode 100644 index 00000000000..8783034a7d8 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c @@ -0,0 +1,360 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3135 + * Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do + * search_near and search for each element. For each set of data, we perform + * these tests first using a custom collator, and second using a custom collator + * and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is + * given a truncated key to compare, and the unpack functions return errors + * because the truncation appeared in the middle of a key. + */ + +#define TEST_ENTRY_COUNT 5 +typedef const char *TEST_SET[TEST_ENTRY_COUNT]; +static TEST_SET test_sets[] = { + { "0", "01", "012", "0123", "01234" }, + { "A", "B", "C", "D", "E" }, + { "5", "54", "543", "5432", "54321" }, + { "54321", "5433", "544", "55", "6" } +}; +#define TEST_SET_COUNT (sizeof(test_sets) / sizeof(test_sets[0])) + +static bool +item_str_equal(WT_ITEM *item, const char *str) +{ + return (item->size == strlen(str) + 1 && strncmp((char *)item->data, + str, item->size) == 0); +} + +static int +compare_int(int64_t a, int64_t b) +{ + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +static int +index_compare_primary(WT_PACK_STREAM *s1, WT_PACK_STREAM *s2, int *cmp) +{ + int64_t pkey1, pkey2; + int rc1, rc2; + + rc1 = wiredtiger_unpack_int(s1, &pkey1); + rc2 = wiredtiger_unpack_int(s2, &pkey2); + + if (rc1 == 0 && rc2 == 0) + *cmp = compare_int(pkey1, pkey2); + else if (rc1 != 0 && rc2 != 0) + *cmp = 0; + else if (rc1 != 0) + *cmp = -1; + else + *cmp = 1; + return (0); +} + +static int +index_compare_S(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_PACK_STREAM *s1, *s2; + const char *skey1, *skey2; + + (void)collator; + + testutil_check(wiredtiger_unpack_start(session, "Si", key1->data, + key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "Si", key2->data, + key2->size, &s2)); + + testutil_check(wiredtiger_unpack_str(s1, &skey1)); + testutil_check(wiredtiger_unpack_str(s2, &skey2)); + + if ((*cmp = strcmp(skey1, skey2)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); + + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); + + return (0); +} + +static int +index_compare_u(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_ITEM skey1, skey2; + WT_PACK_STREAM *s1, *s2; + + (void)collator; + + testutil_check(wiredtiger_unpack_start(session, "ui", key1->data, + key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "ui", key2->data, + key2->size, &s2)); + + testutil_check(wiredtiger_unpack_item(s1, &skey1)); + testutil_check(wiredtiger_unpack_item(s2, &skey2)); + + if ((*cmp = strcmp(skey1.data, skey2.data)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); + + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); + + return (0); +} + +static int +index_extractor_u(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +{ + (void)extractor; + (void)session; + (void)key; + + result_cursor->set_key(result_cursor, value); + return result_cursor->insert(result_cursor); +} + +static WT_COLLATOR collator_S = { index_compare_S, NULL, NULL }; +static WT_COLLATOR collator_u = { index_compare_u, NULL, NULL }; +static WT_EXTRACTOR extractor_u = { index_extractor_u, NULL, NULL }; + +/* + * Check search() and search_near() using the test string indicated + * by test_index. + */ +static void +search_using_str(WT_CURSOR *cursor, TEST_SET test_set, int test_index) +{ + int exact, ret; + const char *result; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + cursor->set_key(cursor, test_str); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &result)); + + if (test_index == 0) + testutil_assert(strcmp(result, str_01) == 0 && exact > 0); + else if (test_index == 1) + testutil_assert(strcmp(result, str_01) == 0 && exact == 0); + else if (test_index == 2) + testutil_assert((strcmp(result, str_0123) == 0 && exact > 0) || + (strcmp(result, str_01) == 0 && exact < 0)); + else if (test_index == 3) + testutil_assert(strcmp(result, str_0123) == 0 && exact == 0); + else if (test_index == 4) + testutil_assert(strcmp(result, str_0123) == 0 && exact < 0); + + cursor->set_key(cursor, test_str); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); +} + +/* + * Check search() and search_near() using the test string indicated + * by test_index against a table containing a variable sized item. + */ +static void +search_using_item(WT_CURSOR *cursor, TEST_SET test_set, int test_index) +{ + WT_ITEM item; + size_t testlen; + int exact, ret; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + testlen = strlen(test_str) + 1; + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &item)); + + if (test_index == 0) + testutil_assert(item_str_equal(&item, str_01) && exact > 0); + else if (test_index == 1) + testutil_assert(item_str_equal(&item, str_01) && exact == 0); + else if (test_index == 2) + testutil_assert((item_str_equal(&item, str_0123) && exact > 0) + || (item_str_equal(&item, str_01) && exact < 0)); + else if (test_index == 3) + testutil_assert(item_str_equal(&item, str_0123) && exact == 0); + else if (test_index == 4) + testutil_assert(item_str_equal(&item, str_0123) && exact < 0); + + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); +} + +/* + * For each set of data, perform tests. + */ +static void +test_one_set(WT_SESSION *session, TEST_SET set) +{ + WT_CURSOR *cursor; + WT_ITEM item; + int32_t i; + + /* + * Part 1: Using a custom collator, insert some elements + * and verify results from search_near. + */ + + testutil_check(session->create(session, + "table:main", "key_format=i,value_format=S,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:def_collator", "columns=(v)")); + testutil_check(session->create(session, + "index:main:custom_collator", + "columns=(v),collator=collator_S")); + + /* Insert only elements #1 and #3. */ + testutil_check(session->open_cursor(session, + "table:main", NULL, NULL, &cursor)); + cursor->set_key(cursor, 0); + cursor->set_value(cursor, set[1]); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, 1); + cursor->set_value(cursor, set[3]); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + /* Check all elements in def_collator index. */ + testutil_check(session->open_cursor(session, + "index:main:def_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* Check all elements in custom_collator index */ + testutil_check(session->open_cursor(session, + "index:main:custom_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* + * Part 2: perform the same checks using a custom collator and + * extractor. + */ + testutil_check(session->create(session, + "table:main2", "key_format=i,value_format=u,columns=(k,v)")); + + testutil_check(session->create(session, "index:main2:idx_w_coll", + "key_format=u,collator=collator_u,extractor=extractor_u")); + + testutil_check(session->open_cursor(session, + "table:main2", NULL, NULL, &cursor)); + + memset(&item, 0, sizeof(item)); + item.size = strlen(set[1]) + 1; + item.data = set[1]; + cursor->set_key(cursor, 1); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + item.size = strlen(set[3]) + 1; + item.data = set[3]; + cursor->set_key(cursor, 3); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + testutil_check(cursor->close(cursor)); + + testutil_check(session->open_cursor(session, + "index:main2:idx_w_coll", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_item(cursor, set, i); + testutil_check(cursor->close(cursor)); + + testutil_check(session->drop(session, "table:main", NULL)); + testutil_check(session->drop(session, "table:main2", NULL)); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_SESSION *session; + size_t i; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", + &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Add any collators and extractors used by tests */ + testutil_check(opts->conn->add_collator(opts->conn, "collator_S", + &collator_S, NULL)); + testutil_check(opts->conn->add_collator(opts->conn, "collator_u", + &collator_u, NULL)); + testutil_check(opts->conn->add_extractor(opts->conn, "extractor_u", + &extractor_u, NULL)); + + for (i = 0; i < TEST_SET_COUNT; i++) { + printf("test set %" WT_SIZET_FMT "\n", i); + test_one_set(session, test_sets[i]); + } + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c new file mode 100644 index 00000000000..c969e7a1d7e --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c @@ -0,0 +1,168 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3184 + * Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do + * search_near and search for each element. For each set of data, we perform + * these tests first using a custom collator, and second using a custom collator + * and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is + * given a truncated key to compare, and the unpack functions return errors + * because the truncation appeared in the middle of a key. + */ + +static int +compare_int(int32_t a, int32_t b) +{ + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +static int32_t +item_to_int(WT_ITEM *item) +{ + testutil_assert(item->size == sizeof(int32_t)); + return (*(int32_t *)item->data); +} + +static int +compare_int_items(WT_ITEM *itema, WT_ITEM *itemb) +{ + testutil_assert(itema->size == sizeof(int32_t)); + testutil_assert(itemb->size == sizeof(int32_t)); + return (compare_int(item_to_int(itema), item_to_int(itemb))); +} + +static void +print_int_item(const char *str, const WT_ITEM *item) +{ + if (item->size > 0) { + testutil_assert(item->size == sizeof(int32_t)); + printf("%s%" PRId32, str, *(int32_t *)item->data); + } else + printf("%s", str); +} + +static int +index_compare(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_ITEM ikey1, pkey1, ikey2, pkey2; + + (void)collator; + testutil_check(wiredtiger_struct_unpack(session, + key1->data, key1->size, "uu", &ikey1, &pkey1)); + testutil_check(wiredtiger_struct_unpack(session, + key2->data, key2->size, "uu", &ikey2, &pkey2)); + + print_int_item("index_compare: index key1 = ", &ikey1); + print_int_item(", primary key1 = ", &pkey1); + print_int_item(", index key2 = ", &ikey2); + print_int_item(", primary key2 = ", &pkey2); + printf("\n"); + + if ((*cmp = compare_int_items(&ikey1, &ikey2)) != 0) + return (0); + + if (pkey1.size != 0 && pkey2.size != 0) + *cmp = compare_int_items(&pkey1, &pkey2); + else if (pkey1.size != 0) + *cmp = 1; + else if (pkey2.size != 0) + *cmp = -1; + else + *cmp = 0; + + return (0); +} + +static WT_COLLATOR index_coll = { index_compare, NULL, NULL }; + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor, *cursor1; + WT_ITEM got, k, v; + WT_SESSION *session; + int32_t ki, vi; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", + &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(opts->conn->add_collator(opts->conn, "index_coll", + &index_coll, NULL)); + + testutil_check(session->create(session, + "table:main", "key_format=u,value_format=u,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:index", "columns=(v),collator=index_coll")); + + printf("adding new record\n"); + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, + &cursor)); + + ki = 13; + vi = 17; + + k.data = &ki; k.size = sizeof(ki); + v.data = &vi; v.size = sizeof(vi); + + cursor->set_key(cursor, &k); + cursor->set_value(cursor, &v); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + printf("positioning index cursor\n"); + + testutil_check(session->open_cursor(session, "index:main:index", NULL, + NULL, &cursor)); + cursor->set_key(cursor, &v); + testutil_check(cursor->search(cursor)); + + printf("duplicating cursor\n"); + testutil_check(session->open_cursor(session, NULL, cursor, NULL, + &cursor1)); + testutil_check(cursor->get_value(cursor, &got)); + testutil_assert(item_to_int(&got) == 17); + testutil_check(cursor1->get_value(cursor1, &got)); + testutil_assert(item_to_int(&got) == 17); + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index 85b8c68e545..d3c64b54ab5 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -29,7 +29,6 @@ #include "cursor_order.h" static char home[512]; /* Program working dir */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); @@ -51,10 +50,7 @@ main(int argc, char *argv[]) int ch, cnt, runs; char *config_open, *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); cfg = &_cfg; config_open = NULL; @@ -185,19 +181,15 @@ wt_connect(SHARED_CONFIG *cfg, char *config_open) }; int ret; char config[512]; - size_t print_count; testutil_clean_work_dir(home); testutil_make_work_dir(home); - print_count = (size_t)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); - - if (print_count >= sizeof(config)) - testutil_die(EINVAL, "Config string too long"); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( home, &event_handler, config, &cfg->conn)) != 0) diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c index 5dc7194b5fb..42d7af54de4 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c @@ -34,23 +34,21 @@ file_create(SHARED_CONFIG *cfg, const char *name) WT_CONNECTION *conn; WT_SESSION *session; int ret; - char *p, *end, config[128]; + char config[128]; conn = cfg->conn; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s," "internal_page_max=%d," "split_deepen_min_child=200," - "leaf_page_max=%d,", - cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024); - if (cfg->ftype == FIX) - (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + "leaf_page_max=%d," + "%s", + cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024, + cfg->ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) @@ -67,9 +65,10 @@ load(SHARED_CONFIG *cfg, const char *name) WT_CURSOR *cursor; WT_ITEM *value, _value; WT_SESSION *session; - char keybuf[64], valuebuf[64]; - int64_t keyno; + size_t len; + uint64_t keyno; int ret; + char keybuf[64], valuebuf[64]; conn = cfg->conn; @@ -83,9 +82,10 @@ load(SHARED_CONFIG *cfg, const char *name) testutil_die(ret, "cursor.open"); value = &_value; - for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { + for (keyno = 1; keyno <= cfg->nkeys; ++keyno) { if (cfg->ftype == ROW) { - snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + testutil_check(__wt_snprintf( + keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -93,8 +93,10 @@ load(SHARED_CONFIG *cfg, const char *name) if (cfg->ftype == FIX) cursor->set_value(cursor, 0x01); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "%37u", (u_int)keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "%37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c index 58da49b2991..299f22684c9 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -69,7 +69,8 @@ ops_start(SHARED_CONFIG *cfg) run_info[i].cfg = cfg; if (i == 0 || cfg->multiple_files) { run_info[i].name = dmalloc(64); - snprintf(run_info[i].name, 64, FNAME, (int)i); + testutil_check(__wt_snprintf( + run_info[i].name, 64, FNAME, (int)i)); /* Vary by orders of magnitude */ if (cfg->vary_nops) @@ -93,8 +94,8 @@ ops_start(SHARED_CONFIG *cfg) run_info[offset].name = dmalloc(64); /* Have reverse scans read from tables with writes. */ name_index = i % cfg->append_inserters; - snprintf( - run_info[offset].name, 64, FNAME, (int)name_index); + testutil_check(__wt_snprintf( + run_info[offset].name, 64, FNAME, (int)name_index)); /* Vary by orders of magnitude */ if (cfg->vary_nops) @@ -231,7 +232,7 @@ reverse_scan(void *arg) id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf(" reverse scan thread %2" PRIuMAX @@ -272,6 +273,7 @@ append_insert_op( { WT_ITEM *value, _value; uint64_t keyno; + size_t len; int ret; char keybuf[64], valuebuf[64]; @@ -281,7 +283,8 @@ append_insert_op( keyno = __wt_atomic_add64(&cfg->key_range, 1); if (cfg->ftype == ROW) { - snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + testutil_check(__wt_snprintf( + keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -291,8 +294,9 @@ append_insert_op( if (cfg->ftype == FIX) cursor->set_value(cursor, 0x10); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "XXX %37u", (u_int)keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) @@ -318,7 +322,7 @@ append_insert(void *arg) id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", diff --git a/src/third_party/wiredtiger/test/fops/file.c b/src/third_party/wiredtiger/test/fops/file.c index ea15f1ee80d..d1cd22ab391 100644 --- a/src/third_party/wiredtiger/test/fops/file.c +++ b/src/third_party/wiredtiger/test/fops/file.c @@ -51,7 +51,7 @@ obj_bulk(void) if ((ret = c->close(c)) != 0) testutil_die(ret, "cursor.close"); } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) - testutil_die(ret, "session.open_cursor"); + testutil_die(ret, "session.open_cursor bulk"); } if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); @@ -71,7 +71,8 @@ obj_bulk_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); @@ -79,12 +80,17 @@ obj_bulk_unique(int force) testutil_die(ret, "session.create: %s", new_uri); __wt_yield(); - if ((ret = - session->open_cursor(session, new_uri, NULL, "bulk", &c)) != 0) - testutil_die(ret, "session.open_cursor: %s", new_uri); - - if ((ret = c->close(c)) != 0) - testutil_die(ret, "cursor.close"); + /* + * Opening a bulk cursor may have raced with a forced checkpoint + * which created a checkpoint of the empty file, and triggers an EINVAL + */ + if ((ret = session->open_cursor( + session, new_uri, NULL, "bulk", &c)) == 0) { + if ((ret = c->close(c)) != 0) + testutil_die(ret, "cursor.close"); + } else if (ret != EINVAL) + testutil_die(ret, + "session.open_cursor bulk unique: %s, new_uri"); while ((ret = session->drop( session, new_uri, force ? "force" : NULL)) != 0) @@ -147,7 +153,8 @@ obj_create_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); @@ -190,9 +197,13 @@ obj_checkpoint(void) if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - /* Force the checkpoint so it has to be taken. */ + /* + * Force the checkpoint so it has to be taken. Forced checkpoints can + * race with other metadata operations and return EBUSY - we'd expect + * applications using forced checkpoints to retry on EBUSY. + */ if ((ret = session->checkpoint(session, "force")) != 0) - if (ret != ENOENT) + if (ret != EBUSY && ret != ENOENT) testutil_die(ret, "session.checkpoint"); if ((ret = session->close(session, NULL)) != 0) diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index 7b4a7cf8fca..07ac07349e3 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -34,7 +34,6 @@ u_int nops; /* Operations */ const char *uri; /* Object */ const char *config; /* Object config */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static char home[512]; @@ -71,22 +70,15 @@ main(int argc, char *argv[]) int ch, cnt, ret, runs; char *config_open, *working_dir; - working_dir = NULL; - - /* Remove directories */ - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); if ((ret = pthread_rwlock_init(&single, NULL)) != 0) testutil_die(ret, "pthread_rwlock_init: single"); - config_open = NULL; nops = 1000; nthreads = 10; runs = 1; - + config_open = working_dir = NULL; while ((ch = __wt_getopt(progname, argc, argv, "C:h:l:n:r:t:")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ @@ -165,11 +157,11 @@ wt_startup(char *config_open) testutil_make_work_dir(home); - snprintf(config_buf, sizeof(config_buf), + testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), "create,error_prefix=\"%s\",cache_size=5MB%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( home, &event_handler, config_buf, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -225,6 +217,11 @@ handle_message(WT_EVENT_HANDLER *handler, (void)(handler); (void)(session); + /* Ignore messages about failing to create forced checkpoints. */ + if (strstr( + message, "forced or named checkpoint") != NULL) + return (0); + if (logfp != NULL) return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 69fdf771de9..8aa614fa970 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -63,7 +63,7 @@ copy_file(WT_SESSION *session, const char *name) len = strlen("BACKUP") + strlen(name) + 10; first = dmalloc(len); - (void)snprintf(first, len, "BACKUP/%s", name); + testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); testutil_check(__wt_copy_and_sync(session, name, first)); /* @@ -72,7 +72,7 @@ copy_file(WT_SESSION *session, const char *name) */ len = strlen("BACKUP_COPY") + strlen(name) + 10; second = dmalloc(len); - (void)snprintf(second, len, "BACKUP_COPY/%s", name); + testutil_check(__wt_snprintf(second, len, "BACKUP_COPY/%s", name)); testutil_check(__wt_copy_and_sync(session, first, second)); free(first); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 839ff5058de..22b40f7164d 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -44,6 +44,7 @@ static void config_map_compression(const char *, u_int *); static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); static void config_map_isolation(const char *, u_int *); +static void config_pct(void); static void config_reset(void); /* @@ -62,39 +63,42 @@ config_setup(void) config_in_memory(); /* - * Choose a data source type and a file type: they're interrelated (LSM - * trees are only compatible with row-store) and other items depend on - * them. + * Choose a file format and a data source: they're interrelated (LSM is + * only compatible with row-store) and other items depend on them. */ + if (!config_is_perm("file_type")) { + if (config_is_perm("data_source") && DATASOURCE("lsm")) + config_single("file_type=row", 0); + else + switch (mmrand(NULL, 1, 10)) { + case 1: /* 10% */ + config_single("file_type=fix", 0); + break; + case 2: case 3: case 4: /* 30% */ + config_single("file_type=var", 0); + break; /* 60% */ + case 5: case 6: case 7: case 8: case 9: case 10: + config_single("file_type=row", 0); + break; + } + } + config_map_file_type(g.c_file_type, &g.type); + if (!config_is_perm("data_source")) switch (mmrand(NULL, 1, 3)) { case 1: config_single("data_source=file", 0); break; case 2: - if (!g.c_in_memory) { - config_single("data_source=lsm", 0); - break; - } - /* FALLTHROUGH */ - case 3: config_single("data_source=table", 0); break; - } - - if (!config_is_perm("file_type")) - switch (DATASOURCE("lsm") ? 5 : mmrand(NULL, 1, 10)) { - case 1: - config_single("file_type=fix", 0); - break; - case 2: case 3: case 4: - config_single("file_type=var", 0); - break; - case 5: case 6: case 7: case 8: case 9: case 10: - config_single("file_type=row", 0); + case 3: + if (g.c_in_memory || g.type != ROW) + config_single("data_source=table", 0); + else + config_single("data_source=lsm", 0); break; } - config_map_file_type(g.c_file_type, &g.type); /* * If data_source and file_type were both "permanent", we may still @@ -103,7 +107,7 @@ config_setup(void) if (DATASOURCE("lsm") && g.type != ROW) { fprintf(stderr, "%s: lsm data_source is only compatible with row file_type\n", - g.progname); + progname); exit(EXIT_FAILURE); } @@ -159,31 +163,19 @@ config_setup(void) config_encryption(); config_isolation(); config_lrt(); + config_pct(); /* - * Periodically, set the delete percentage to 0 so salvage gets run, - * as long as the delete percentage isn't nailed down. - * Don't do it on the first run, all our smoke tests would hit it. - */ - if (!g.replay && g.run_cnt % 10 == 9 && !config_is_perm("delete_pct")) - config_single("delete_pct=0", 0); - - /* - * If this is an LSM run, set the cache size and crank up the insert - * percentage. + * If this is an LSM run, ensure cache size sanity. + * Ensure there is at least 1MB of cache per thread. */ - if (DATASOURCE("lsm")) { - if (!config_is_perm("cache")) + if (!config_is_perm("cache")) { + if (DATASOURCE("lsm")) g.c_cache = 30 * g.c_chunk_size; - - if (!config_is_perm("insert_pct")) - g.c_insert_pct = mmrand(NULL, 50, 85); + if (g.c_cache < g.c_threads) + g.c_cache = g.c_threads; } - /* Ensure there is at least 1MB of cache per thread. */ - if (!config_is_perm("cache") && g.c_cache < g.c_threads) - g.c_cache = g.c_threads; - /* Give in-memory configuration a final review. */ config_in_memory_check(); @@ -265,8 +257,8 @@ config_compression(const char *conf_name) */ cstr = "none"; if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) { - (void)snprintf( - confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + testutil_check(__wt_snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); config_single(confbuf, 0); return; } @@ -301,7 +293,7 @@ config_compression(const char *conf_name) break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - case 15: case 16 case 17: /* 15% zstd */ + case 15: case 16: case 17: /* 15% zstd */ cstr = "zstd"; break; #endif @@ -310,7 +302,8 @@ config_compression(const char *conf_name) break; } - (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + testutil_check(__wt_snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); config_single(confbuf, 0); } @@ -473,7 +466,7 @@ config_lrt(void) * stores. */ if (g.type == FIX) { - if (config_is_perm("long_running_txn")) + if (config_is_perm("long_running_txn") && g.c_long_running_txn) testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); @@ -481,6 +474,83 @@ config_lrt(void) } } +/* + * config_pct -- + * Configure operation percentages. + */ +static void +config_pct(void) +{ + static struct { + const char *name; /* Operation */ + uint32_t *vp; /* Value store */ + u_int order; /* Order of assignment */ + } list[] = { +#define CONFIG_DELETE_ENTRY 0 + { "delete_pct", &g.c_delete_pct, 0 }, + { "insert_pct", &g.c_insert_pct, 0 }, + { "read_pct", &g.c_read_pct, 0 }, + { "write_pct", &g.c_write_pct, 0 }, + }; + u_int i, max_order, max_slot, n, pct; + + /* + * Walk the list of operations, checking for an illegal configuration + * and creating a random order in the list. + */ + pct = 0; + for (i = 0; i < WT_ELEMENTS(list); ++i) + if (config_is_perm(list[i].name)) + pct += *list[i].vp; + else + list[i].order = mmrand(NULL, 1, 1000); + if (pct > 100) + testutil_die(EINVAL, + "operation percentages total to more than 100%%"); + + /* + * If the delete percentage isn't nailed down, periodically set it to + * 0 so salvage gets run. Don't do it on the first run, all our smoke + * tests would hit it. + */ + if (!config_is_perm("delete_pct") && !g.replay && g.run_cnt % 10 == 9) { + list[CONFIG_DELETE_ENTRY].order = 0; + *list[CONFIG_DELETE_ENTRY].vp = 0; + } + + /* + * Walk the list, allocating random numbers of operations in a random + * order. + * + * If the "order" field is non-zero, we need to create a value for this + * operation. Find the largest order field in the array; if one non-zero + * order field is found, it's the last entry and gets the remainder of + * the operations. + */ + for (pct = 100 - pct;;) { + for (i = n = + max_order = max_slot = 0; i < WT_ELEMENTS(list); ++i) { + if (list[i].order != 0) + ++n; + if (list[i].order > max_order) { + max_order = list[i].order; + max_slot = i; + } + } + if (n == 0) + break; + if (n == 1) { + *list[max_slot].vp = pct; + break; + } + *list[max_slot].vp = mmrand(NULL, 0, pct); + list[max_slot].order = 0; + pct -= *list[max_slot].vp; + } + testutil_assert(g.c_delete_pct + + g.c_insert_pct + g.c_read_pct + g.c_write_pct == 100); +} + /* * config_error -- * Display configuration information on error. @@ -609,13 +679,14 @@ void config_single(const char *s, int perm) { CONFIG *cp; - long v; + long vlong; + uint32_t v; char *p; const char *ep; if ((ep = strchr(s, '=')) == NULL) { fprintf(stderr, - "%s: %s: illegal configuration value\n", g.progname, s); + "%s: %s: illegal configuration value\n", progname, s); exit(EXIT_FAILURE); } @@ -674,34 +745,35 @@ config_single(const char *s, int perm) return; } - v = -1; + vlong = -1; if (F_ISSET(cp, C_BOOL)) { if (strncmp(ep, "off", strlen("off")) == 0) - v = 0; + vlong = 0; else if (strncmp(ep, "on", strlen("on")) == 0) - v = 1; + vlong = 1; } - if (v == -1) { - v = strtol(ep, &p, 10); + if (vlong == -1) { + vlong = strtol(ep, &p, 10); if (*p != '\0') { fprintf(stderr, "%s: %s: illegal numeric value\n", - g.progname, s); + progname, s); exit(EXIT_FAILURE); } } + v = (uint32_t)vlong; if (F_ISSET(cp, C_BOOL)) { if (v != 0 && v != 1) { fprintf(stderr, "%s: %s: value of boolean not 0 or 1\n", - g.progname, s); + progname, s); exit(EXIT_FAILURE); } } else if (v < cp->min || v > cp->maxset) { fprintf(stderr, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", - g.progname, s, cp->min, cp->maxset); + progname, s, cp->min, cp->maxset); exit(EXIT_FAILURE); } - *cp->v = (uint32_t)v; + *cp->v = v; } /* @@ -817,7 +889,7 @@ config_find(const char *s, size_t len) return (cp); fprintf(stderr, - "%s: %s: unknown configuration keyword\n", g.progname, s); + "%s: %s: unknown configuration keyword\n", progname, s); config_error(); exit(EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 9bfba3cd0df..b5feb7a5321 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -65,6 +65,10 @@ static CONFIG c[] = { "if timed run should drop core", /* 0% */ C_BOOL, 0, 0, 0, &g.c_abort, NULL }, + { "alter", + "if altering the table is enabled", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_alter, NULL }, + { "auto_throttle", "if LSM inserts are throttled", /* 90% */ C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL }, @@ -127,7 +131,7 @@ static CONFIG c[] = { { "delete_pct", "percent operations that are deletes", - 0x0, 0, 45, 90, &g.c_delete_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL }, { "dictionary", "if values are dictionary compressed", /* 20% */ @@ -167,7 +171,7 @@ static CONFIG c[] = { { "insert_pct", "percent operations that are inserts", - 0x0, 0, 45, 90, &g.c_insert_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL }, { "internal_key_truncation", "if internal keys are truncated", /* 95% */ @@ -250,6 +254,14 @@ static CONFIG c[] = { "quiet run (same as -q)", C_IGNORE|C_BOOL, 0, 0, 0, &g.c_quiet, NULL }, + { "read_pct", + "percent operations that are reads", + C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL }, + + { "rebalance", + "rebalance testing", /* 100% */ + C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, + { "repeat_data_pct", "percent duplicate values in row- or var-length column-stores", 0x0, 0, 90, 90, &g.c_repeat_data_pct, NULL }, @@ -266,17 +278,13 @@ static CONFIG c[] = { "the number of runs", C_IGNORE, 0, UINT_MAX, UINT_MAX, &g.c_runs, NULL }, - { "rebalance", - "rebalance testing", /* 100% */ - C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, - { "salvage", "salvage testing", /* 100% */ C_BOOL, 100, 1, 0, &g.c_salvage, NULL }, { "split_pct", "page split size as a percentage of the maximum page size", - 0x0, 40, 85, 85, &g.c_split_pct, NULL }, + 0x0, 50, 100, 100, &g.c_split_pct, NULL }, { "statistics", "maintain statistics", /* 20% */ @@ -316,7 +324,7 @@ static CONFIG c[] = { { "write_pct", "percent operations that are writes", - 0x0, 0, 90, 90, &g.c_write_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL }, { NULL, NULL, 0x0, 0, 0, 0, NULL, NULL } }; diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 820bc020c9b..41cc48c4278 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -79,8 +79,6 @@ #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ typedef struct { - char *progname; /* Program name */ - char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ @@ -140,6 +138,7 @@ typedef struct { char *config_open; /* Command-line configuration */ uint32_t c_abort; /* Config values */ + uint32_t c_alter; uint32_t c_auto_throttle; uint32_t c_backups; uint32_t c_bitcnt; @@ -191,6 +190,7 @@ typedef struct { uint32_t c_reverse; uint32_t c_rows; uint32_t c_runs; + uint32_t c_read_pct; uint32_t c_rebalance; uint32_t c_salvage; uint32_t c_split_pct; @@ -242,7 +242,7 @@ typedef struct { } GLOBAL; extern GLOBAL g; -typedef struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) { +typedef struct { WT_RAND_STATE rnd; /* thread RNG state */ uint64_t search; /* operations */ @@ -276,6 +276,7 @@ void bdb_remove(uint64_t, int *); void bdb_update(const void *, size_t, const void *, size_t); #endif +void *alter(void *); void *backup(void *); void *compact(void *); void config_clear(void); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 283e2912daa..72e885bd0d6 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -28,14 +28,17 @@ #include "format.h" -static int col_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); -static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t); -static int col_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int col_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); +static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int col_update( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int nextprev(WT_CURSOR *, int); static void *ops(void *); -static int row_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); -static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t); -static int row_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int row_insert( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); +static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int row_update( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static void table_append_init(void); #ifdef HAVE_BERKELEY_DB @@ -50,10 +53,10 @@ static void print_item(const char *, WT_ITEM *); void wts_ops(int lastrun) { - TINFO *tinfo, total; + TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - pthread_t backup_tid, compact_tid, lrt_tid; + pthread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -61,6 +64,7 @@ wts_ops(int lastrun) conn = g.wts_conn; session = NULL; /* -Wconditional-uninitialized */ + memset(&alter_tid, 0, sizeof(alter_tid)); memset(&backup_tid, 0, sizeof(backup_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); memset(&lrt_tid, 0, sizeof(lrt_tid)); @@ -102,19 +106,24 @@ wts_ops(int lastrun) "=============== thread ops start ==============="); } - /* Create thread structure; start the worker threads. */ - tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo)); + /* + * Create the per-thread structures and start the worker threads. + * Allocate the thread structures separately to minimize false sharing. + */ + tinfo_list = dcalloc((size_t)g.c_threads, sizeof(TINFO *)); for (i = 0; i < g.c_threads; ++i) { - tinfo[i].id = (int)i + 1; - tinfo[i].state = TINFO_RUNNING; - testutil_check( - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); + tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); + tinfo->id = (int)i + 1; + tinfo->state = TINFO_RUNNING; + testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo)); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ + if (g.c_alter) + testutil_check(pthread_create(&alter_tid, NULL, alter, NULL)); if (g.c_backups) testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); if (g.c_compact) @@ -128,21 +137,22 @@ wts_ops(int lastrun) /* Clear out the totals each pass. */ memset(&total, 0, sizeof(total)); for (i = 0, running = 0; i < g.c_threads; ++i) { - total.commit += tinfo[i].commit; - total.deadlock += tinfo[i].deadlock; - total.insert += tinfo[i].insert; - total.remove += tinfo[i].remove; - total.rollback += tinfo[i].rollback; - total.search += tinfo[i].search; - total.update += tinfo[i].update; - - switch (tinfo[i].state) { + tinfo = tinfo_list[i]; + total.commit += tinfo->commit; + total.deadlock += tinfo->deadlock; + total.insert += tinfo->insert; + total.remove += tinfo->remove; + total.rollback += tinfo->rollback; + total.search += tinfo->search; + total.update += tinfo->update; + + switch (tinfo->state) { case TINFO_RUNNING: running = 1; break; case TINFO_COMPLETE: - tinfo[i].state = TINFO_JOINED; - (void)pthread_join(tinfo[i].tid, NULL); + tinfo->state = TINFO_JOINED; + (void)pthread_join(tinfo->tid, NULL); break; case TINFO_JOINED: break; @@ -154,7 +164,7 @@ wts_ops(int lastrun) */ if (fourths == 0 || (thread_ops != -1 && - tinfo[i].ops >= (uint64_t)thread_ops)) { + tinfo->ops >= (uint64_t)thread_ops)) { /* * On the last execution, optionally drop core * for recovery testing. @@ -163,7 +173,7 @@ wts_ops(int lastrun) static char *core = NULL; *core = 0; } - tinfo[i].quit = 1; + tinfo->quit = 1; } } track("ops", 0ULL, &total); @@ -173,10 +183,14 @@ wts_ops(int lastrun) if (fourths != -1) --fourths; } - free(tinfo); + for (i = 0; i < g.c_threads; ++i) + free(tinfo_list[i]); + free(tinfo_list); /* Wait for the backup, compaction, long-running reader threads. */ g.workers_finished = 1; + if (g.c_alter) + (void)pthread_join(alter_tid, NULL); if (g.c_backups) (void)pthread_join(backup_tid, NULL); if (g.c_compact) @@ -232,6 +246,9 @@ typedef struct { bool insert; /* Insert operation */ } SNAP_OPS; +#define SNAP_TRACK \ + (snap != NULL && (size_t)(snap - snap_list) < WT_ELEMENTS(snap_list)) + /* * snap_track -- * Add a single snapshot isolation returned value to the list. @@ -384,15 +401,16 @@ snap_check(WT_CURSOR *cursor, static void * ops(void *arg) { + enum { INSERT, READ, REMOVE, UPDATE } op; SNAP_OPS *snap, snap_list[64]; TINFO *tinfo; WT_CONNECTION *conn; - WT_CURSOR *cursor, *cursor_insert; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM *key, _key, *value, _value; WT_SESSION *session; uint64_t keyno, ckpt_op, reset_op, session_op; - uint32_t op, rnd; + uint32_t rnd; u_int i; int dir; char *ckpt_config, ckpt_name[64]; @@ -418,9 +436,9 @@ ops(void *arg) val_gen_setup(&tinfo->rnd, value); /* Set the first operation where we'll create sessions and cursors. */ - session_op = 0; + cursor = NULL; session = NULL; - cursor = cursor_insert = NULL; + session_op = 0; /* Set the first operation where we'll perform checkpoint operations. */ ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0; @@ -455,7 +473,7 @@ ops(void *arg) * 10% of the time, perform some read-only operations * from a checkpoint. * - * Skip that if we single-threaded and doing checks + * Skip that if we are single-threaded and doing checks * against a Berkeley DB database, because that won't * work because the Berkeley DB database records won't * match the checkpoint. Also skip if we are using @@ -474,24 +492,11 @@ ops(void *arg) readonly = true; } else { /* - * Open two cursors: one for overwriting and one - * for append (if it's a column-store). - * - * The reason is when testing with existing - * records, we don't track if a record was - * deleted or not, which means we must use - * cursor->insert with overwriting configured. - * But, in column-store files where we're - * testing with new, appended records, we don't - * want to have to specify the record number, - * which requires an append configuration. + * Configure "append", in the case of column + * stores, we append when inserting new rows. */ - testutil_check(session->open_cursor(session, - g.uri, NULL, "overwrite", &cursor)); - if (g.type == FIX || g.type == VAR) - testutil_check(session->open_cursor( - session, g.uri, - NULL, "append", &cursor_insert)); + testutil_check(session->open_cursor( + session, g.uri, NULL, "append", &cursor)); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -525,8 +530,9 @@ ops(void *arg) pthread_rwlock_trywrlock(&g.backup_lock) == EBUSY) ckpt_config = NULL; else { - (void)snprintf(ckpt_name, sizeof(ckpt_name), - "name=thread-%d", tinfo->id); + testutil_check(__wt_snprintf( + ckpt_name, sizeof(ckpt_name), + "name=thread-%d", tinfo->id)); ckpt_config = ckpt_name; } @@ -552,8 +558,9 @@ ops(void *arg) strcpy(ckpt_name, "checkpoint=WiredTigerCheckpoint"); else - (void)snprintf(ckpt_name, sizeof(ckpt_name), - "checkpoint=thread-%d", tinfo->id); + testutil_check(__wt_snprintf( + ckpt_name, sizeof(ckpt_name), + "checkpoint=thread-%d", tinfo->id)); ckpt_available = true; skip_checkpoint: /* Pick the next checkpoint operation. */ @@ -589,111 +596,174 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ intxn = true; } + /* Select a row. */ keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); positioned = false; + /* Select an operation. */ + op = READ; + if (!readonly) { + i = mmrand(&tinfo->rnd, 1, 100); + if (i < g.c_delete_pct) + op = REMOVE; + else if (i < g.c_delete_pct + g.c_insert_pct) + op = INSERT; + else if (i < + g.c_delete_pct + g.c_insert_pct + g.c_write_pct) + op = UPDATE; + else + op = READ; + } + /* - * Perform some number of operations: the percentage of deletes, - * inserts and writes are specified, reads are the rest. The - * percentages don't have to add up to 100, a high percentage - * of deletes will mean fewer inserts and writes. Modifications - * are always followed by a read to confirm it worked. + * Inserts, removes and updates can be done following a cursor + * set-key, or based on a cursor position taken from a previous + * search. If not already doing a read, position the cursor at + * an existing point in the tree 20% of the time. */ - op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100); - if (op < g.c_delete_pct) { - ++tinfo->remove; + positioned = false; + if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { + ++tinfo->search; + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + } +#if 0 + /* Optionally reserve a row. */ + if (!readonly && intxn && mmrand(&tinfo->rnd, 0, 20) == 1) { switch (g.type) { case ROW: - ret = row_remove(cursor, key, keyno); + ret = + row_reserve(cursor, key, keyno, positioned); break; case FIX: case VAR: - ret = col_remove(cursor, key, keyno); + ret = col_reserve(cursor, keyno, positioned); break; } if (ret == 0) { positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) - snap_track(snap++, keyno, NULL, NULL); + __wt_yield(); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == WT_NOTFOUND); } - } else if (op < g.c_delete_pct + g.c_insert_pct) { - ++tinfo->insert; + } +#endif + /* Perform the operation. */ + switch (op) { + case INSERT: switch (g.type) { case ROW: - key_gen_insert(&tinfo->rnd, key, keyno); - val_gen(&tinfo->rnd, value, keyno); - ret = row_insert(cursor, key, value, keyno); + ret = row_insert(tinfo, + cursor, key, value, keyno, positioned); break; case FIX: case VAR: /* - * We can only append so many new records, if - * we've reached that limit, update a record - * instead of doing an insert. + * We can only append so many new records, once + * we reach that limit, update a record instead + * of inserting. */ if (g.append_cnt >= g.append_max) - goto skip_insert; + goto update_instead_of_insert; - /* Insert, then reset the insert cursor. */ - val_gen(&tinfo->rnd, value, g.rows + 1); ret = col_insert( - cursor_insert, key, value, &keyno); - testutil_check( - cursor_insert->reset(cursor_insert)); + tinfo, cursor, key, value, &keyno); break; } + + /* Insert never leaves the cursor positioned. */ positioned = false; if (ret == 0) { - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) + ++tinfo->insert; + if (SNAP_TRACK) snap_track(snap++, keyno, g.type == ROW ? key : NULL, value); - } else + } else { if (ret == WT_ROLLBACK && intxn) goto deadlock; - } else if ( - op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { - ++tinfo->update; + testutil_assert(ret == 0 || ret == WT_ROLLBACK); + } + break; + case READ: + ++tinfo->search; + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + break; + case REMOVE: switch (g.type) { case ROW: - key_gen(key, keyno); - val_gen(&tinfo->rnd, value, keyno); - ret = row_update(cursor, key, value, keyno); + ret = + row_remove(cursor, key, keyno, positioned); break; case FIX: case VAR: -skip_insert: val_gen(&tinfo->rnd, value, keyno); - ret = col_update(cursor, key, value, keyno); + ret = + col_remove(cursor, key, keyno, positioned); break; } if (ret == 0) { - positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) - snap_track(snap++, keyno, NULL, value); + ++tinfo->remove; + /* + * Don't set positioned: it's unchanged from the + * previous state, but not necessarily set. + */ + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, NULL); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + break; + case UPDATE: +update_instead_of_insert: + ++tinfo->update; + + /* Update the row. */ + switch (g.type) { + case ROW: + ret = row_update(tinfo, + cursor, key, value, keyno, positioned); + break; + case FIX: + case VAR: + ret = col_update(tinfo, + cursor, key, value, keyno, positioned); + break; } - } else { - ++tinfo->search; - ret = read_row(cursor, key, value, keyno); if (ret == 0) { positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) + if (SNAP_TRACK) snap_track(snap++, keyno, NULL, value); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == 0 || ret == WT_ROLLBACK); } + break; } /* @@ -716,8 +786,8 @@ skip_insert: val_gen(&tinfo->rnd, value, keyno); testutil_check(cursor->reset(cursor)); /* - * If we're in a transaction, commit 40% of the time and - * rollback 10% of the time. + * Continue if not in a transaction, else add more operations + * to the transaction half the time. */ if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5) continue; @@ -730,6 +800,10 @@ skip_insert: val_gen(&tinfo->rnd, value, keyno); cursor, snap_list, snap, key, value)) == WT_ROLLBACK) goto deadlock; + /* + * If we're in a transaction, commit 40% of the time and + * rollback 10% of the time. + */ switch (rnd) { case 1: case 2: case 3: case 4: /* 40% */ testutil_check( @@ -1029,27 +1103,94 @@ nextprev(WT_CURSOR *cursor, int next) return (ret); } +#if 0 +/* + * row_reserve -- + * Reserve a row in a row-store file. + */ +static int +row_reserve(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } + + if (g.logging == LOG_OPS) + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, + "%-10s{%.*s}", "reserve", (int)key->size, key->data); + + switch (ret = cursor->reserve(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, + "row_reserve: reserve row %" PRIu64 " by key", keyno); + } + return (0); +} + +/* + * col_reserve -- + * Reserve a row in a column-store file. + */ +static int +col_reserve(WT_CURSOR *cursor, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + + if (!positioned) + cursor->set_key(cursor, keyno); + + if (g.logging == LOG_OPS) + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, + "%-10s%" PRIu64, "reserve", keyno); + + switch (ret = cursor->reserve(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, "col_reserve: %" PRIu64, keyno); + } + return (0); +} +#endif + /* * row_update -- * Update a row in a row-store file. */ static int -row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_update(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } + val_gen(&tinfo->rnd, value, keyno); + cursor->set_value(cursor, value); - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s{%.*s}, {%.*s}", "put", (int)key->size, key->data, (int)value->size, value->data); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); switch (ret = cursor->update(cursor)) { case 0: break; @@ -1075,32 +1216,32 @@ row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) * Update a row in a column-store file. */ static int -col_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +col_update(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) + cursor->set_key(cursor, keyno); + val_gen(&tinfo->rnd, value, keyno); + if (g.type == FIX) + cursor->set_value(cursor, *(uint8_t *)value->data); + else + cursor->set_value(cursor, value); - /* Log the operation */ if (g.logging == LOG_OPS) { if (g.type == FIX) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "update", keyno, ((uint8_t *)value->data)[0]); else - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {%.*s}", "update", keyno, (int)value->size, (char *)value->data); } - cursor->set_key(cursor, keyno); - if (g.type == FIX) - cursor->set_value(cursor, *(uint8_t *)value->data); - else - cursor->set_value(cursor, value); switch (ret = cursor->update(cursor)) { case 0: break; @@ -1227,22 +1368,29 @@ table_append(uint64_t keyno) * Insert a row in a row-store file. */ static int -row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_insert(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + /* + * If we positioned the cursor already, it's a test of an update using + * the insert method. Otherwise, generate a unique key and insert. + */ + if (!positioned) { + key_gen_insert(&tinfo->rnd, key, keyno); + cursor->set_key(cursor, key); + } + val_gen(&tinfo->rnd, value, keyno); + cursor->set_value(cursor, value); /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s{%.*s}, {%.*s}", "insert", (int)key->size, key->data, (int)value->size, value->data); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); switch (ret = cursor->insert(cursor)) { case 0: break; @@ -1268,14 +1416,13 @@ row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) * Insert an element in a column-store file. */ static int -col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) +col_insert(TINFO *tinfo, + WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) { WT_DECL_RET; - WT_SESSION *session; uint64_t keyno; - session = cursor->session; - + val_gen(&tinfo->rnd, value, g.rows + 1); if (g.type == FIX) cursor->set_value(cursor, *(uint8_t *)value->data); else @@ -1296,12 +1443,12 @@ col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) if (g.logging == LOG_OPS) { if (g.type == FIX) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "insert", keyno, ((uint8_t *)value->data)[0]); else - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {%.*s}", "insert", keyno, (int)value->size, (char *)value->data); @@ -1324,21 +1471,19 @@ col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) * Remove an row from a row-store file. */ static int -row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) +row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; - - key_gen(key, keyno); + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf( - g.wt_api, session, "%-10s%" PRIu64, "remove", keyno); + (void)g.wt_api->msg_printf(g.wt_api, + cursor->session, "%-10s%" PRIu64, "remove", keyno); - cursor->set_key(cursor, key); /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); @@ -1374,19 +1519,17 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) * Remove a row from a column-store file. */ static int -col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) +col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) + cursor->set_key(cursor, keyno); - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf( - g.wt_api, session, "%-10s%" PRIu64, "remove", keyno); + (void)g.wt_api->msg_printf(g.wt_api, + cursor->session, "%-10s%" PRIu64, "remove", keyno); - cursor->set_key(cursor, keyno); /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); @@ -1437,7 +1580,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) return (1); if (bdb_notfound) { - fprintf(stderr, "%s: %s:", g.progname, f); + fprintf(stderr, "%s: %s:", progname, f); if (keyno != 0) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, @@ -1445,7 +1588,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) testutil_die(0, NULL); } if (wt_ret == WT_NOTFOUND) { - fprintf(stderr, "%s: %s:", g.progname, f); + fprintf(stderr, "%s: %s:", progname, f); if (keyno != 0) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, diff --git a/src/third_party/wiredtiger/test/format/rebalance.c b/src/third_party/wiredtiger/test/format/rebalance.c index 9849b7df82b..e35c62e7255 100644 --- a/src/third_party/wiredtiger/test/format/rebalance.c +++ b/src/third_party/wiredtiger/test/format/rebalance.c @@ -41,10 +41,10 @@ wts_rebalance(void) track("rebalance", 0ULL, NULL); /* Dump the current object. */ - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" " -h %s dump -f %s/rebalance.orig %s", - g.home, g.home, g.uri); + g.home, g.home, g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Rebalance, then verify the object. */ @@ -66,21 +66,21 @@ wts_rebalance(void) wts_verify("post-rebalance verify"); wts_close(); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" " -h %s dump -f %s/rebalance.new %s", - g.home, g.home, g.uri); + g.home, g.home, g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Compare the old/new versions of the object. */ #ifdef _WIN32 - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "fc /b %s\\rebalance.orig %s\\rebalance.new > NUL", - g.home, g.home); + g.home, g.home)); #else - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", - g.home, g.home); + g.home, g.home)); #endif testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index 69805fb1018..f82dc34dd5f 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -70,29 +70,31 @@ corrupt(void) * It's a little tricky: if the data source is a file, we're looking * for "wt", if the data source is a table, we're looking for "wt.wt". */ - (void)snprintf(buf, sizeof(buf), "%s/%s", g.home, WT_NAME); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/%s", g.home, WT_NAME)); if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "copy %s\\%s %s\\slvg.copy\\%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #else - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s %s/slvg.copy/%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #endif goto found; } - (void)snprintf(buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME)); if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "copy %s\\%s.wt %s\\slvg.copy\\%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #else - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s.wt %s/slvg.copy/%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #endif goto found; } @@ -103,7 +105,8 @@ found: if (fstat(fd, &sb) == -1) offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); - (void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/slvg.corrupt", g.home)); if ((fp = fopen(buf, "w")) == NULL) testutil_die(errno, "salvage-corrupt: open: %s", buf); (void)fprintf(fp, diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 7701595776c..c6686ae8b91 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -49,14 +49,7 @@ main(int argc, char *argv[]) config = NULL; -#ifdef _WIN32 - g.progname = "t_format.exe"; -#else - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; -#endif + (void)testutil_set_progname(argv); #if 0 /* Configure the GNU malloc for debugging. */ @@ -74,7 +67,7 @@ main(int argc, char *argv[]) home = NULL; onerun = 0; while ((ch = __wt_getopt( - g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) + progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) switch (ch) { case '1': /* One run */ onerun = 1; @@ -179,7 +172,7 @@ main(int argc, char *argv[]) testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL)); testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); + printf("%s: process %" PRIdMAX "\n", progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { startup(); /* Start a run */ @@ -344,7 +337,7 @@ usage(void) "usage: %s [-1Llqr] [-C wiredtiger-config]\n " "[-c config-file] [-H mount] [-h home] " "[name=value ...]\n", - g.progname); + progname); fprintf(stderr, "%s", "\t-1 run once\n" "\t-C specify wiredtiger_open configuration arguments\n" diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index a709aa93a2e..983d03e2525 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -241,20 +241,23 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) void track(const char *tag, uint64_t cnt, TINFO *tinfo) { - static int lastlen = 0; - int len; + static size_t lastlen = 0; + size_t len; char msg[128]; if (g.c_quiet || tag == NULL) return; if (tinfo == NULL && cnt == 0) - len = snprintf(msg, sizeof(msg), "%4d: %s", g.run_cnt, tag); + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4d: %s", g.run_cnt, tag)); else if (tinfo == NULL) - len = snprintf( - msg, sizeof(msg), "%4d: %s: %" PRIu64, g.run_cnt, tag, cnt); + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, + "%4d: %s: %" PRIu64, g.run_cnt, tag, cnt)); else - len = snprintf(msg, sizeof(msg), + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4d: %s: " "search %" PRIu64 "%s, " "insert %" PRIu64 "%s, " @@ -268,7 +271,7 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, tinfo->update > M(9) ? "M" : "", tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : ""); + tinfo->remove > M(9) ? "M" : "")); if (lastlen > len) { memset(msg + len, ' ', (size_t)(lastlen - len)); @@ -297,27 +300,30 @@ path_setup(const char *home) /* Log file. */ len = strlen(g.home) + strlen("log") + 2; g.home_log = dmalloc(len); - snprintf(g.home_log, len, "%s/%s", g.home, "log"); + testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, "log")); /* RNG log file. */ len = strlen(g.home) + strlen("rand") + 2; g.home_rand = dmalloc(len); - snprintf(g.home_rand, len, "%s/%s", g.home, "rand"); + testutil_check(__wt_snprintf( + g.home_rand, len, "%s/%s", g.home, "rand")); /* Run file. */ len = strlen(g.home) + strlen("CONFIG") + 2; g.home_config = dmalloc(len); - snprintf(g.home_config, len, "%s/%s", g.home, "CONFIG"); + testutil_check(__wt_snprintf( + g.home_config, len, "%s/%s", g.home, "CONFIG")); /* Statistics file. */ len = strlen(g.home) + strlen("stats") + 2; g.home_stats = dmalloc(len); - snprintf(g.home_stats, len, "%s/%s", g.home, "stats"); + testutil_check(__wt_snprintf( + g.home_stats, len, "%s/%s", g.home, "stats")); /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); - snprintf(g.home_bdb, len, "%s/%s", g.home, "bdb"); + testutil_check(__wt_snprintf(g.home_bdb, len, "%s/%s", g.home, "bdb")); /* * Home directory initialize command: create the directory if it doesn't @@ -336,21 +342,23 @@ path_setup(const char *home) "cd %s & mkdir KVS" len = strlen(g.home) * 7 + strlen(CMD) + 1; g.home_init = dmalloc(len); - snprintf(g.home_init, len, CMD, - g.home, g.home, g.home, g.home, g.home, g.home, g.home); + testutil_check(__wt_snprintf(g.home_init, len, CMD, + g.home, g.home, g.home, g.home, g.home, g.home, g.home)); #else #define CMD "test -e %s || mkdir %s; " \ "cd %s > /dev/null && rm -rf `ls | sed /rand/d`; " \ "mkdir KVS" len = strlen(g.home) * 3 + strlen(CMD) + 1; g.home_init = dmalloc(len); - snprintf(g.home_init, len, CMD, g.home, g.home, g.home); + testutil_check(__wt_snprintf( + g.home_init, len, CMD, g.home, g.home, g.home)); #endif /* Primary backup directory. */ len = strlen(g.home) + strlen("BACKUP") + 2; g.home_backup = dmalloc(len); - snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + testutil_check(__wt_snprintf( + g.home_backup, len, "%s/%s", g.home, "BACKUP")); /* * Backup directory initialize command, remove and re-create the primary @@ -365,9 +373,9 @@ path_setup(const char *home) len = strlen(g.home) * 4 + strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, + testutil_check(__wt_snprintf(g.home_backup_init, len, CMD, g.home, "BACKUP", g.home, "BACKUP_COPY", - g.home, "BACKUP", g.home, "BACKUP_COPY"); + g.home, "BACKUP", g.home, "BACKUP_COPY")); /* * Salvage command, save the interesting files so we can replay the @@ -390,7 +398,7 @@ path_setup(const char *home) #endif len = strlen(g.home) + strlen(CMD) + 1; g.home_salvage_copy = dmalloc(len); - snprintf(g.home_salvage_copy, len, CMD, g.home); + testutil_check(__wt_snprintf(g.home_salvage_copy, len, CMD, g.home)); } /* @@ -459,3 +467,48 @@ fclose_and_clear(FILE **fpp) testutil_die(errno, "fclose"); return; } + +/* + * alter -- + * Periodically alter a table's metadata. + */ +void * +alter(void *arg) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + u_int period; + bool access_value; + char buf[32]; + + (void)(arg); + conn = g.wts_conn; + + /* + * Only alter the access pattern hint. If we alter the cache resident + * setting we may end up with a setting that fills cache and doesn't + * allow it to be evicted. + */ + access_value = false; + + /* Open a session */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + while (!g.workers_finished) { + period = mmrand(NULL, 1, 10); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "access_pattern_hint=%s", + access_value ? "random" : "none")); + access_value = !access_value; + if (session->alter(session, g.uri, buf) != 0) + break; + while (period > 0 && !g.workers_finished) { + --period; + sleep(1); + } + } + + testutil_check(session->close(session, NULL)); + return (NULL); +} diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index da234ce53c7..6aa4784d1c1 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -120,8 +120,15 @@ static WT_EVENT_HANDLER event_handler = { NULL /* Close handler. */ }; -#undef REMAIN -#define REMAIN(p, end) (size_t)((p) >= (end) ? 0 : (end) - (p)) +#define CONFIG_APPEND(p, ...) do { \ + size_t __len; \ + testutil_check( \ + __wt_snprintf_len_set(p, max, &__len, __VA_ARGS__)); \ + if (__len > max) \ + __len = max; \ + p += __len; \ + max -= __len; \ +} while (0) /* * wts_open -- @@ -132,42 +139,42 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) { WT_CONNECTION *conn; WT_DECL_RET; - char *config, *end, *p, helium_config[1024]; + size_t max; + char *config, *p, helium_config[1024]; *connp = NULL; config = p = g.wiredtiger_open_config; - end = config + sizeof(g.wiredtiger_open_config); + max = sizeof(g.wiredtiger_open_config); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "create=true," "cache_size=%" PRIu32 "MB," "checkpoint_sync=false," "error_prefix=\"%s\"", - g.c_cache, g.progname); + g.c_cache, progname); /* In-memory configuration. */ if (g.c_in_memory != 0) - p += snprintf(p, REMAIN(p, end), ",in_memory=1"); + CONFIG_APPEND(p, ",in_memory=1"); /* LSM configuration. */ if (DATASOURCE("lsm")) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", g.c_lsm_worker_threads); - if (DATASOURCE("lsm") || g.c_cache < 20) { - p += snprintf(p, REMAIN(p, end), ",eviction_dirty_trigger=95"); - } + if (DATASOURCE("lsm") || g.c_cache < 20) + CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); /* Eviction worker configuration. */ if (g.c_evict_max != 0) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); /* Logging configuration. */ if (g.c_logging) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d" ",compressor=\"%s\")", g.c_logging_archive ? 1 : 0, @@ -175,21 +182,21 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) compressor(g.c_logging_compression_flag)); if (g.c_encryption) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",encryption=(name=%s)", encryptor(g.c_encryption_flag)); /* Miscellaneous. */ #ifdef HAVE_POSIX_MEMALIGN - p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512"); + CONFIG_APPEND(p, ",buffer_alignment=512"); #endif - p += snprintf(p, REMAIN(p, end), ",mmap=%d", g.c_mmap ? 1 : 0); + CONFIG_APPEND(p, ",mmap=%d", g.c_mmap ? 1 : 0); if (g.c_direct_io) - p += snprintf(p, REMAIN(p, end), ",direct_io=(data)"); + CONFIG_APPEND(p, ",direct_io=(data)"); if (g.c_data_extend) - p += snprintf(p, REMAIN(p, end), ",file_extend=(data=8MB)"); + CONFIG_APPEND(p, ",file_extend=(data=8MB)"); /* * Run the statistics server and/or maintain statistics in the engine. @@ -198,18 +205,18 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if (g.c_statistics_server) { if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(fast)" ",statistics_log=(wait=5,sources=(\"file:\"))"); else - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(fast),statistics_log=(wait=5)"); } else - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); /* Extensions. */ - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",extensions=[" "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", @@ -227,11 +234,11 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) * override the standard configuration. */ if (g.c_config_open != NULL) - p += snprintf(p, REMAIN(p, end), ",%s", g.c_config_open); + CONFIG_APPEND(p, ",%s", g.c_config_open); if (g.config_open != NULL) - p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); + CONFIG_APPEND(p, ",%s", g.config_open); - if (REMAIN(p, end) == 0) + if (max == 0) testutil_die(ENOMEM, "wiredtiger_open configuration buffer too small"); @@ -259,12 +266,13 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if (DATASOURCE("helium")) { if (g.helium_mount == NULL) testutil_die(EINVAL, "no Helium mount point specified"); - (void)snprintf(helium_config, sizeof(helium_config), + testutil_check( + __wt_snprintf(helium_config, sizeof(helium_config), "entry=wiredtiger_extension_init,config=[" "helium_verbose=0," "dev1=[helium_devices=\"he://./%s\"," "helium_o_volume_truncate=1]]", - g.helium_mount); + g.helium_mount)); if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) testutil_die(ret, @@ -299,13 +307,13 @@ wts_init(void) { WT_CONNECTION *conn; WT_SESSION *session; + size_t max; uint32_t maxintlpage, maxintlkey, maxleafpage, maxleafkey, maxleafvalue; - char config[4096], *end, *p; + char config[4096], *p; conn = g.wts_conn; - p = config; - end = config + sizeof(config); + max = sizeof(config); /* * Ensure that we can service at least one operation per-thread @@ -326,7 +334,7 @@ wts_init(void) if (maxleafpage > 512) maxleafpage >>= 1; } - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "key_format=%s," "allocation_size=512,%s" "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, @@ -340,43 +348,35 @@ wts_init(void) */ maxintlkey = mmrand(NULL, maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) - p += snprintf(p, REMAIN(p, end), - ",internal_key_max=%" PRIu32, maxintlkey); + CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); maxleafkey = mmrand(NULL, maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) - p += snprintf(p, REMAIN(p, end), - ",leaf_key_max=%" PRIu32, maxleafkey); + CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); maxleafvalue = mmrand(NULL, maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) - p += snprintf(p, REMAIN(p, end), - ",leaf_value_max=%" PRIu32, maxleafvalue); + CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); switch (g.type) { case FIX: - p += snprintf(p, REMAIN(p, end), - ",value_format=%" PRIu32 "t", g.c_bitcnt); + CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); break; case ROW: if (g.c_huffman_key) - p += snprintf(p, REMAIN(p, end), - ",huffman_key=english"); + CONFIG_APPEND(p, ",huffman_key=english"); if (g.c_prefix_compression) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",prefix_compression_min=%" PRIu32, g.c_prefix_compression_min); else - p += snprintf(p, REMAIN(p, end), - ",prefix_compression=false"); + CONFIG_APPEND(p, ",prefix_compression=false"); if (g.c_reverse) - p += snprintf(p, REMAIN(p, end), - ",collator=reverse"); + CONFIG_APPEND(p, ",collator=reverse"); /* FALLTHROUGH */ case VAR: if (g.c_huffman_value) - p += snprintf(p, REMAIN(p, end), - ",huffman_value=english"); + CONFIG_APPEND(p, ",huffman_value=english"); if (g.c_dictionary) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } @@ -384,66 +384,63 @@ wts_init(void) /* Configure checksums. */ switch (g.c_checksum_flag) { case CHECKSUM_OFF: - p += snprintf(p, REMAIN(p, end), ",checksum=\"off\""); + CONFIG_APPEND(p, ",checksum=\"off\""); break; case CHECKSUM_ON: - p += snprintf(p, REMAIN(p, end), ",checksum=\"on\""); + CONFIG_APPEND(p, ",checksum=\"on\""); break; case CHECKSUM_UNCOMPRESSED: - p += snprintf(p, REMAIN(p, end), ",checksum=\"uncompressed\""); + CONFIG_APPEND(p, ",checksum=\"uncompressed\""); break; } /* Configure compression. */ if (g.c_compression_flag != COMPRESS_NONE) - p += snprintf(p, REMAIN(p, end), ",block_compressor=\"%s\"", + CONFIG_APPEND(p, ",block_compressor=\"%s\"", compressor(g.c_compression_flag)); /* Configure Btree internal key truncation. */ - p += snprintf(p, REMAIN(p, end), ",internal_key_truncate=%s", + CONFIG_APPEND(p, ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); /* Configure Btree page key gap. */ - p += snprintf(p, REMAIN(p, end), ",key_gap=%" PRIu32, g.c_key_gap); + CONFIG_APPEND(p, ",key_gap=%" PRIu32, g.c_key_gap); /* Configure Btree split page percentage. */ - p += snprintf(p, REMAIN(p, end), ",split_pct=%" PRIu32, g.c_split_pct); + CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); /* Configure LSM and data-sources. */ if (DATASOURCE("helium")) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",type=helium,helium_o_compress=%d,helium_o_truncate=1", g.c_compression_flag == COMPRESS_NONE ? 0 : 1); if (DATASOURCE("kvsbdb")) - p += snprintf(p, REMAIN(p, end), ",type=kvsbdb"); + CONFIG_APPEND(p, ",type=kvsbdb"); if (DATASOURCE("lsm")) { - p += snprintf(p, REMAIN(p, end), ",type=lsm,lsm=("); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",type=lsm,lsm=("); + CONFIG_APPEND(p, "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), - "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); + CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); /* * We can't set bloom_oldest without bloom, and we want to test * with Bloom filters on most of the time anyway. */ if (g.c_bloom_oldest) g.c_bloom = 1; - p += snprintf(p, REMAIN(p, end), - "bloom=%s,", g.c_bloom ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); + CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), - "merge_max=%" PRIu32 ",", g.c_merge_max); - p += snprintf(p, REMAIN(p, end), ",)"); + CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); + CONFIG_APPEND(p, ",)"); } - if (REMAIN(p, end) == 0) + if (max == 0) testutil_die(ENOMEM, "WT_SESSION.create configuration buffer too small"); @@ -490,14 +487,14 @@ wts_dump(const char *tag, int dump_bdb) len = strlen(g.home) + strlen(BERKELEY_DB_PATH) + strlen(g.uri) + 100; cmd = dmalloc(len); - (void)snprintf(cmd, len, + testutil_check(__wt_snprintf(cmd, len, "sh s_dumpcmp -h %s %s %s %s %s %s", g.home, dump_bdb ? "-b " : "", dump_bdb ? BERKELEY_DB_PATH : "", g.type == FIX || g.type == VAR ? "-c" : "", g.uri == NULL ? "" : "-n", - g.uri == NULL ? "" : g.uri); + g.uri == NULL ? "" : g.uri)); testutil_checkfmt(system(cmd), "%s: dump comparison failed", tag); free(cmd); @@ -587,7 +584,7 @@ wts_stats(void) fprintf(fp, "\n\n====== Data source statistics:\n"); len = strlen("statistics:") + strlen(g.uri) + 1; stat_name = dmalloc(len); - snprintf(stat_name, len, "statistics:%s", g.uri); + testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); testutil_check(session->open_cursor( session, stat_name, NULL, NULL, &cursor)); free(stat_name); diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index 17e2db353d5..2b0d5f498e3 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -29,7 +29,6 @@ #include "test_util.h" static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ static uint8_t *big; /* Big key/value buffer */ #define GIGABYTE (1073741824) @@ -167,14 +166,10 @@ main(int argc, char *argv[]) int ch, small; char *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); small = 0; working_dir = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "h:s")) != EOF) switch (ch) { case 'h': diff --git a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java new file mode 100644 index 00000000000..64f33f4d7b6 --- /dev/null +++ b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java @@ -0,0 +1,175 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +package com.wiredtiger.test; + +import com.wiredtiger.db.Connection; +import com.wiredtiger.db.Cursor; +import com.wiredtiger.db.SearchStatus; +import com.wiredtiger.db.Session; +import com.wiredtiger.db.WiredTigerPackingException; +import com.wiredtiger.db.WiredTigerException; +import com.wiredtiger.db.wiredtiger; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.Assert; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/* + * Test cases for WT-3238. + * + * Most WiredTiger methods return int, and our SWIG typemaps for Java add + * checking that throws exceptions for non-zero returns. Certain methods + * (Cursor.compare, Cursor.equals) are declared as returning int in Java, + * but should not throw exceptions for normal returns (which may be + * non-zero). + */ +public class CursorTest03 { + Connection conn; + Session s; + static String values[] = { "key0", "key1" }; + + @Test + public void cursor_int_methods() + throws WiredTigerPackingException { + setup(); + + Cursor c1 = s.open_cursor("table:t", null, null); + Cursor c2 = s.open_cursor("table:t", null, null); + for (String s : values) { + c1.putKeyString(s); + c1.putValueString(s); + c1.insert(); + } + c1.reset(); + + // "key1" compared to "key1" + c1.putKeyString(values[1]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + c2.putKeyString(values[1]); + Assert.assertEquals(c2.search_near(), SearchStatus.FOUND); + Assert.assertEquals(c1.compare(c2), 0); + Assert.assertEquals(c2.compare(c1), 0); + Assert.assertEquals(c1.compare(c1), 0); + Assert.assertEquals(c1.equals(c2), 1); + Assert.assertEquals(c2.equals(c1), 1); + Assert.assertEquals(c1.equals(c1), 1); + + // "key0" compared to "key1" + c1.putKeyString(values[0]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + Assert.assertEquals(c1.compare(c2), -1); + Assert.assertEquals(c2.compare(c1), 1); + Assert.assertEquals(c1.equals(c2), 0); + Assert.assertEquals(c2.equals(c1), 0); + + c1.close(); + c2.close(); + teardown(); + } + + public void expectException(Cursor c1, Cursor c2) + { + boolean caught = false; + try { + c1.compare(c2); + } + catch (WiredTigerException wte) { + caught = true; + } + Assert.assertTrue(caught); + + caught = false; + try { + c1.equals(c2); + } + catch (WiredTigerException wte) { + caught = true; + } + Assert.assertTrue(caught); + } + + @Test + public void cursor_int_methods_errors() + throws WiredTigerPackingException { + setup(); + + Cursor c1 = s.open_cursor("table:t", null, null); + Cursor c2 = s.open_cursor("table:t", null, null); + Cursor cx = s.open_cursor("table:t2", null, null); + for (String s : values) { + c1.putKeyString(s); + c1.putValueString(s); + c1.insert(); + cx.putKeyString(s); + cx.putValueString(s); + cx.insert(); + } + c1.reset(); + cx.reset(); + + // With both cursors not set, should be an exception. + expectException(c1, c2); + expectException(c1, c2); + + // With any one cursor not set, should be an exception. + c1.putKeyString(values[1]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + expectException(c1, c2); + expectException(c1, c2); + + // With two cursors from different tables, should be an exception. + cx.putKeyString(values[1]); + Assert.assertEquals(cx.search_near(), SearchStatus.FOUND); + expectException(c1, cx); + expectException(c1, cx); + + c1.close(); + c2.close(); + cx.close(); + teardown(); + } + + private void setup() { + conn = wiredtiger.open("WT_HOME", "create"); + s = conn.open_session(null); + s.create("table:t", "key_format=S,value_format=S"); + s.create("table:t2", "key_format=S,value_format=S"); + } + + private void teardown() { + s.drop("table:t", ""); + s.drop("table:t2", ""); + s.close(""); + conn.close(""); + } + +} + diff --git a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java index 5bd98d53fac..9322d30671a 100644 --- a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java +++ b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java @@ -38,6 +38,7 @@ import org.junit.runners.Suite; ConfigTest.class, CursorTest.class, CursorTest02.class, + CursorTest03.class, ExceptionTest.class, PackTest.class, PackTest02.class, diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index 7e986d47af3..42020d6ce9a 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -32,7 +32,6 @@ #define HOME_BASE "WT_TEST" static char home[HOME_SIZE]; /* Base home directory */ static char hometmp[HOME_SIZE]; /* Each conn home directory */ -static const char *progname; /* Program name */ static const char * const uri = "table:main"; #define WTOPEN_CFG_COMMON \ @@ -129,10 +128,8 @@ main(int argc, char *argv[]) const char *working_dir, *wt_cfg; char cmd[128]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); + dbs = MAX_DBS; working_dir = HOME_BASE; idle = false; @@ -171,7 +168,8 @@ main(int argc, char *argv[]) testutil_make_work_dir(home); __wt_random_init(&rnd); for (i = 0; i < dbs; ++i) { - snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); + testutil_check(__wt_snprintf( + hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i)); testutil_make_work_dir(hometmp); /* * Open each database. Rotate different configurations diff --git a/src/third_party/wiredtiger/test/mciproject.yml b/src/third_party/wiredtiger/test/mciproject.yml index eb74914eb46..6456475aa00 100644 --- a/src/third_party/wiredtiger/test/mciproject.yml +++ b/src/third_party/wiredtiger/test/mciproject.yml @@ -65,7 +65,7 @@ tasks: ./build_posix/reconf ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-verbose ${make_command|make} ${smp_command|} 2>&1 - ${make_command|make} VERBOSE=1 check 2>&1 + TESTUTIL_ENABLE_LONG_TESTS=1 ${make_command|make} VERBOSE=1 check 2>&1 fi - command: archive.targz_pack params: diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index a4b79f5859f..66c7a0ca692 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -39,7 +39,6 @@ static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; #define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; -static const char *progname; /* Program name */ static const char *saved_argv0; /* Program command */ static const char * const uri = "table:main"; @@ -172,10 +171,8 @@ main(int argc, char *argv[]) char cmd[512]; uint8_t buf[MAX_VAL]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); + /* * Needed unaltered for system command later. */ @@ -209,10 +206,12 @@ main(int argc, char *argv[]) * Set up all the directory names. */ testutil_work_dir_from_path(home, sizeof(home), working_dir); - (void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX); - (void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX); - (void)snprintf( - home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX); + testutil_check(__wt_snprintf( + home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX)); + testutil_check(__wt_snprintf( + home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX)); + testutil_check(__wt_snprintf( + home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX)); if (!child) { testutil_make_work_dir(home); testutil_make_work_dir(home_wr); @@ -271,22 +270,22 @@ main(int argc, char *argv[]) * Copy the database. Remove any lock file from one copy * and chmod the copies to be read-only permissions. */ - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", - home, home_wr, home_wr); + home, home_wr, home_wr)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd, home_rd, home_rd); + home, home_rd, home_rd, home_rd)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " "chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd2, home_rd2, home_rd2, home_rd2); + home, home_rd2, home_rd2, home_rd2, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); @@ -330,8 +329,8 @@ main(int argc, char *argv[]) * * The child will exit with success if its test passes. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -340,8 +339,8 @@ main(int argc, char *argv[]) /* * Scenario 2. Run child with writable config. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -361,8 +360,8 @@ main(int argc, char *argv[]) /* * Scenario 3. Child read-only. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -371,8 +370,8 @@ main(int argc, char *argv[]) /* * Scenario 4. Run child with writable config. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -393,11 +392,12 @@ main(int argc, char *argv[]) * We need to chmod the read-only databases back so that they can * be removed by scripts. */ - (void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", - home_rd, home_rd2); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", home_rd, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); printf(" *** Readonly test successful ***\n"); diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index 21460646017..febe6530534 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -31,10 +31,14 @@ #include #include -static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ +static char home[1024]; /* Program working dir */ + +/* + * These two names for the URI and file system must be maintained in tandem. + */ static const char * const uri = "table:main"; -bool inmem; +static const char * const fs_main = "main.wt"; +static bool inmem; #define MAX_TH 12 #define MIN_TH 5 @@ -90,14 +94,16 @@ thread_run(void *arg) /* * The value is the name of the record file with our id appended. */ - snprintf(buf, sizeof(buf), RECORDS_FILE, td->id); + testutil_check(__wt_snprintf(buf, sizeof(buf), RECORDS_FILE, td->id)); /* * Set up a large value putting our id in it. Write it in there a * bunch of times, but the rest of the buffer can just be zero. */ - snprintf(lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id); + testutil_check(__wt_snprintf( + lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id)); for (i = 0; i < 128; i += strlen(lgbuf)) - snprintf(&large[i], lsize - i, "%s", lgbuf); + testutil_check(__wt_snprintf( + &large[i], lsize - i, "%s", lgbuf)); /* * Keep a separate file with the records we wrote for checking. */ @@ -120,7 +126,8 @@ thread_run(void *arg) * Write our portion of the key space until we're killed. */ for (i = td->start; ; ++i) { - snprintf(kname, sizeof(kname), "%" PRIu64, i); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, i)); cursor->set_key(cursor, kname); /* * Every 30th record write a very large record that exceeds the @@ -211,6 +218,7 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { + struct stat sb; FILE *fp; WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -222,12 +230,9 @@ main(int argc, char *argv[]) pid_t pid; bool fatal, rand_th, rand_time, verify_only; const char *working_dir; - char fname[64], kname[64]; + char fname[64], kname[64], statname[1024]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); inmem = false; nth = MIN_TH; @@ -263,7 +268,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - testutil_work_dir_from_path(home, 512, working_dir); + testutil_work_dir_from_path(home, sizeof(home), working_dir); /* * If the user wants to verify they need to tell us how many threads * there were so we can find the old record files. @@ -305,8 +310,16 @@ main(int argc, char *argv[]) /* parent */ /* * Sleep for the configured amount of time before killing - * the child. + * the child. Start the timeout from the time we notice that + * the table has been created. That allows the test to run + * correctly on really slow machines. Verify the process ID + * still exists in case the child aborts for some reason we + * don't stay in this loop forever. */ + testutil_check(__wt_snprintf( + statname, sizeof(statname), "%s/%s", home, fs_main)); + while (stat(statname, &sb) != 0 && kill(pid, 0) == 0) + sleep(1); sleep(timeout); /* @@ -339,12 +352,10 @@ main(int argc, char *argv[]) fatal = false; for (i = 0; i < nth; ++i) { middle = 0; - snprintf(fname, sizeof(fname), RECORDS_FILE, i); - if ((fp = fopen(fname, "r")) == NULL) { - fprintf(stderr, - "Failed to open %s. i %" PRIu32 "\n", fname, i); - testutil_die(errno, "fopen"); - } + testutil_check(__wt_snprintf( + fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); /* * For every key in the saved file, verify that the key exists @@ -370,7 +381,8 @@ main(int argc, char *argv[]) fname, key, last_key); break; } - snprintf(kname, sizeof(kname), "%" PRIu64, key); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, key)); cursor->set_key(cursor, kname); if ((ret = cursor->search(cursor)) != 0) { if (ret != WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/recovery/truncated-log.c b/src/third_party/wiredtiger/test/recovery/truncated-log.c index c265263d44c..a127d8c1c63 100644 --- a/src/third_party/wiredtiger/test/recovery/truncated-log.c +++ b/src/third_party/wiredtiger/test/recovery/truncated-log.c @@ -30,13 +30,7 @@ #include -#ifdef _WIN32 -/* snprintf is not supported on <= VS2013 */ -#define snprintf _snprintf -#endif - -static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ +static char home[1024]; /* Program working dir */ static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -138,7 +132,8 @@ usage(void) * Child process creates the database and table, and then writes data into * the table until it is killed by the parent. */ -static void fill_db(void)WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void fill_db(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(void) { @@ -194,9 +189,9 @@ fill_db(void) max_key = min_key * 2; first = true; for (i = 0; i < max_key; ++i) { - snprintf(k, sizeof(k), "key%03d", (int)i); - snprintf(v, sizeof(v), "value%0*d", - (int)(V_SIZE - strlen("value")), (int)i); + testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i)); + testutil_check(__wt_snprintf(v, sizeof(v), "value%0*d", + (int)(V_SIZE - (strlen("value") + 1)), (int)i)); cursor->set_key(cursor, k); cursor->set_value(cursor, v); if ((ret = cursor->insert(cursor)) != 0) @@ -271,10 +266,7 @@ main(int argc, char *argv[]) pid_t pid; const char *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); working_dir = "WT_TEST.truncated-log"; while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF) @@ -290,7 +282,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - testutil_work_dir_from_path(home, 512, working_dir); + testutil_work_dir_from_path(home, sizeof(home), working_dir); testutil_make_work_dir(home); /* diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index b8553bbd72d..83f9c6349bc 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -54,8 +54,6 @@ void run(int); void t(int, u_int, int); int usage(void); -static const char *progname; /* Program name */ - static FILE *res_fp; /* Results file */ static u_int page_type; /* File types */ static int value_unique; /* Values are unique */ @@ -70,10 +68,7 @@ main(int argc, char *argv[]) u_int ptype; int ch, r; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); r = 0; ptype = 0; @@ -445,7 +440,8 @@ run(int r) process(); - snprintf(buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT)); if (system(buf)) { fprintf(stderr, "check failed, salvage results were incorrect\n"); @@ -490,28 +486,28 @@ build(int ikey, int ivalue, int cnt) switch (page_type) { case WT_PAGE_COL_FIX: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=r,value_format=7t," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_COL_VAR: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=r," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_ROW_LEAF: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=u," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; default: assert(0); @@ -525,7 +521,8 @@ build(int ikey, int ivalue, int cnt) case WT_PAGE_COL_VAR: break; case WT_PAGE_ROW_LEAF: - snprintf(kbuf, sizeof(kbuf), "%010d KEY------", ikey); + testutil_check(__wt_snprintf( + kbuf, sizeof(kbuf), "%010d KEY------", ikey)); key.data = kbuf; key.size = 20; cursor->set_key(cursor, &key); @@ -538,8 +535,8 @@ build(int ikey, int ivalue, int cnt) break; case WT_PAGE_COL_VAR: case WT_PAGE_ROW_LEAF: - snprintf(vbuf, sizeof(vbuf), - "%010d VALUE----", value_unique ? ivalue : 37); + testutil_check(__wt_snprintf(vbuf, sizeof(vbuf), + "%010d VALUE----", value_unique ? ivalue : 37)); value.data = vbuf; value.size = 20; cursor->set_value(cursor, &value); @@ -626,9 +623,9 @@ process(void) /* Salvage. */ config[0] = '\0'; if (verbose) - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "error_prefix=\"%s\",verbose=[salvage,verify],", - progname); + progname)); strcat(config, "log=(enabled=false),"); CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0); diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index ba6d9f78503..97c58bfdccf 100644 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -324,7 +324,8 @@ if __name__ == '__main__': # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, - verbose, dirarg, longtest) + verbose, wt_builddir, dirarg, + longtest) # Without any tests listed as arguments, do discovery if len(testargs) == 0: diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py new file mode 100644 index 00000000000..dfdf6b7a17e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter01.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_alter01.py +# Smoke-test the session alter operations. +class test_alter01(wttest.WiredTigerTestCase): + name = "alter01" + entries = 100 + # Settings for access_pattern_hint + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + hints = [ + ('default', dict(acreate='')), + ('none', dict(acreate='none')), + ('random', dict(acreate='random')), + ('sequential', dict(acreate='sequential')), + ] + access_alter=('', 'none', 'random', 'sequential') + # Settings for cache_resident + resid = [ + ('default', dict(ccreate='')), + ('false', dict(ccreate='false')), + ('true', dict(ccreate='true')), + ] + reopen = [ + ('no-reopen', dict(reopen=False)), + ('reopen', dict(reopen=True)), + ] + cache_alter=('', 'false', 'true') + scenarios = make_scenarios(types, hints, resid, reopen) + + def verify_metadata(self, metastr): + if metastr == '': + return + cursor = self.session.open_cursor('metadata:', None, None) + # + # Walk through all the metadata looking for the entries that are + # the file URIs for components of the table. + # + found = False + while True: + ret = cursor.next() + if ret != 0: + break + key = cursor.get_key() + check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ + and key.find(self.name) != -1) + if check_meta: + value = cursor[key] + found = True + self.assertTrue(value.find(metastr) != -1) + cursor.close() + self.assertTrue(found == True) + + # Alter: Change the access pattern hint after creation + def test_alter01_access(self): + uri = self.uri + self.name + create_params = 'key_format=i,value_format=i,' + complex_params = '' + # + # If we're not explicitly setting the parameter, then don't + # modify create_params to test using the default. + # + if self.acreate != '': + access_param = 'access_pattern_hint=%s' % self.acreate + create_params += '%s,' % access_param + complex_params += '%s,' % access_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + access_param = 'access_pattern_hint=none' + if self.ccreate != '': + cache_param = 'cache_resident=%s' % self.ccreate + create_params += '%s,' % cache_param + complex_params += '%s,' % cache_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + cache_param = 'cache_resident=false' + + cgparam = '' + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + self.session.create(uri, create_params + cgparam) + # Add in column group or index settings. + if self.use_cg: + cgparam = 'columns=(v),' + suburi = 'colgroup:' + self.name + ':g0' + self.session.create(suburi, complex_params + cgparam) + if self.use_index: + suburi = 'index:' + self.name + ':i0' + self.session.create(suburi, complex_params + cgparam) + + # Put some data in table. + c = self.session.open_cursor(uri, None) + for k in range(self.entries): + c[k+1] = 1 + c.close() + + # Verify the string in the metadata + self.verify_metadata(access_param) + self.verify_metadata(cache_param) + + # Run through all combinations of the alter commands + # for all allowed settings. This tests having only one or + # the other set as well as having both set. It will also + # cover trying to change the setting to its current value. + for a in self.access_alter: + alter_param = '' + access_str = '' + if a != '': + access_str = 'access_pattern_hint=%s' % a + for c in self.cache_alter: + alter_param = '%s' % access_str + cache_str = '' + if c != '': + cache_str = 'cache_resident=%s' % c + alter_param += ',%s' % cache_str + if alter_param != '': + self.session.alter(uri, alter_param) + if self.reopen: + self.reopen_conn() + special = self.use_cg or self.use_index + if not special: + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + else: + self.session.alter(suburi, alter_param) + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_async01.py b/src/third_party/wiredtiger/test/suite/test_async01.py index cbb3dad8de6..158c16a9381 100644 --- a/src/third_party/wiredtiger/test/suite/test_async01.py +++ b/src/third_party/wiredtiger/test/suite/test_async01.py @@ -132,7 +132,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): ]) # Enable async for this test. - def conn_config(self, dir): + def conn_config(self): return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ 'threads=%s)' % self.async_threads diff --git a/src/third_party/wiredtiger/test/suite/test_async02.py b/src/third_party/wiredtiger/test/suite/test_async02.py index 50652da6dfd..28435fe85b2 100644 --- a/src/third_party/wiredtiger/test/suite/test_async02.py +++ b/src/third_party/wiredtiger/test/suite/test_async02.py @@ -129,7 +129,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): ]) # Enable async for this test. - def conn_config(self, dir): + def conn_config(self): return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ 'threads=%s)' % self.async_threads diff --git a/src/third_party/wiredtiger/test/suite/test_backup03.py b/src/third_party/wiredtiger/test/suite/test_backup03.py index 73d05f0b0a1..c1ed3cc9e1a 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup03.py +++ b/src/third_party/wiredtiger/test/suite/test_backup03.py @@ -74,7 +74,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): ('backup_9', dict(big=3,list=[])), # Backup everything ] - scenarios = make_scenarios(list) + scenarios = make_scenarios(list, prune=3, prunelong=1000) # Create a large cache, otherwise this test runs quite slowly. conn_config = 'cache_size=1G' diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index 919649fed57..be52a5e1e97 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -60,7 +60,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): ]) # Create a large cache, otherwise this test runs quite slowly. - def conn_config(self, dir): + def conn_config(self): return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \ self.logmax diff --git a/src/third_party/wiredtiger/test/suite/test_bug011.py b/src/third_party/wiredtiger/test/suite/test_bug011.py index 29bb08ec2e5..5e0721b93f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug011.py +++ b/src/third_party/wiredtiger/test/suite/test_bug011.py @@ -30,25 +30,28 @@ import random, wiredtiger, wttest from wtdataset import SimpleDataSet # test_bug011.py -# Eviction working on more files than there are hazard pointers. +# Eviction working on more trees than the eviction server can walk +# simultaneously. There is a builtin limit of 1000 trees, we open double +# that, which makes this a long-running test. class test_bug011(wttest.WiredTigerTestCase): """ Test having eviction working on more files than the number of allocated hazard pointers. """ table_name = 'test_bug011' - ntables = 50 + ntables = 2000 nrows = 10000 nops = 10000 # Add connection configuration for this test. - def conn_config(self, dir): - return 'cache_size=10MB,eviction_dirty_target=99,eviction_dirty_trigger=99,hazard_max=' + str(self.ntables / 2) + def conn_config(self): + return 'cache_size=1GB' + @wttest.longtest("Eviction copes with lots of files") def test_eviction(self): cursors = [] datasets = [] for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) ds = SimpleDataSet(self, this_uri, self.nrows, config='allocation_size=1KB,leaf_page_max=1KB') ds.populate() @@ -57,9 +60,9 @@ class test_bug011(wttest.WiredTigerTestCase): # Switch over to on-disk trees with multiple leaf pages self.reopen_conn() - # Make sure we have a cursor for the table so it stays in cache. + # Make sure we have a cursor for every table so it stays in cache. for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) cursors.append(self.session.open_cursor(this_uri, None)) # Make use of the cache. diff --git a/src/third_party/wiredtiger/test/suite/test_collator.py b/src/third_party/wiredtiger/test/suite/test_collator.py index 3fae4ff47cb..7ce135c8976 100644 --- a/src/third_party/wiredtiger/test/suite/test_collator.py +++ b/src/third_party/wiredtiger/test/suite/test_collator.py @@ -48,34 +48,10 @@ class test_collator(wttest.WiredTigerTestCase): nentries = 100 nindices = 4 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor'), - ('collators', 'revint', 'revint_collator')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') + extlist.extension('collators', 'revint') def create_indices(self): # Create self.nindices index files, each with a column from the CSV diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index fe31aa1cbe1..803600eea14 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -38,7 +38,7 @@ from wtscenario import make_scenarios class test_compact02(wttest.WiredTigerTestCase): types = [ - ('file', dict(uri='file:test_compact02')), + ('table', dict(uri='table:test_compact02')), ] cacheSize = [ ('default', dict(cacheSize='')), @@ -69,8 +69,9 @@ class test_compact02(wttest.WiredTigerTestCase): # 1. Create a table with the data, alternating record size. # 2. Checkpoint and get stats on the table to confirm the size. # 3. Delete the half of the records with the larger record size. - # 4. Call compact. - # 5. Get stats on compacted table. + # 4. Checkpoint so compact finds something to work with. + # 5. Call compact. + # 6. Get stats on compacted table. # nrecords = 22000 bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666 @@ -80,8 +81,11 @@ class test_compact02(wttest.WiredTigerTestCase): # Return the size of the file def getSize(self): + # To allow this to work on systems without ftruncate, + # get the portion of the file allocated, via 'statistics=(all)', + # not the physical file size, via 'statistics=(size)'. cstat = self.session.open_cursor( - 'statistics:' + self.uri, None, 'statistics=(size)') + 'statistics:' + self.uri, None, 'statistics=(all)') sz = cstat[stat.dsrc.block_size][2] cstat.close() return sz @@ -95,8 +99,9 @@ class test_compact02(wttest.WiredTigerTestCase): def ConnectionOpen(self, cacheSize): self.home = '.' conn_params = 'create,' + \ - cacheSize + ',error_prefix="%s: ",' % self.shortid() + \ - 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99' + cacheSize + ',error_prefix="%s",' % self.shortid() + \ + 'statistics=(all),' + \ + 'eviction_dirty_target=99,eviction_dirty_trigger=99' try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: @@ -137,10 +142,13 @@ class test_compact02(wttest.WiredTigerTestCase): c.close() self.pr('Removed total ' + str((count * 9666) / mb) + 'MB') - # 4. Call compact. + # 4. Checkpoint + self.session.checkpoint() + + # 5. Call compact. self.session.compact(self.uri, None) - # 5. Get stats on compacted table. + # 6. Get stats on compacted table. sz = self.getSize() self.pr('After compact ' + str(sz / mb) + 'MB') diff --git a/src/third_party/wiredtiger/test/suite/test_compress01.py b/src/third_party/wiredtiger/test/suite/test_compress01.py index 606f7b63235..ef1064d294e 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress01.py +++ b/src/third_party/wiredtiger/test/suite/test_compress01.py @@ -51,22 +51,10 @@ class test_compress01(wttest.WiredTigerTestCase): nrecords = 10000 bigvalue = "abcdefghij" * 1000 - # Load the compression extension, compression is enabled elsewhere. - def conn_config(self, dir): - return self.extensionArg(self.compress) - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None: - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + # Load the compression extension, skip the test if missing + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) # Create a table, add keys with both big and small values, then verify them. def test_compress(self): diff --git a/src/third_party/wiredtiger/test/suite/test_config03.py b/src/third_party/wiredtiger/test/suite/test_config03.py index 88ca6ae3f39..89038d71319 100644 --- a/src/third_party/wiredtiger/test/suite/test_config03.py +++ b/src/third_party/wiredtiger/test/suite/test_config03.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!usr/bin/env python # # Public Domain 2014-2016 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. @@ -48,8 +48,6 @@ class test_config03(test_base03.test_base03): eviction_trigger_scenarios = wtscenario.quick_scenarios( 's_eviction_trigger', [50, 90, 95, 99], None) - hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max', - [15, 50, 500], [0.4, 0.8, 0.8]) multiprocess_scenarios = wtscenario.quick_scenarios('s_multiprocess', [True,False], [1.0,1.0]) session_max_scenarios = wtscenario.quick_scenarios('s_session_max', @@ -66,14 +64,14 @@ class test_config03(test_base03.test_base03): verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None) config_vars = [ 'cache_size', 'create', 'error_prefix', 'eviction_target', - 'eviction_trigger', 'hazard_max', 'multiprocess', - 'session_max', 'verbose' ] + 'eviction_trigger', 'multiprocess', 'session_max', + 'verbose' ] scenarios = wtscenario.make_scenarios( cache_size_scenarios, create_scenarios, error_prefix_scenarios, eviction_target_scenarios, eviction_trigger_scenarios, - hazard_max_scenarios, multiprocess_scenarios, session_max_scenarios, - transactional_scenarios, verbose_scenarios, prune=1000) + multiprocess_scenarios, session_max_scenarios, + transactional_scenarios, verbose_scenarios, prune=100, prunelong=1000) #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ # str(len(scenarios)) + ' of ' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index 204aa7e27d5..db8a5f4a16a 100644 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py @@ -154,10 +154,6 @@ class test_config04(wttest.WiredTigerTestCase): 'eviction_trigger=86'), "/eviction target must be lower than the eviction trigger/") - def test_hazard_max(self): - # Note: There isn't any direct way to know that this was set. - self.common_test('hazard_max=50') - def test_invalid_config(self): msg = '/Unbalanced brackets/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_cursor01.py b/src/third_party/wiredtiger/test/suite/test_cursor01.py index 8c66042eec0..2a33b15dccc 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor01.py @@ -99,6 +99,7 @@ class test_cursor01(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor(tablearg, None, None) self.assertCursorHasNoKeyValue(cursor) + self.assertEqual(cursor.uri, tablearg) for i in range(0, self.nentries): cursor[self.genkey(i)] = self.genvalue(i) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor07.py b/src/third_party/wiredtiger/test/suite/test_cursor07.py index d6078183fc1..19db718fd11 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor07.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor07.py @@ -49,7 +49,7 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): ('reopen', dict(reopen=True)) ]) # Enable logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' diff --git a/src/third_party/wiredtiger/test/suite/test_cursor08.py b/src/third_party/wiredtiger/test/suite/test_cursor08.py index 3f8f50defa7..cc76f528aa9 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor08.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor08.py @@ -54,24 +54,14 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): ] scenarios = make_scenarios(reopens, compress) # Load the compression extension, and enable it for logging. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ 'compressor=%s),' % self.compress + \ - 'transaction_sync="(method=dsync,enabled)",' + \ - self.extensionArg(self.compress) + 'transaction_sync="(method=dsync,enabled)"' - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None or name == 'none': - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py new file mode 100644 index 00000000000..6cabfde9f1f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_cursor10.py +# Cursors with projections. +class test_cursor10(wttest.WiredTigerTestCase): + """ + Test cursor search and search_near + """ + table_name1 = 'test_cursor10' + nentries = 20 + + scenarios = make_scenarios([ + ('row', dict(key_format='S', uri='table')), + ('col', dict(key_format='r', uri='table')) + ]) + + def genkey(self, i): + if self.key_format == 'S': + return 'key' + str(i).zfill(5) # return key00001, key00002, etc. + else: + return long(i+1) + + def genvalue(self, i): + return [ 'v0:' + str(i), i+1, 'v2' + str(i+2), i+3 ] + + def extractkey(self, k): + if self.key_format == 'S': + return int(k[3:]) + else: + return long(k-1) + + def test_projection(self): + """ + Create entries, and read back in a regular and projected cursor + """ + tablearg = self.uri + ":" + self.table_name1 + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg, None, None) + count = 0 + for k,v0,v1,v2,v3 in cursor: + i = self.extractkey(k) + self.assertEqual(self.genkey(i), k) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + + def test_index_projection(self): + """ + Create entries, and read back in an index cursor with a projection + """ + tablearg = self.uri + ":" + self.table_name1 + indexarg = 'index:' + self.table_name1 + ':index1' + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + self.session.create(indexarg, 'columns=(v0,v2,v1,v3)') + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor11.py b/src/third_party/wiredtiger/test/suite/test_cursor11.py new file mode 100644 index 00000000000..e159ec499e6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor11.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet +from wtscenario import make_scenarios + +# test_cursor11.py +# WT_CURSOR position tests: remove (if not already positioned), and insert +# leave the cursor without position or information. +class test_cursor11(wttest.WiredTigerTestCase): + + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm', ds=SimpleDataSet)), + ('table-complex', dict(uri='table', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') + + # Do a remove using the cursor after setting a position, and confirm + # the key and position remain set but no value. + def test_cursor_remove_with_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + self.assertEquals(c.search(), 0) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(26)) + c.remove() + self.assertEquals(c.get_key(), ds.key(26)) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(27)) + + # Do a remove using the cursor without setting a position, and confirm + # no key, value or position remains. + def test_cursor_remove_without_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + c.remove() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + + # Do a remove using the key after also setting a position, and confirm + # no key, value or position remains. + def test_cursor_remove_with_key_and_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + self.assertEquals(c.search(), 0) + c.set_key(ds.key(25)) + c.remove() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + + # Do an insert and confirm no key, value or position remains. + def test_cursor_insert(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + c.set_value(ds.value(300)) + c.insert() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 3bda6dc9946..ee0f85a29ee 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -71,6 +71,15 @@ class test_cursor_random(wttest.WiredTigerTestCase): self.assertEquals(cursor.reset(), 0) cursor.close() + # Check that next_random fails with an empty tree, repeatedly. + def test_cursor_random_empty(self): + uri = self.type + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,5): + self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) + cursor.close + # Check that next_random works with a single value, repeatedly. def test_cursor_random_single_record(self): uri = self.type @@ -127,6 +136,46 @@ class test_cursor_random(wttest.WiredTigerTestCase): def test_cursor_random_multiple_page_records(self): self.cursor_random_multiple_page_records(0) + # Check that next_random fails in the presence of a set of values, some of + # which are deleted. + def test_cursor_random_deleted_partial(self): + uri = self.type + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() + + # Close the connection so everything is forced to disk. + self.reopen_conn() + + start = self.session.open_cursor(uri, None) + start.set_key(ds.key(10)) + end = self.session.open_cursor(uri, None) + end.set_key(ds.key(10000-10)) + self.session.truncate(None, start, end, None) + self.assertEqual(start.close(), 0) + self.assertEqual(end.close(), 0) + + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,10): + self.assertEqual(cursor.next(), 0) + + # Check that next_random fails in the presence of a set of values, all of + # which are deleted. + def test_cursor_random_deleted_all(self): + uri = self.type + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() + + # Close the connection so everything is forced to disk. + self.reopen_conn() + + self.session.truncate(uri, None, None, None) + + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,10): + self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) + # Check that opening a random cursor on column-store returns not-supported. class test_cursor_random_column(wttest.WiredTigerTestCase): scenarios = make_scenarios([ diff --git a/src/third_party/wiredtiger/test/suite/test_dump.py b/src/third_party/wiredtiger/test/suite/test_dump.py index f6a83c32489..3127c7aef00 100644 --- a/src/third_party/wiredtiger/test/suite/test_dump.py +++ b/src/third_party/wiredtiger/test/suite/test_dump.py @@ -32,7 +32,7 @@ import wiredtiger, wttest from suite_subprocess import suite_subprocess from wtscenario import make_scenarios from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \ - ComplexDataSet, ComplexLSMDataSet + ComplexDataSet, ComplexLSMDataSet, ProjectionDataSet, ProjectionIndexDataSet # test_dump.py # Utilities: wt dump @@ -62,6 +62,10 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)), + ('table-simple-proj', dict(uri='table:', + dataset=ProjectionDataSet, projection=True)), + ('table-index-proj', dict(uri='table:', + dataset=ProjectionIndexDataSet, projection=True)), ] scenarios = make_scenarios(types, keyfmt, dumpfmt) @@ -158,5 +162,53 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) pop.check() +# test_dump_projection +# Utilities: wt dump +# Test the dump utility with projections +class test_dump_projection(wttest.WiredTigerTestCase, suite_subprocess): + dir = 'dump.dir' # Backup directory name + + name = 'test_dump' + nentries = 2500 + uri = 'table:' + + # Dump, re-load and do a content comparison. + def test_dump(self): + + # Create the object. + uri = self.uri + self.name + pop = ProjectionDataSet(self, uri, self.nentries, key_format='S') + pop.populate() + + # Check some cases with invalid projections. + self.runWt(['dump', '-x', uri + '('], \ + outfilename='bad1.out', errfilename='err1.out', failure=True) + self.check_non_empty_file('err1.out') + self.runWt(['dump', '-x', uri + '(xx)'], \ + outfilename='bad2.out', errfilename='err2.out', failure=True) + self.check_non_empty_file('err2.out') + self.runWt(['dump', '-x', uri + pop.projection[:-1]], \ + outfilename='bad3.out', errfilename='err3.out', failure=True) + self.check_non_empty_file('err3.out') + + # Dump the object with a valid projection. + self.runWt(['dump', '-x', uri + pop.projection], outfilename='dump.out') + + # Re-load the object in a new home. + os.mkdir(self.dir) + self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) + + # Check the database contents. + self.runWt(['list'], outfilename='list.out') + self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') + s1 = set(open('list.out').read().split()) + s2 = set(open('list.out.new').read().split()) + self.assertEqual(not s1.symmetric_difference(s2), True) + + # Check the object's contents. + self.reopen_conn(self.dir) + pop_reload = ProjectionDataSet(self, uri, self.nentries, key_format='S') + pop_reload.check() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py index 746c9d13e96..317bed93246 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py @@ -66,41 +66,20 @@ class test_encrypt01(wttest.WiredTigerTestCase): nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) + extlist.extension('compressors', self.block_compress) + extlist.extension('compressors', self.log_compress) + + def conn_config(self): encarg = 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) comparg = '' if self.log_compress != None: comparg='log=(compressor={0}),'.format(self.log_compress) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt), - ('compressors', self.block_compress), - ('compressors', self.log_compress)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg)) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + return encarg + comparg # Create a table, add keys with both big and small values, then verify them. def test_encrypt(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt02.py b/src/third_party/wiredtiger/test/suite/test_encrypt02.py index 648686274c4..d950be067e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt02.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt02.py @@ -39,51 +39,27 @@ from wtscenario import make_scenarios class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): uri = 'file:test_encrypt02' encrypt_type = [ - ('noarg', dict( encrypt='rotn', encrypt_args='name=rotn', - secret_arg=None)), - ('keyid', dict( encrypt='rotn', encrypt_args='name=rotn,keyid=11', - secret_arg=None)), - ('pass', dict( encrypt='rotn', encrypt_args='name=rotn', - secret_arg='ABC')), - ('keyid-pass', dict( encrypt='rotn', encrypt_args='name=rotn,keyid=11', - secret_arg='ABC')), + ('noarg', dict( encrypt_args='name=rotn', secret_arg=None)), + ('keyid', dict( encrypt_args='name=rotn,keyid=11', secret_arg=None)), + ('pass', dict( encrypt_args='name=rotn', secret_arg='ABC')), + ('keyid-pass', dict( + encrypt_args='name=rotn,keyid=11', secret_arg='ABC')), ] scenarios = make_scenarios(encrypt_type) + def conn_extensions(self, extlist): + # Load the compression extension, skip the test if missing + extlist.skip_if_missing = True + extlist.extension('encryptors', 'rotn') + nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_config(self): secretarg = '' if self.secret_arg != None: secretarg = ',secretkey=' + self.secret_arg - encarg = 'encryption=({0}{1})'.format(self.encrypt_args, secretarg) - extarg = self.extensionArg([('encryptors', self.encrypt)]) - connarg = 'create,error_prefix="{0}: ",{1},{2}'.format( - self.shortid(), encarg, extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + return 'encryption=({0}{1})'.format(self.encrypt_args, secretarg) # Create a table, add keys with both big and small values, then verify them. def test_pass(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt03.py b/src/third_party/wiredtiger/test/suite/test_encrypt03.py index cf459190637..302572bd044 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt03.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt03.py @@ -50,37 +50,14 @@ class test_encrypt03(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(types, encrypt) - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt)]) - self.pr('encarg = ' + encarg + ' extarg = ' + extarg) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( - self.shortid(), encarg, extarg)) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( + self.sys_encrypt, self.sys_encrypt_args) # Create a table with encryption values that are in error. def test_encrypt(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt04.py b/src/third_party/wiredtiger/test/suite/test_encrypt04.py index a244cf97961..19c0b85d427 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt04.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt04.py @@ -77,9 +77,16 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): wttest.WiredTigerTestCase.__init__(self, *args, **kwargs) self.part = 1 + def conn_extensions(self, extlist): + extarg = None + if self.expect_forceerror: + extarg='(config=\"rotn_force_error=true\")' + extlist.skip_if_missing = True + extlist.extension('encryptors', self.name, extarg) + # Override WiredTigerTestCase, we have extensions. def setUpConnectionOpen(self, dir): - forceerror = None + self.expect_forceerror = False if self.part == 1: self.name = self.name1 self.keyid = self.keyid1 @@ -93,21 +100,20 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.fileinclear = self.fileinclear2 if \ hasattr(self, 'fileinclear2') else False if hasattr(self, 'forceerror1') and hasattr(self, 'forceerror2'): - forceerror = "rotn_force_error=true" - self.expect_forceerror = forceerror != None + self.expect_forceerror = True self.got_forceerror = False encarg = 'encryption=(name={0},keyid={1},secretkey={2}),'.format( self.name, self.keyid, self.secretkey) - # If forceerror is set for this test, add a config arg to - # the extension string. That signals rotn to return a (-1000) - # error code, which we'll detect here. - extarg = self.extensionArg([('encryptors', self.name, forceerror)]) + # If forceerror is set for this test, conn_extensions adds a + # config arg to the extension string. That signals rotn to + # return a (-1000) error code, which we'll detect here. + extarg = self.extensionsConfig() self.pr('encarg = ' + encarg + ' extarg = ' + extarg) completed = False try: conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( + 'create,error_prefix="{0}",{1}{2}'.format( self.shortid(), encarg, extarg)) except (BaseException) as err: # Capture the recognizable error created by rotn @@ -135,29 +141,6 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(cursor.search(), 0) self.assertEquals(cursor.get_value(), val) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, extarg) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - extfile = '"' + extfile + '"' - if not extfile in extfiles: - s = extfile - if extarg != None: - s += "=(config=\"" + extarg + "\")" - extfiles.append(s) - if len(extfiles) == 0: - return '' - else: - return ',extensions=[' + ','.join(extfiles) + ']' - # Evaluate expression, which either must succeed (if expect_okay) # or must fail (if !expect_okay). def check_okay(self, expect_okay, expr): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt05.py b/src/third_party/wiredtiger/test/suite/test_encrypt05.py index 19a3522b3d5..d8862321821 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt05.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt05.py @@ -49,41 +49,20 @@ class test_encrypt05(wttest.WiredTigerTestCase): nrecords = 500 bigvalue = 'a' * 500 # we use values that will definitely give compression - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) + extlist.extension('compressors', self.block_compress) + extlist.extension('compressors', self.log_compress) + + def conn_config(self): encarg = 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) comparg = '' if self.log_compress != None: comparg='log=(compressor={0}),'.format(self.log_compress) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt), - ('compressors', self.block_compress), - ('compressors', self.log_compress)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg)) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + return encarg + comparg def getvalue(self, r, n): if n < len(self.bigvalue): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt06.py b/src/third_party/wiredtiger/test/suite/test_encrypt06.py index 893c4ba3095..72718e53b2b 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt06.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt06.py @@ -89,38 +89,15 @@ class test_encrypt06(wttest.WiredTigerTestCase): scenarios = make_scenarios(encrypt, storagetype) nrecords = 1000 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file0_encrypt) + extlist.extension('encryptors', self.file1_encrypt) + + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) - comparg = '' - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file0_encrypt), - ('encryptors', self.file1_encrypt)]) - self.open_params = 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg) - conn = self.wiredtiger_open(dir, self.open_params) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' def encrypt_file_params(self, name, args): if name == None: diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt07.py b/src/third_party/wiredtiger/test/suite/test_encrypt07.py index 97ab1987d4f..81c9f1a49ea 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt07.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt07.py @@ -44,35 +44,14 @@ class test_encrypt07(test_salvage.test_salvage): nrecords = 5000 bigvalue = "abcdefghij" * 1007 # len(bigvalue) = 10070 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - extarg = self.extensionArg([('encryptors', self.sys_encrypt)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( - self.shortid(), encarg, extarg)) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + # Load the compression extension, skip the test if missing + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( + self.sys_encrypt, self.sys_encrypt_args) def rot13(self, s): return codecs.encode(s, 'rot_13') diff --git a/src/third_party/wiredtiger/test/suite/test_hazard.py b/src/third_party/wiredtiger/test/suite/test_hazard.py new file mode 100644 index 00000000000..f2891fce526 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hazard.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_hazard.py +# Hazard pointer tests. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# Regression tests. +class test_hazard(wttest.WiredTigerTestCase): + + # Allocate a large number of hazard pointers in a session, forcing the + # hazard pointer array to repeatedly grow. + def test_hazard(self): + uri = "table:hazard" + ds = SimpleDataSet(self, uri, 1000) + ds.populate() + + # Open 10,000 cursors and pin a page to set a hazard pointer. + cursors = [] + for i in range(0, 10000): + c = self.session.open_cursor(uri, None) + c.set_key(ds.key(10)) + c.search() + cursors.append(c) + + # Close the cursors, clearing the hazard pointer. + for c in cursors: + c.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_index02.py b/src/third_party/wiredtiger/test/suite/test_index02.py index 9f39df003b1..4f424e5d3d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_index02.py +++ b/src/third_party/wiredtiger/test/suite/test_index02.py @@ -27,20 +27,26 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_index02.py # test search_near in indices class test_index02(wttest.WiredTigerTestCase): '''Test search_near in indices''' + scenarios = make_scenarios([ + ('index', dict(indexconfig='columns=(v)', ncol=1)), + ('index-with-key', dict(indexconfig='columns=(v,k)', ncol=2)), + ]) + basename = 'test_index02' tablename = 'table:' + basename indexname = 'index:' + basename + ":inverse" - def test_search_near(self): - '''Create a table, look for a nonexistent key''' + def test_search_near_exists(self): + '''Create a table, look for an existing key''' self.session.create(self.tablename, 'key_format=r,value_format=Q,columns=(k,v)') - self.session.create(self.indexname, 'columns=(v)') + self.session.create(self.indexname, self.indexconfig) cur = self.session.open_cursor(self.tablename, None, "append") cur.set_value(1) cur.insert() @@ -52,17 +58,60 @@ class test_index02(wttest.WiredTigerTestCase): cur.insert() cur.set_value(10) cur.insert() + cur.close() + + # Retry after reopening + for runs in xrange(2): + # search near should find a match + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(5) + else: + cur.set_key(5, 3) + self.assertEqual(cur.search_near(), 0) + + # Retry after reopening + self.reopen_conn() + + def test_search_near_between(self): + '''Create a table, look for a non-existing key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + cur = self.session.open_cursor(self.tablename) + for k in xrange(3): + cur[k] = 5 * k + 10 + cur.close() + + search_keys = [ 1, 11, 15, 19, 21 ] # search near should find a match - cur2 = self.session.open_cursor(self.indexname, None, None) - cur2.set_key(5) - self.assertEqual(cur2.search_near(), 0) + for runs in xrange(2): + cur = self.session.open_cursor(self.indexname, None, None) + for k in search_keys: + if self.ncol == 1: + cur.set_key(k) + else: + cur.set_key(k, 1) # [15,1] will completely match + exact = cur.search_near() + if self.ncol == 1: + found_key = cur.get_key() + else: + [ found_key, index ] = cur.get_key() + self.pr("search_near for " + str(k) + " found " + str(found_key) + " with exact " + str(exact)) + self.assertEqual(exact, cmp(found_key, k), "for key " + str(k)) + self.reopen_conn() - # Retry after reopening - self.reopen_conn() - cur3 = self.session.open_cursor(self.indexname, None, None) - cur3.set_key(5) - self.assertEqual(cur3.search_near(), 0) + def test_search_near_empty(self): + '''Create an empty table, look for a key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(3) + else: + cur.set_key(3, 1) + self.assertEqual(cur.search_near(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 1af43bbd9d9..388485db29b 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -59,7 +59,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) # Figure out the last key we successfully inserted, and check all # previous inserts are still there. @@ -77,7 +77,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. @@ -93,7 +93,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() @@ -120,5 +120,50 @@ class test_inmem01(wttest.WiredTigerTestCase): break self.assertTrue(inserted) + # Custom "keep filling" helper + def fill(self, cursor, ds, start, end): + for i in xrange(start + 1, end + 1): + cursor[ds.key(i)] = ds.value(i) + + # Keep adding data to the cache until it becomes really full, make sure + # that reads aren't blocked. + @wttest.longtest("Try to wedge an in-memory cache") + def test_wedge(self): + # Try to really wedge the cache full + ds = SimpleDataSet(self, self.uri, 0, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.populate() + cursor = self.session.open_cursor(self.uri, None) + + run = 0 + start, last_key = -1000, 0 + while last_key - start > 100: + msg = '/WT_CACHE_FULL.*/' + start = last_key + self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, + lambda: self.fill(cursor, ds, start, 10000000), msg) + cursor.reset() + sleep(1) + + # Figure out the last key we successfully inserted, and check all + # previous inserts are still there. + cursor.prev() + last_key = int(cursor.get_key()) + run += 1 + self.pr('Finished iteration ' + str(run) + ', last_key = ' + str(last_key)) + + self.pr('Checking ' + str(last_key) + ' keys') + ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + + # This test is *much* slower for fixed-length column stores: we fit + # many more records into the cache, so don't do as many passes through + # the data. + checks = 10 if self.valuefmt.endswith('t') else 100 + for run in xrange(checks): + ds.check() + self.pr('Finished check ' + str(run)) + sleep(1) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_intpack.py b/src/third_party/wiredtiger/test/suite/test_intpack.py index b0cece09494..ae391e68fca 100644 --- a/src/third_party/wiredtiger/test/suite/test_intpack.py +++ b/src/third_party/wiredtiger/test/suite/test_intpack.py @@ -126,8 +126,8 @@ class PackTester: class test_intpack(wttest.WiredTigerTestCase): name = 'test_intpack' - # We have to be a bit verbose here with naming, as there can be problems with - # case insensitive test names:w + # We have to be a bit verbose here with naming, scenario names are + # case insensitive and must be unique. scenarios = make_scenarios([ ('int8_t_b', dict(formatcode='b', low=-128, high=127, nbits=8)), diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index f3b13026896..bdd86a06d4f 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -69,7 +69,7 @@ class test_join01(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(type_scen, bloom0_scen, bloom1_scen, projection_scen, nested_scen, stats_scen, - order_scen) + order_scen, prune=50, prunelong=1000) # We need statistics for these tests. conn_config = 'statistics=(all)' @@ -110,8 +110,9 @@ class test_join01(wttest.WiredTigerTestCase): while jc.next() == 0: [k] = jc.get_keys() i = k - 1 - if do_proj: # our projection test simply reverses the values - [v2,v1,v0] = jc.get_values() + if do_proj: # our projection reverses the values and adds the key + [v2,v1,v0,kproj] = jc.get_values() + self.assertEquals(k, kproj) else: [v0,v1,v2] = jc.get_values() self.assertEquals(self.gen_values(i), [v0,v1,v2]) @@ -136,7 +137,7 @@ class test_join01(wttest.WiredTigerTestCase): if self.ref == 'index': expectstats.append('join: index:join01:index0: ' + statdesc) elif self.do_proj: - expectstats.append('join: table:join01(v2,v1,v0): ' + statdesc) + expectstats.append('join: table:join01(v2,v1,v0,k): ' + statdesc) else: expectstats.append('join: table:join01: ' + statdesc) self.check_stats(statcur, expectstats) @@ -228,7 +229,7 @@ class test_join01(wttest.WiredTigerTestCase): c.close() if do_proj: - proj_suffix = '(v2,v1,v0)' # Reversed values + proj_suffix = '(v2,v1,v0,k)' # Reversed values plus key else: proj_suffix = '' # Default projection (v0,v1,v2) diff --git a/src/third_party/wiredtiger/test/suite/test_join03.py b/src/third_party/wiredtiger/test/suite/test_join03.py index edab7146a6b..dd8111f6ead 100644 --- a/src/third_party/wiredtiger/test/suite/test_join03.py +++ b/src/third_party/wiredtiger/test/suite/test_join03.py @@ -36,33 +36,9 @@ class test_join03(wttest.WiredTigerTestCase): table_name1 = 'test_join03' nentries = 100 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def gen_key(self, i): return [ i + 1 ] diff --git a/src/third_party/wiredtiger/test/suite/test_join04.py b/src/third_party/wiredtiger/test/suite/test_join04.py index a71418d9f05..e65b8b53333 100644 --- a/src/third_party/wiredtiger/test/suite/test_join04.py +++ b/src/third_party/wiredtiger/test/suite/test_join04.py @@ -36,33 +36,9 @@ class test_join04(wttest.WiredTigerTestCase): table_name1 = 'test_join04' nentries = 100 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') # JIRA WT-2308: # Test extractors with equality joins diff --git a/src/third_party/wiredtiger/test/suite/test_join07.py b/src/third_party/wiredtiger/test/suite/test_join07.py index 2a32e678d72..8fae3539246 100644 --- a/src/third_party/wiredtiger/test/suite/test_join07.py +++ b/src/third_party/wiredtiger/test/suite/test_join07.py @@ -200,33 +200,9 @@ class test_join07(wttest.WiredTigerTestCase): scenarios = make_scenarios(extractscen) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def expect(self, token, expected): if token == None or token.kind not in expected: diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump02.py b/src/third_party/wiredtiger/test/suite/test_jsondump02.py index 8482851fb94..60863c4aa97 100644 --- a/src/third_party/wiredtiger/test/suite/test_jsondump02.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump02.py @@ -234,6 +234,24 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): ('"ikey" : 4,\n"Skey" : "key4"', '"S1" : "val16",\n"i2" : 16,\n"S3" : "val64",\n"i4" : 64')) self.check_json(self.table_uri4, table4_json) + # This projection has 3 value fields reversed with a key at the end. + table4_json_projection = ( + ('"ikey" : 1,\n"Skey" : "key1"', + '"i4" : 1,\n"S3" : "val1",\n"i2" : 1,\n"ikey" : 1'), + ('"ikey" : 2,\n"Skey" : "key2"', + '"i4" : 8,\n"S3" : "val8",\n"i2" : 4,\n"ikey" : 2'), + ('"ikey" : 3,\n"Skey" : "key3"', + '"i4" : 27,\n"S3" : "val27",\n"i2" : 9,\n"ikey" : 3'), + ('"ikey" : 4,\n"Skey" : "key4"', + '"i4" : 64,\n"S3" : "val64",\n"i2" : 16,\n"ikey" : 4')) + # bad projection URI + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.check_json(self.table_uri4 + '(i4,S3,i2,ikey', + table4_json_projection), + '/Unbalanced brackets/') + # This projection should work. + self.check_json(self.table_uri4 + '(i4,S3,i2,ikey)', + table4_json_projection) # The dump config currently is not supported for the index type. self.check_json(uri4index1, ( ('"Skey" : "key1"', diff --git a/src/third_party/wiredtiger/test/suite/test_lsm01.py b/src/third_party/wiredtiger/test/suite/test_lsm01.py index b44df4bae14..f705b09b0a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_lsm01.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm01.py @@ -57,7 +57,7 @@ class test_lsm01(wttest.WiredTigerTestCase): scenarios = wtscenario.make_scenarios( chunk_size_scenarios, merge_max_scenarios, bloom_scenarios, bloom_bit_scenarios, bloom_hash_scenarios, record_count_scenarios, - prune=500) + prune=100, prunelong=500) # Test drop of an object. def test_lsm(self): diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap04.py b/src/third_party/wiredtiger/test/suite/test_nsnap04.py index 60901dd2ee3..8d491540d74 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap04.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap04.py @@ -38,14 +38,18 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename nrows_per_itr = 10 - def check_named_snapshot(self, snapshot, expected): + def check_named_snapshot(self, snapshot, expected, skip_snapshot=False): new_session = self.conn.open_session() c = new_session.open_cursor(self.uri) - new_session.begin_transaction("snapshot=" + str(snapshot)) + if skip_snapshot: + new_session.begin_transaction() + else: + new_session.begin_transaction("snapshot=" + str(snapshot)) count = 0 for row in c: count += 1 new_session.commit_transaction() + new_session.close() # print "Checking snapshot %d, expect %d, found %d" % (snapshot, expected, count) self.assertEqual(count, expected) @@ -80,5 +84,34 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=0") self.check_named_snapshot(0, 2 * self.nrows_per_itr) + def test_include_updates(self): + # Populate a table + end = start = 0 + SimpleDataSet(self, self.uri, 0, key_format='i').populate() + + snapshots = [] + c = self.session.open_cursor(self.uri) + for i in xrange(self.nrows_per_itr): + c[i] = "some value" + + self.session.begin_transaction("isolation=snapshot") + count = 0 + for row in c: + count += 1 + self.session.snapshot("name=0,include_updates=true") + + self.check_named_snapshot(0, self.nrows_per_itr) + + # Insert some more content using the active session. + for i in xrange(self.nrows_per_itr): + c[self.nrows_per_itr + i] = "some value" + + self.check_named_snapshot(0, 2 * self.nrows_per_itr) + # Ensure transactions not tracking the snapshot don't see the updates + self.check_named_snapshot(0, self.nrows_per_itr, skip_snapshot=True) + self.session.commit_transaction() + # Ensure content is visible to non-snapshot transactions after commit + self.check_named_snapshot(0, 2 * self.nrows_per_itr, skip_snapshot=True) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_overwrite.py b/src/third_party/wiredtiger/test/suite/test_overwrite.py index 4739abaa578..c894de99bd0 100644 --- a/src/third_party/wiredtiger/test/suite/test_overwrite.py +++ b/src/third_party/wiredtiger/test/suite/test_overwrite.py @@ -27,32 +27,47 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from wtdataset import SimpleDataSet +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_overwrite.py # cursor overwrite configuration method class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' - scenarios = make_scenarios([ - ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), - ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), - ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), - ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), - ]) + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file:', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm:', ds=SimpleDataSet)), + ('table-complex', dict(uri='table:', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table:', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table:', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table:', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1000)) self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) # One additional test for the insert method: duplicate the cursor with @@ -63,30 +78,33 @@ class test_overwrite(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") - dupc.set_value('XXXXXXXXXX') + dupc.set_value(ds.value(1001)) self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1002)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1003)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1004)) self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Remove of an existing record with overwrite off succeeds. @@ -110,32 +128,35 @@ class test_overwrite(wttest.WiredTigerTestCase): self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1005)) self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1006)) self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1007)) self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1008)) self.assertEquals(cursor.update(), 0) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_perf001.py b/src/third_party/wiredtiger/test/suite/test_perf001.py index b22ed2baeb0..6331a3f64d6 100644 --- a/src/third_party/wiredtiger/test/suite/test_perf001.py +++ b/src/third_party/wiredtiger/test/suite/test_perf001.py @@ -40,7 +40,8 @@ class test_perf001(wttest.WiredTigerTestCase): scenarios = make_scenarios([ #('file-file', dict(tabletype='file',indextype='file')), - ('file-lsm', dict(tabletype='file',indextype='lsm')), + ('file-lsm', dict(tabletype='file',indextype='lsm', cfg='', + conn_config="statistics=(fast),statistics_log=(wait=1)")), #('lsm-file', dict(tabletype='lsm',indextype='file')), #('lsm-lsm', dict(tabletype='lsm',indextype='lsm')), ]) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py index e4b431ca1da..f41280a3283 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly01.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -75,8 +75,7 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios(basecfg_list, dir_list, log_list, types) - def conn_config(self, dir): - self.home = dir + def conn_config(self): params = \ 'error_prefix="%s",' % self.shortid() + \ '%s' % self.logcfg + \ diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig01.py b/src/third_party/wiredtiger/test/suite/test_reconfig01.py index e76becac76a..cbc8bca5740 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig01.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig01.py @@ -64,6 +64,18 @@ class test_reconfig01(wttest.WiredTigerTestCase): # same ops_max of 512 and thread of 8. self.conn.reconfigure("async=(enabled=true)") + def test_reconfig_eviction(self): + # Increase the max number of running threads (default 8). + self.conn.reconfigure("eviction=(threads_max=10)") + # Increase the min number of running threads (default 1). + self.conn.reconfigure("eviction=(threads_min=5)") + # Decrease the max number of running threads. + self.conn.reconfigure("eviction=(threads_max=7)") + # Decrease the min number of running threads. + self.conn.reconfigure("eviction=(threads_min=2)") + # Set min and max the same. + self.conn.reconfigure("eviction=(threads_min=6,threads_max=6)") + def test_reconfig_lsm_manager(self): # We create and populate a tiny LSM so that we can start off with # the LSM threads running and change the numbers of threads. diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index 36a78a1805f..042d3bbe71f 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -62,7 +62,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.reconfigure("log=(path=foo)"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.reconfigure("log=(recovery=true)"), msg) + lambda: self.conn.reconfigure("log=(recover=true)"), msg) # Logging starts on, but prealloc is off. Verify it is off. # Reconfigure it on and run again, making sure that log files @@ -109,6 +109,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): # Now turn on archive, sleep a bit to allow the archive thread # to run and then confirm that all original logs are gone. self.conn.reconfigure("log=(archive=true)") + self.session.checkpoint("force") time.sleep(2) cur_logs = fnmatch.filter(os.listdir('.'), "*Log*") for o in orig_logs: diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig04.py b/src/third_party/wiredtiger/test/suite/test_reconfig04.py new file mode 100644 index 00000000000..51d9b91c1f4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_reconfig04.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_reconfig04.py +# Test WT_SESSION::reconfigure +class test_reconfig04(wttest.WiredTigerTestCase): + def test_session_reconfigure(self): + self.session.reconfigure('ignore_cache_size=false') + + self.session.reconfigure('isolation=snapshot') + self.session.reconfigure('isolation=read-committed') + self.session.reconfigure('isolation=read-uncommitted') + + self.session.reconfigure('ignore_cache_size=true') + self.session.reconfigure('isolation=snapshot') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_schema05.py b/src/third_party/wiredtiger/test/suite/test_schema05.py index 28ad51b3c92..d536a629373 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema05.py +++ b/src/third_party/wiredtiger/test/suite/test_schema05.py @@ -57,33 +57,9 @@ class test_schema05(wttest.WiredTigerTestCase): ('index-after', { 'create_index' : 2 }), ]) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def create_indices(self): # Create self.nindices index files, each with a column from the CSV diff --git a/src/third_party/wiredtiger/test/suite/test_schema07.py b/src/third_party/wiredtiger/test/suite/test_schema07.py index ac397c6e1a1..3e4b1d28a4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema07.py +++ b/src/third_party/wiredtiger/test/suite/test_schema07.py @@ -33,8 +33,7 @@ import wiredtiger, wttest class test_schema07(wttest.WiredTigerTestCase): tablename = 'table:test_schema07' - def conn_config(self, dir): - return 'cache_size=10MB' + conn_config = 'cache_size=10MB' @wttest.longtest("Creating many tables shouldn't fill the cache") def test_many_tables(self): diff --git a/src/third_party/wiredtiger/test/suite/test_shared_cache01.py b/src/third_party/wiredtiger/test/suite/test_shared_cache01.py index 70560a625ee..c3bd946cc4b 100644 --- a/src/third_party/wiredtiger/test/suite/test_shared_cache01.py +++ b/src/third_party/wiredtiger/test/suite/test_shared_cache01.py @@ -73,7 +73,7 @@ class test_shared_cache01(wttest.WiredTigerTestCase): os.mkdir(name) next_conn = self.wiredtiger_open( name, - 'create,error_prefix="' + self.shortid() + ': "' + + 'create,error_prefix="%s",' % self.shortid() + pool_opts + extra_opts) self.conns.append(next_conn) self.sessions.append(next_conn.open_session(None)) diff --git a/src/third_party/wiredtiger/test/suite/test_shared_cache02.py b/src/third_party/wiredtiger/test/suite/test_shared_cache02.py index 7cde6c86695..67f9bf7c6b7 100644 --- a/src/third_party/wiredtiger/test/suite/test_shared_cache02.py +++ b/src/third_party/wiredtiger/test/suite/test_shared_cache02.py @@ -73,7 +73,7 @@ class test_shared_cache02(wttest.WiredTigerTestCase): os.mkdir(name) next_conn = self.wiredtiger_open( name, - 'create,error_prefix="' + self.shortid() + ': "' + + 'create,error_prefix="%s",' % self.shortid() + pool_opts + extra_opts) self.conns.append(next_conn) self.sessions.append(next_conn.open_session(None)) diff --git a/src/third_party/wiredtiger/test/suite/test_stat02.py b/src/third_party/wiredtiger/test/suite/test_stat02.py index cecda7f1ddc..45af283ed02 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat02.py +++ b/src/third_party/wiredtiger/test/suite/test_stat02.py @@ -59,7 +59,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): scenarios = make_scenarios(uri, data_config, cursor_config) # Turn on statistics for this test. - def conn_config(self, dir): + def conn_config(self): return 'statistics=(%s)' % self.data_config # For each database/cursor configuration, confirm the right combinations diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index 71f8fcb180e..5559190caca 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -116,10 +116,15 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # Give slow machines time to process files. stat_cursor = self.session.open_cursor('statistics:', None, None) this_nfile = stat_cursor[stat.conn.file_open][2] + removed = stat_cursor[stat.conn.dh_sweep_remove][2] stat_cursor.close() self.pr("==== loop " + str(sleep)) self.pr("this_nfile " + str(this_nfile)) - if this_nfile == final_nfile: + self.pr("removed " + str(removed)) + # On slow machines there can be a lag where files get closed but + # the sweep server cannot yet remove the handles. So wait for the + # removed statistic to indicate forward progress too. + if this_nfile == final_nfile and removed != remove1: break c.close() self.pr("Sweep loop took " + str(sleep)) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index 2319eeddbef..98b741ba6a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -128,6 +128,7 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase): msg = '/the start cursor position is after the stop cursor position/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) + c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.session.truncate(None, c1, c2, None) @@ -183,11 +184,11 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # those tests to file objects. types = [ ('file', dict(type='file:', valuefmt='S', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.25)), ('file8t', dict(type='file:', valuefmt='8t', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.25)), ('table', dict(type='table:', valuefmt='S', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.5)), ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -203,7 +204,8 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): ('big', dict(nentries=1000,skip=37)), ] - scenarios = make_scenarios(types, keyfmt, size, reopen) + scenarios = make_scenarios(types, keyfmt, size, reopen, + prune=10, prunelong=1000) # Set a cursor key. def cursorKey(self, ds, uri, key): diff --git a/src/third_party/wiredtiger/test/suite/test_truncate02.py b/src/third_party/wiredtiger/test/suite/test_truncate02.py index 73fed362354..729825b26d4 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate02.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate02.py @@ -85,7 +85,8 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): ('txn2', dict(commit=False)), ] - scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn) + scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn, + prune=20, prunelong=1000) # Return the number of records visible to the cursor; test both forward # and backward iteration, they are different code paths in this case. diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index a0c2c12a47c..01626057b9e 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -93,11 +93,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): checklog_calls = 100 if wttest.islongtest() else 2 checklog_mod = (len(scenarios) / checklog_calls + 1) - def setUpConnectionOpen(self, dir): - self.home = dir + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] # # We don't want to run zero fill with only the same settings, such @@ -107,17 +106,9 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): zerofill = 'false' if self.scenario_number % freq == 0: zerofill = 'true' - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'log=(zero_fill=%s),' % zerofill + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'log=(zero_fill=%s),' % zerofill + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -176,8 +167,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: - self.check(backup_conn.open_session(), None, committed) + session = backup_conn.open_session() finally: + session.checkpoint("force") + self.check(backup_conn.open_session(), None, committed) # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) @@ -204,6 +197,8 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(cur_logs, pr_logs) def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1, 2, 10 and 11. @@ -226,6 +221,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Close and reopen the connection and cursor. if reopen == 'reopen': self.reopen_conn() + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') self.session.begin_transaction( diff --git a/src/third_party/wiredtiger/test/suite/test_txn04.py b/src/third_party/wiredtiger/test/suite/test_txn04.py index ade39272f84..d8f6774ded1 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn04.py +++ b/src/third_party/wiredtiger/test/suite/test_txn04.py @@ -63,24 +63,15 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] scenarios = make_scenarios(types, op1s, txn1s) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir + + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") # Set archive false on the home directory. - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -146,6 +137,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # The runWt command closes our connection and sessions so # we need to reopen them here. self.hot_backup(None, committed) + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') c.set_value(1) # Then do the given modification. @@ -193,6 +185,8 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): self.hot_backup(self.uri, committed) def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() with self.expectedStdoutPattern('recreating metadata'): self.ops() diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 9e84fe7d3fe..7aaff221ba4 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -64,23 +64,15 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] scenarios = make_scenarios(types, op1s, txn1s) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir + + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + # Set archive false on the home directory. + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -139,8 +131,12 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: - self.check(backup_conn.open_session(), None, committed) + session = backup_conn.open_session() finally: + self.check(session, None, committed) + # Force a checkpoint because we don't record the recovery + # checkpoint as available for archiving. + session.checkpoint("force") # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) @@ -163,6 +159,8 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out') def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index 2bff97f6aac..c91dc6a623b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -40,10 +40,10 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 - def setUpConnectionOpen(self, *args): + def conn_config(self): if not wiredtiger.verbose_build(): self.skipTest('requires a verbose build') - return super(test_txn06, self).setUpConnectionOpen(*args) + return '' def test_long_running(self): # Populate a table diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index f9577bad7f2..e26cf5aaaea 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -70,43 +70,20 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ('none', dict(compress='')), ] - scenarios = make_scenarios(types, op1s, txn1s, compress) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir - # Cycle through the different transaction_sync values in a - # deterministic manner. - self.txn_sync = self.sync_list[ - self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ - 'compressor=%s)' % self.compress + \ - self.extensionArg(self.compress) + \ - ',create,error_prefix="%s: ",' % self.shortid() + \ - "statistics=(fast)," + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - try: - conn = self.wiredtiger_open(dir, conn_params) - except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None or name == '': - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + scenarios = make_scenarios(types, op1s, txn1s, compress, + prune=30, prunelong=1000) + + def conn_config(self): + return 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ + 'compressor=%s)' % self.compress + \ + ',create,error_prefix="%s",' % self.shortid() + \ + "statistics=(fast)," + \ + 'transaction_sync="%s",' % \ + self.sync_list[self.scenario_number % len(self.sync_list)] + + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -139,7 +116,7 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s,' % self.logmax + \ 'compressor=%s)' % self.compress + \ - self.extensionArg(self.compress) + self.extensionsConfig() backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: self.check(backup_conn.open_session(), None, committed) @@ -147,6 +124,9 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): backup_conn.close() def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() + # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index f0cdf08df07..04faed9d45a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -41,7 +41,7 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index cfad8270ab1..768d714e248 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -80,19 +80,9 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): op1s, txn1s, op2s, txn2s, op3s, txn3s, op4s, txn4s, prune=20, prunelong=5000) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir - conn_params = \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \ - 'transaction_sync=(enabled=false),' - - # print "Opening conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + def conn_config(self): + return 'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \ + 'transaction_sync=(enabled=false)' # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -141,6 +131,7 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): # Close and reopen the connection and cursor, toggling the log self.log_enabled = not self.log_enabled self.reopen_conn() + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') self.session.begin_transaction( diff --git a/src/third_party/wiredtiger/test/suite/test_txn11.py b/src/third_party/wiredtiger/test/suite/test_txn11.py index 147bf3a76c0..3c02b1e86e3 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn11.py +++ b/src/third_party/wiredtiger/test/suite/test_txn11.py @@ -44,7 +44,7 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=%s,' % self.archive + \ 'enabled,file_max=%s,prealloc=false),' % self.logmax + \ 'transaction_sync=(enabled=false),' diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index ae0250c06e8..2bf49486b3a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -50,7 +50,7 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): ]) # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s)' % self.logmax + \ ',cache_size=8G' diff --git a/src/third_party/wiredtiger/test/suite/test_txn15.py b/src/third_party/wiredtiger/test/suite/test_txn15.py index c061c093b02..a2bfb626338 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn15.py +++ b/src/third_party/wiredtiger/test/suite/test_txn15.py @@ -41,7 +41,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' entries = 100 # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'statistics=(fast),' + \ 'log=(archive=false,enabled,file_max=100K),' + \ 'use_environment=false,' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_util14.py b/src/third_party/wiredtiger/test/suite/test_util14.py new file mode 100644 index 00000000000..e2a9f41f0d4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util14.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util14.py +# Utilities: wt truncate +class test_util14(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util14.a' + nentries = 1000 + + def test_truncate_process(self): + """ + Test truncate in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + cursor[str(i)] = str(i) + cursor.close() + + self.runWt(["truncate", "table:" + self.tablename]) + + """ + Test to confirm table exists and is empty + """ + outfile="outfile.txt" + errfile="errfile.txt" + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'NoMatch: not found\n') + + """ + Tests for error cases + 1. Missing URI + 2. Invalid URI + 3. Valid but incorrect URI + 4. Double URI + """ + self.runWt(["truncate"], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'usage:') + + self.runWt(["truncate", "foobar"], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'No such file or directory') + + self.runWt(["truncate", 'table:xx' + self.tablename], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'No such file or directory') + + self.runWt(["truncate", 'table:' + self.tablename, 'table:' + self.tablename], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'usage:') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util15.py b/src/third_party/wiredtiger/test/suite/test_util15.py new file mode 100644 index 00000000000..33096e71bee --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util15.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util15.py +# Utilities: wt alter +class test_util15(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util15.a' + + def test_alter_process(self): + """ + Test alter in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + + """ + Alter access pattern and confirm + """ + acc_pat_seq="access_pattern_hint=sequential" + self.runWt(["alter", "table:" + self.tablename, acc_pat_seq]) + cursor = self.session.open_cursor("metadata:create", None, None) + cursor.set_key("table:" + self.tablename) + self.assertEqual(cursor.search(),0) + string = cursor.get_value() + cursor.close() + self.assertTrue(acc_pat_seq in string) + + """ + Alter access pattern again and confirm + """ + acc_pat_rand="access_pattern_hint=random" + self.runWt(["alter", "table:" + self.tablename, acc_pat_rand]) + cursor = self.session.open_cursor("metadata:create", None, None) + cursor.set_key("table:" + self.tablename) + self.assertEqual(cursor.search(),0) + string = cursor.get_value() + cursor.close() + self.assertTrue(acc_pat_rand in string) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util16.py b/src/third_party/wiredtiger/test/suite/test_util16.py new file mode 100644 index 00000000000..00e68c1017a --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util16.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util16.py +# Utilities: wt rename +class test_util16(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util16.a' + tablename2 = 'test_util16.b' + nentries = 1000 + + def test_rename_process(self): + """ + Test alter in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + cursor[str(i)] = str(i) + cursor.close() + + self.runWt(["rename", "table:" + self.tablename, "table:" + self.tablename2]) + self.assertTrue(os.path.exists(self.tablename2 + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename2, None, None) + count = 0 + while cursor.next() == 0: + count +=1 + cursor.close() + self.assertEquals(self.nentries, count) + + self.runWt(["rename", "table:" + self.tablename2, "table:" + self.tablename]) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + count = 0 + while cursor.next() == 0: + count +=1 + cursor.close() + self.assertEquals(self.nentries, count) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util17.py b/src/third_party/wiredtiger/test/suite/test_util17.py new file mode 100644 index 00000000000..decc1fabf1d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util17.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util17.py +# Utilities: wt stat +class test_util17(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util17.a' + + def test_stat_process(self): + """ + Test stat in a 'wt' process + This test is just here to confirm that stat produces a correct looking + output, it isn't here to do statistics validation. + """ + params = 'key_format=S,value_format=S' + outfile = "wt-stat.out" + expected_string = "cursor: cursor create calls=" + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["stat"], outfilename=outfile) + self.check_file_contains(outfile, expected_string) + + expected_string = "cache_walk: Entries in the root page=1" + self.runWt(["stat", "table:" + self.tablename ], outfilename=outfile) + self.check_file_contains(outfile, expected_string) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index cb77735510c..946b97d995f 100644 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -41,6 +41,7 @@ class BaseDataSet(object): self.key_format = kwargs.get('key_format', 'S') self.value_format = kwargs.get('value_format', 'S') self.config = kwargs.get('config', '') + self.projection = kwargs.get('projection', '') def create(self): self.testcase.session.create(self.uri, 'key_format=' + self.key_format @@ -49,7 +50,7 @@ class BaseDataSet(object): def fill(self): c = self.testcase.session.open_cursor(self.uri, None) - for i in range(1, self.rows + 1): + for i in xrange(1, self.rows + 1): c[self.key(i)] = self.value(i) c.close() @@ -103,7 +104,8 @@ class BaseDataSet(object): def check(self): self.testcase.pr('check: ' + self.uri) - cursor = self.testcase.session.open_cursor(self.uri, None) + cursor = self.testcase.session.open_cursor( + self.uri + self.projection, None, None) self.check_cursor(cursor) cursor.close() @@ -166,7 +168,7 @@ class SimpleIndexDataSet(SimpleDataSet): # Check values in the index. idxcursor = self.testcase.session.open_cursor(self.indexname) - for i in range(1, self.rows + 1): + for i in xrange(1, self.rows + 1): k = self.key(i) v = self.value(i) ik = (v, k) # The index key is columns=(v,k). @@ -289,6 +291,94 @@ class ComplexLSMDataSet(ComplexDataSet): def is_lsm(cls): return True +class ProjectionDataSet(SimpleDataSet): + """ + ProjectionDataSet creates a table with predefined data identical to + SimpleDataSet (single key and value), but when checking it, uses + a cursor with a projection. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',columns=(k,v0)' + kwargs['projection'] = '(v0,v0,v0)' + super(ProjectionDataSet, self).__init__(testcase, uri, rows, **kwargs) + + # A value suitable for checking the value returned by a cursor. + def comparable_value(self, i): + v0 = self.value(i) + return [v0, v0, v0] + + def check_cursor(self, cursor): + i = 0 + for key, got0, got1, got2 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and got0 == 0: # deleted + continue + self.testcase.assertEqual([got0, got1, got2], + self.comparable_value(i)) + self.testcase.assertEqual(i, self.rows) + +class ProjectionIndexDataSet(BaseDataSet): + """ + ProjectionIndexDataSet creates a table with three values and + an index. Checks are made against a projection of the main table + and a projection of the index. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.origconfig = kwargs.get('config', '') + self.indexname = 'index:' + uri.split(":")[1] + ':index0' + kwargs['config'] = self.origconfig + ',columns=(k,v0,v1,v2)' + kwargs['value_format'] = kwargs.get('value_format', 'SiS') + kwargs['projection'] = '(v1,v2,v0)' + super(ProjectionIndexDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + def value(self, i): + return ('v0:' + str(i), i*i, 'v2:' + str(i)) + + # Suitable for checking the value returned by a cursor using a projection. + def comparable_value(self, i): + return [i*i, 'v2:' + str(i), 'v0:' + str(i)] + + def create(self): + super(ProjectionIndexDataSet, self).create() + self.testcase.session.create(self.indexname, 'columns=(v2,v1),' + + self.origconfig) + + def check_cursor(self, cursor): + i = 0 + for key, got0, got1, got2 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and got0 == 0: # deleted + continue + self.testcase.assertEqual([got0, got1, got2], + self.comparable_value(i)) + self.testcase.assertEqual(i, self.rows) + + def check_index_cursor(self, cursor): + for i in xrange(1, self.rows + 1): + k = self.key(i) + v = self.value(i) + ik = (v[2], v[1]) # The index key is (v2,v2) + expect = [v[1],k,v[2],v[0]] + self.testcase.assertEqual(expect, cursor[ik]) + + def check(self): + BaseDataSet.check(self) + + # Check values in the index. + idxcursor = self.testcase.session.open_cursor( + self.indexname + '(v1,k,v2,v0)') + self.check_index_cursor(idxcursor) + idxcursor.close() + + def index_count(self): + return 1 + + def index_name(self, i): + return self.indexname + # create a key based on a cursor as a shortcut to creating a SimpleDataSet def simple_key(cursor, i): return BaseDataSet.key_by_format(i, cursor.key_format) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 4d6df0bc8bd..e91838544b9 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -37,9 +37,8 @@ except ImportError: import unittest from contextlib import contextmanager -import os, re, shutil, sys, time, traceback -import wtscenario -import wiredtiger +import glob, os, re, shutil, sys, time, traceback +import wiredtiger, wtscenario def shortenWithEllipsis(s, maxlen): if len(s) > maxlen: @@ -152,6 +151,14 @@ class TestSuiteConnection(object): else: return getattr(self._conn, attr) +# Just like a list of strings, but with a convenience function +class ExtensionList(list): + skipIfMissing = False + def extension(self, dirname, name, extarg=None): + if name != None and name != 'none': + ext = '' if extarg == None else '=' + extarg + self.append(dirname + '/' + name + ext) + class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} @@ -160,9 +167,16 @@ class WiredTigerTestCase(unittest.TestCase): # Can be a string or a callable function or lambda expression. conn_config = '' + # conn_extensions can be overridden to add a list of extensions to load. + # Each entry is a string (directory and extension name) and optional config. + # Example: + # conn_extensions = ('extractors/csv_extractor', + # 'test/fail_fs={allow_writes=100}') + conn_extensions = () + @staticmethod def globalSetup(preserveFiles = False, useTimestamp = False, - gdbSub = False, verbose = 1, dirarg = None, + gdbSub = False, verbose = 1, builddir = None, dirarg = None, longtest = False): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg @@ -172,6 +186,7 @@ class WiredTigerTestCase(unittest.TestCase): os.makedirs(d) wtscenario.set_long_run(longtest) WiredTigerTestCase._parentTestdir = d + WiredTigerTestCase._builddir = builddir WiredTigerTestCase._origcwd = os.getcwd() WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered WiredTigerTestCase._gdbSubprocess = gdbSub @@ -224,16 +239,70 @@ class WiredTigerTestCase(unittest.TestCase): return "%s.%s.%s" % (self.__module__, self.className(), self._testMethodName) - # Can be overridden, but first consider setting self.conn_config . + # Return the wiredtiger_open extension argument for + # any needed shared library. + def extensionsConfig(self): + exts = self.conn_extensions + if hasattr(exts, '__call__'): + exts = ExtensionList() + self.conn_extensions(exts) + result = '' + extfiles = {} + skipIfMissing = False + if hasattr(exts, 'skip_if_missing'): + skipIfMissing = exts.skip_if_missing + for ext in exts: + extconf = '' + if '=' in ext: + splits = ext.split('=', 1) + ext = splits[0] + extconf = '=' + splits[1] + splits = ext.split('/') + if len(splits) != 2: + raise Exception(self.shortid() + + ": " + ext + + ": extension is not named /") + libname = splits[1] + dirname = splits[0] + pat = os.path.join(WiredTigerTestCase._builddir, 'ext', + dirname, libname, '.libs', 'libwiredtiger_*.so') + filenames = glob.glob(pat) + if len(filenames) == 0: + if skipIfMissing: + self.skipTest('extension "' + ext + '" not built') + continue + else: + raise Exception(self.shortid() + + ": " + ext + + ": no extensions library found matching: " + pat) + elif len(filenames) > 1: + raise Exception(self.shortid() + + ": " + ext + + ": multiple extensions libraries found matching: " + pat) + complete = '"' + filenames[0] + '"' + extconf + if ext in extfiles: + if extfiles[ext] != complete: + raise Exception(self.shortid() + + ": non-matching extension arguments in " + + str(exts)) + else: + extfiles[ext] = complete + if len(extfiles) != 0: + result = ',extensions=[' + ','.join(extfiles.values()) + ']' + return result + + # Can be overridden, but first consider setting self.conn_config + # or self.conn_extensions def setUpConnectionOpen(self, home): self.home = home config = self.conn_config if hasattr(config, '__call__'): - config = config(home) + config = self.conn_config() + config += self.extensionsConfig() # In case the open starts additional threads, flush first to # avoid confusion. sys.stdout.flush() - conn_param = 'create,error_prefix="%s: ",%s' % (self.shortid(), config) + conn_param = 'create,error_prefix="%s",%s' % (self.shortid(), config) try: conn = self.wiredtiger_open(home, conn_param) except wiredtiger.WiredTigerError as e: @@ -287,6 +356,7 @@ class WiredTigerTestCase(unittest.TestCase): self.testsubdir = self.className() + '.' + str(self.__class__.wt_ntests) self.testdir = os.path.join(WiredTigerTestCase._parentTestdir, self.testsubdir) self.__class__.wt_ntests += 1 + self.starttime = time.time() if WiredTigerTestCase._verbose > 2: self.prhead('started in ' + self.testdir, True) # tearDown needs connections list, set it here in case the open fails. @@ -355,6 +425,9 @@ class WiredTigerTestCase(unittest.TestCase): else: self.pr('preserving directory ' + self.testdir) + elapsed = time.time() - self.starttime + if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: + print "%s: %.2f seconds" % (str(self), elapsed) if not passed and not skipped: print "ERROR in " + str(self) self.pr('FAIL') diff --git a/src/third_party/wiredtiger/test/syscall/Makefile.am b/src/third_party/wiredtiger/test/syscall/Makefile.am new file mode 100644 index 00000000000..1d55b06c388 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/Makefile.am @@ -0,0 +1,7 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +LDADD = $(top_builddir)/libwiredtiger.la +AM_LDFLAGS = -static +noinst_PROGRAMS = + +test_wt2336_base_SOURCES = wt2336_base/main.c +noinst_PROGRAMS += test_wt2336_base diff --git a/src/third_party/wiredtiger/test/syscall/syscall.py b/src/third_party/wiredtiger/test/syscall/syscall.py new file mode 100644 index 00000000000..59c2f347146 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/syscall.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# syscall.py +# Command line syscall test runner +# +# For each .run file, run the corresponding program and collect strace +# output, comparing it to the contents of the .run file. +# +# run files are first preprocessed, which means the use of #ifdefs, #defines +# and #includes are allowed, as well as /**/ and // comments. +# Expressions are evaluated using the Python parser, so that: +# hex and octal numbers are accepted, constant values can be or-ed. +# +from __future__ import print_function +import argparse, distutils.spawn, fnmatch, os, platform, re, shutil, \ + subprocess, sys + +# A class that represents a context in which predefined constants can be +# set, and new variables can be assigned. +class VariableContext(object): + def __getitem__(self, key): + if key not in dir(self) or key.startswith('__'): + raise KeyError(key) + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + +################################################################ +# Changable parameters +# We expect these values to evolve as tests are added or modified. + +# Generally, system calls must be wrapped in an ASSERT_XX() "macro". +# Exceptions are calls in this list that return 0 on success, or +# those that are hardcoded in Runner.call_compare() +calls_returning_zero = [ 'close', 'ftruncate', 'fdatasync', 'rename' ] + +# Encapsulate all the defines we can use in our scripts. +# When this program is run, we'll find out their actual values on +# the host system. +defines_used = [ + 'HAVE_FTRUNCATE', 'O_ACCMODE', 'O_APPEND', 'O_ASYNC', + 'O_CLOEXEC', 'O_CREAT', 'O_EXCL', 'O_EXLOCK', 'O_NOATIME', + 'O_NOFOLLOW', 'O_NONBLOCK', 'O_RDONLY', 'O_RDWR', 'O_SHLOCK', + 'O_TRUNC', 'O_WRONLY' ] + +################################################################ + +# Patterns that are used to match the .run file and/or the output. +ident = r'([a-zA-Z_][a-zA-Z0-9_]*)' +outputpat = re.compile(r'OUTPUT\("([^"]*)"\)') +argpat = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''') +discardpat = re.compile(r';') + +# e.g. fd = open("blah", 0, 0); +assignpat = re.compile(ident + r'\s*=\s*' + ident + r'(\([^;]*\));') + +# e.g. ASSERT_EQ(close(fd), 0); +assertpat = re.compile(r'ASSERT_([ENLG][QET])\s*\(\s*' + ident + r'\s*(\(.*\))\s*,\s*([a-zA-Z0-9_]+)\);') + +# e.g. close(fd); must return 0 +callpat = re.compile(ident + r'(\(.*\));') + +# e.g. open("blah", 0x0, 0x0) = 6 0 +# We capture the errno (e.g. "0" or "Err#60"), but don't do anything with it. +# We don't currently test anything that is errno dependent. +dtruss_pat = re.compile(ident + r'(\(.*\))\s*=\s*(-*[0-9xA-F]+)\s+([-A-Za-z#0-9]*)') +# At the top of the dtruss output is a fixed string. +dtruss_init_pat = re.compile(r'\s*SYSCALL\(args\)\s*=\s*return\s*') + +strace_pat = re.compile(ident + r'(\(.*\))\s*=\s(-*[0-9]+)()') + +tracepat = re.compile(r'TRACE\("([^"]*)"\)') +runpat = re.compile(r'RUN\(([^\)]*)\)') +systempat = re.compile(r'SYSTEM\("([^"]*)"\)') +# If tracepat matches, set map['trace_syscalls'] to the 0'th group, etc. +headpatterns = [ [ tracepat, 'trace_syscalls', 0], + [ systempat, 'required_system', 0], + [ runpat, 'run_args', 0] ] + +pwrite_in = r'pwrite64' +pwrite_out = r'pwrite' + +# To create breakpoints while debugging this script +def bp(): + import pdb + pdb.set_trace() + +def msg(s): + print("syscall.py: " + s, file=sys.stderr) + +def die(s): + msg(s) + sys.exit(1) + +# If wttop appears as a prefix of pathname, strip it off. +def simplify_path(wttop, pathname): + wttop = os.path.join(wttop, "") + if pathname.startswith(wttop): + pathname = pathname[len(wttop):] + return pathname + +def printfile(pathname, abbrev): + print("================================================================") + print(abbrev + " (" + pathname + "):") + with open(pathname, 'r') as f: + shutil.copyfileobj(f, sys.stdout) + print("================================================================") + +# A line from a file: a modified string with the file name and line number +# associated with it. +class FileLine(str): + filename = None + linenum = 0 + def __new__(cls, value, *args, **kwargs): + return super(FileLine, cls).__new__(cls, value) + + def prefix(self): + return self.filename + ':' + str(self.linenum) + ': ' + + def range_prefix(self, otherline): + if self == otherline: + othernum = '' + elif otherline == None: + othernum = '-EOF' + else: + othernum = '-' + str(otherline.linenum) + return self.filename + ':' + str(self.linenum) + othernum + ': ' + +# Manage reading from a file, tracking line numbers. +class Reader(object): + # 'raw' means we don't ignore any lines + # 'is_cpp' means input lines beginning with '#' indicate file/linenumber + def __init__(self, wttop, filename, f, raw = True, is_cpp = False): + self.wttop = wttop + self.orig_filename = filename + self.filename = filename + self.f = f + self.linenum = 1 + self.raw = raw + self.is_cpp = is_cpp + if not self.f: + die(self.filename + ': cannot open') + + def __enter__(self): + if not self.f: + return False + return self + + def __exit__(self, typ, value, traceback): + if self.f: + self.f.close() + self.f = None + + # Return True if the line is to be ignored. + def ignore(self, line): + if self.raw: + return False + return line == '' + + # strip a line of comments + def strip_line(self, line): + if not line: + return None + line = line.strip() + if self.is_cpp and line.startswith('#'): + parts = line.split() + if len(parts) < 3 or not parts[1].isdigit(): + msg('bad cpp input: ' + line) + line = '' + self.linenum = int(parts[1]) - 1 + self.filename = parts[2].strip('"') + if self.filename == '': + self.filename = self.orig_filename + if '//' in line: + if line.startswith('//'): + line = '' + else: + # This isn't exactly right, it would see "; //" + # within a string or comment. + m = re.match(r'^(.*;|.*\.\.\.)\s*//', line) + if m: + line = m.groups()[0].strip() + return line + + def readline(self): + line = self.strip_line(self.f.readline()) + while line != None and self.ignore(line): + self.linenum += 1 + line = self.strip_line(self.f.readline()) + if line: + line = FileLine(line) + line.filename = self.filename + line.linenum = self.linenum + self.linenum += 1 + else: + line = '' # make this somewhat compatible with file.readline + return line + + def close(self): + self.f.close() + +# Read from a regular file. +class FileReader(Reader): + def __init__(self, wttop, filename, raw = True): + return super(FileReader, self).__init__(wttop, filename, + open(filename), raw, False) + +# Read from the C preprocessor run on a file. +class PreprocessedReader(Reader): + def __init__(self, wttop, filename, predefines, raw = True): + sourcedir = os.path.dirname(filename) + cmd = ['cc', '-E', '-I' + sourcedir] + for name in dir(predefines): + if not name.startswith('__'): + cmd.append('-D' + name + '=' + str(predefines[name])) + cmd.append('-') + proc = subprocess.Popen(cmd, stdin=open(filename), + stdout=subprocess.PIPE) + super(PreprocessedReader, self).__init__(wttop, filename, + proc.stdout, raw, True) + +# Track options discovered in the 'head' section of the .run file. +class HeadOpts: + def __init__(self): + self.run_args = None + self.required_system = None + self.trace_syscalls = None + +# Manage a run of the target program characterized by a .run file, +# comparing output from the run and reporting differences. +class Runner: + def __init__(self, wttopdir, runfilename, exedir, testexe, + strace, args, variables, defines): + self.variables = variables + self.defines = defines + self.wttopdir = wttopdir + self.runfilename = runfilename + self.testexe = testexe + self.exedir = exedir + self.strace = strace + self.args = args + self.headopts = HeadOpts() + self.dircreated = False + self.strip_syscalls = None + outfilename = args.outfilename + errfilename = args.errfilename + if outfilename == None: + self.outfilename = os.path.join(exedir, 'stdout.txt') + else: + self.outfilename = outfilename + if errfilename == None: + self.errfilename = os.path.join(exedir, 'stderr.txt') + else: + self.errfilename = errfilename + + self.runfile = PreprocessedReader(self.wttopdir, runfilename, + self.defines, False) + + def init(self, systemtype): + # Read up until 'RUN()', setting attributes of self.headopts + runline = '?' + m = None + while runline: + runline = self.runfile.readline() + m = None + for pat,attr,group in headpatterns: + m = re.match(pat, runline) + if m: + setattr(self.headopts, attr, m.groups()[group]) + break + if not m: + self.fail(runline, "unknown header option: " + runline) + return [ False, False ] + if self.headopts.run_args != None: # found RUN()? + break + if not self.headopts.trace_syscalls: + msg("'" + self.runfile.filename + "': needs TRACE(...)") + return [ False, False ] + runargs = self.headopts.run_args.strip() + if len(runargs) > 0: + if len(runargs) < 2 or runargs[0] != '"' or runargs[-1] != '"': + msg("'" + self.runfile.filename + + "': Missing double quotes in RUN arguments") + return [ False, False ] + runargs = runargs[1:-1] + self.runargs = runargs.split() + #print("SYSCALLS: " + self.headopts.trace_syscalls + if self.headopts.required_system != None and \ + self.headopts.required_system != systemtype: + msg("skipping '" + self.runfile.filename + "': for '" + + self.headopts.required_system + "', this system is '" + + systemtype + "'") + return [ False, True ] + return [ True, False ] + + def close(self, forcePreserve): + self.runfile.close() + if self.exedir and self.dircreated and \ + not self.args.preserve and not forcePreserve: + os.chdir('..') + shutil.rmtree(self.exedir) + + def fail(self, line, s): + # make it work if line is None or is a plain string. + try: + prefix = simplify_path(self.wttopdir, line.prefix()) + except: + prefix = 'syscall.py: ' + print(prefix + s, file=sys.stderr) + + def failrange(self, line, lineto, s): + # make it work if line is None or is a plain string. + try: + prefix = simplify_path(self.wttopdir, line.range_prefix(lineto)) + except: + prefix = 'syscall.py: ' + print(prefix + s, file=sys.stderr) + + def str_match(self, s1, s2): + fuzzyRight = False + if len(s1) < 2 or len(s2) < 2: + return False + if s1[-3:] == '...': + fuzzyRight = True + s1 = s1[:-3] + if s2[-3:] == '...': + s2 = s2[:-3] + if s1[0] != '"' or s1[-1] != '"' or s2[0] != '"' or s2[-1] != '"': + return False + s1 = s1[1:-1] + s2 = s2[1:-1] + # We allow a trailing \0 + if s1[-2:] == '\\0': + s1 = s1[:-2] + if s2[-2:] == '\\0': + s2 = s2[:-2] + if fuzzyRight: + return s2.startswith(s2) + else: + return s1 == s2 + + def expr_eval(self, s): + return eval(s, {}, self.variables) + + def arg_match(self, a1, a2): + a1 = a1.strip() + a2 = a2.strip() + if a1 == a2: + return True + if len(a1) == 0 or len(a2) == 0: + return False + if a1[0] == '"': + return self.str_match(a1, a2) + #print(' arg_match: <' + a1 + '> <' + a2 + '>') + try: + a1value = self.expr_eval(a1) + except Exception: + self.fail(a1, 'unknown expression: ' + a1) + return False + try: + a2value = self.expr_eval(a2) + except Exception: + self.fail(a2, 'unknown expression: ' + a2) + return False + return a1value == a2value or int(a1value) == int(a2value) + + def split_args(self, s): + if s[0] == '(': + s = s[1:] + if s[-1] == ')': + s = s[:-1] + return argpat.split(s)[1::2] + + def args_match(self, args1, args2): + #print('args_match: ' + str(s1) + ', ' + str(s2)) + pos = 0 + for a1 in args1: + a1 = a1.strip() + if a1 == '...': # match anything? + return True + if pos >= len(args2): + return False + if not self.arg_match(a1, args2[pos]): + return False + pos += 1 + if pos < len(args2): + return False + return True + + # func(args); is shorthand for for ASSERT_EQ(func(args), xxx); + # where xxx may be 0 or may be derived from one of the args. + def call_compare(self, callname, result, eargs, errline): + if callname in calls_returning_zero: + return self.compare("EQ", result, "0", errline) + elif callname == 'pwrite' or callname == 'pwrite64': + return self.compare("EQ", + re.sub(pwrite_in, pwrite_out, result), + re.sub(pwrite_in, pwrite_out, eargs[2]), + errline) + else: + self.fail(errline, 'call ' + callname + + ': not known, use ASSERT_EQ()') + + def compare(self, compareop, left, right, errline): + l = self.expr_eval(left) + r = self.expr_eval(right) + if (compareop == "EQ" and l == r) or \ + (compareop == "NE" and l != r) or \ + (compareop == "LT" and l < r) or \ + (compareop == "LE" and l <= r) or \ + (compareop == "GT" and l > r) or \ + (compareop == "GE" and l >= r): + return True + else: + self.fail(errline, + 'call returned value: ' + left + ', comparison: (' + + left + ' ' + compareop + ' ' + right + + ') at line: ' + errline) + return False + + def match_report(self, runline, errline, verbose, skiplines, result, desc): + if result: + if verbose: + print('MATCH:') + print(' ' + runline.prefix() + runline) + print(' ' + errline.prefix() + errline) + else: + if verbose: + if not skiplines: + msg('Expecting ' + desc) + print(' ' + runline.prefix() + runline + + ' does not match:') + print(' ' + errline.prefix() + errline) + else: + print(' (... match) ' + errline.prefix() + errline) + return result + + def match(self, runline, errline, verbose, skiplines): + m = re.match(outputpat, runline) + if m: + outwant = m.groups()[0] + return self.match_report(runline, errline, verbose, skiplines, + errline == outwant, 'output line') + if self.args.systype == 'Linux': + em = re.match(strace_pat, errline) + elif self.args.systype == 'Darwin': + em = re.match(dtruss_pat, errline) + if not em: + self.fail(errline, 'Unknown strace/dtruss output: ' + errline) + return False + gotcall = re.sub(pwrite_in, pwrite_out, em.groups()[0]) + # filtering syscalls here if needed. If it's not a match, + # mark the errline so it is retried. + if self.strip_syscalls != None and gotcall not in self.strip_syscalls: + errline.skip = True + return False + m = re.match(assignpat, runline) + if m: + if m.groups()[1] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall to match assignment') + + rargs = self.split_args(m.groups()[2]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if result: + self.variables[m.groups()[0]] = em.groups()[2] + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall to match assignment') + + # pattern groups using example ASSERT_EQ(close(fd), 0); + # 0 : comparison op ("EQ") + # 1 : function call name "close" + # 2 : function call args "(fd)" + # 3 : comparitor "0" + m = re.match(assertpat, runline) + if m: + if m.groups()[1] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall to match ASSERT') + + rargs = self.split_args(m.groups()[2]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if not result: + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall to match ASSERT') + result = self.compare(m.groups()[0], em.groups()[2], + m.groups()[3], errline) + return self.match_report(runline, errline, verbose, skiplines, + result, 'ASSERT') + + # A call without an enclosing ASSERT is reduced to an ASSERT, + # depending on the particular system call. + m = re.match(callpat, runline) + if m: + if m.groups()[0] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall') + + rargs = self.split_args(m.groups()[1]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if not result: + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall') + result = self.call_compare(m.groups()[0], em.groups()[2], + eargs, errline) + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall') + + self.fail(runline, 'unrecognized pattern in runfile:' + runline) + return False + + def match_lines(self): + outfile = FileReader(self.wttopdir, self.outfilename, True) + errfile = FileReader(self.wttopdir, self.errfilename, True) + + if outfile.readline(): + self.fail(None, 'output file has content, expected to be empty') + return False + + with outfile, errfile: + runlines = self.order_runfile(self.runfile) + errline = errfile.readline() + errline = re.sub(pwrite_in, pwrite_out, errline) + if re.match(dtruss_init_pat, errline): + errline = errfile.readline() + skiplines = False + for runline in runlines: + runline = re.sub(pwrite_in, pwrite_out, runline) + if runline == '...': + skiplines = True + if self.args.verbose: + print('Fuzzy matching:') + print(' ' + runline.prefix() + runline) + continue + first_errline = errline + while errline and not self.match(runline, errline, + self.args.verbose, skiplines): + if skiplines or hasattr(errline, 'skip'): + errline = errfile.readline() + else: + self.fail(runline, "expecting " + runline) + self.failrange(first_errline, errline, "does not match") + return False + if not errline: + self.fail(runline, "failed to match line: " + runline) + self.failrange(first_errline, errline, "does not match") + return False + errline = errfile.readline() + if re.match(dtruss_init_pat, errline): + errline = errfile.readline() + skiplines = False + if errline and not skiplines: + self.fail(errline, "extra lines seen starting at " + errline) + return False + return True + + def order_runfile(self, f): + # In OS X, dtruss is implemented using dtrace's apparently buffered + # printf writes to stdout, but that is all redirected to stderr. + # Because of that, the test program's writes to stderr do not + # interleave with dtruss output as it does with Linux's strace + # (which writes directly to stderr). On OS X, we get the program's + # output first, we compensate for this by moving all the + # OUTPUT statements in the runfile to match first. This simple + # approach will break if there is more data generated by OUTPUT + # statements than a stdio buffer's size. + matchout = (self.args.systype == 'Darwin') + out = [] + nonout = [] + s = f.readline() + while s: + if matchout and re.match(outputpat, s): + out.append(s) + elif not re.match(discardpat, s): + nonout.append(s) + s = f.readline() + out.extend(nonout) + return out + + def run(self): + if not self.exedir: + self.fail(None, "Execution directory not set") + return False + if not os.path.isfile(self.testexe): + msg("'" + self.testexe + "': no such file") + return False + + shutil.rmtree(self.exedir, ignore_errors=True) + os.mkdir(self.exedir) + self.dircreated = True + os.chdir(self.exedir) + + callargs = list(self.strace) + trace_syscalls = self.headopts.trace_syscalls + if self.args.systype == 'Linux': + callargs.extend(['-e', 'trace=' + trace_syscalls ]) + elif self.args.systype == 'Darwin': + # dtrace has no option to limit the syscalls to be traced, + # so we'll filter the output. + self.strip_syscalls = re.sub(pwrite_in, pwrite_out, + self.headopts.trace_syscalls).split(',') + callargs.append(self.testexe) + callargs.extend(self.runargs) + + outfile = open(self.outfilename, 'w') + errfile = open(self.errfilename, 'w') + if self.args.verbose: + print('RUNNING: ' + str(callargs)) + subret = subprocess.call(callargs, stdout=outfile, stderr=errfile) + outfile.close() + errfile.close() + if subret != 0: + msg("'" + self.testexe + "': exit value " + str(subret)) + printfile(self.outfilename, "output") + printfile(self.errfilename, "error") + return False + return True + +# Run the syscall program. +class SyscallCommand: + def __init__(self, disttop, builddir): + self.disttop = disttop + self.builddir = builddir + + def parse_args(self, argv): + srcdir = os.path.join(self.disttop, 'test', 'syscall') + self.exetopdir = os.path.join(self.builddir, 'test', 'syscall') + self.incdir1 = os.path.join(self.disttop, 'src', 'include') + self.incdir2 = self.builddir + + ap = argparse.ArgumentParser('Syscall test runner') + ap.add_argument('--systype', + help='override system type (Linux/Windows/Darwin)') + ap.add_argument('--errfile', dest='errfilename', + help='do not run the program, use this file as stderr') + ap.add_argument('--outfile', dest='outfilename', + help='do not run the program, use this file as stdout') + ap.add_argument('--preserve', action="store_true", + help='keep the WT_TEST.* directories') + ap.add_argument('--verbose', action="store_true", + help='add some verbose information') + ap.add_argument('tests', nargs='*', + help='the tests to run (defaults to all)') + args = ap.parse_args() + + if not args.systype: + args.systype = platform.system() # Linux, Windows, Darwin + + self.dorun = True + if args.errfilename or args.outfilename: + if len(args.tests) != 1: + msg("one test is required when --errfile or --outfile" + + " is specified") + return False + if not args.outfilename: + args.outfilename = os.devnull + if not args.errfilename: + args.errfilename = os.devnull + self.dorun = False + + # for now, we permit Linux and Darwin + straceexe = None + if args.systype == 'Linux': + strace = [ 'strace' ] + straceexe = 'strace' + elif args.systype == 'Darwin': + strace = [ 'sudo', 'dtruss' ] + straceexe = 'dtruss' + else: + msg("systype '" + args.systype + "' unsupported") + return False + if not distutils.spawn.find_executable(straceexe): + msg("strace: does not exist") + return False + self.args = args + self.strace = strace + return True + + def runone(self, runfilename, exedir, testexe, args): + result = True + runner = Runner(self.disttop, runfilename, exedir, testexe, + self.strace, args, self.variables, self.defines) + okay, skip = runner.init(args.systype) + if not okay: + if not skip: + result = False + else: + if testexe: + print('running ' + testexe) + if not runner.run(): + result = False + if result: + print('comparing:') + print(' ' + simplify_path(self.disttop, runfilename)) + print(' ' + simplify_path(self.disttop, runner.errfilename)) + result = runner.match_lines() + if not result and args.verbose: + printfile(runfilename, "runfile") + printfile(runner.errfilename, "trace output") + runner.close(not result) + if not result: + print('************************ FAILED ************************') + print(' see results in ' + exedir) + print('') + return result + + # Create a C program to get values for all defines we need. + # The output of the program is Python code that we'll execute + # directly to set the values. + def build_system_defines(self): + # variables is a symbol table that is used to + # evaluate expressions both in the .run file and + # in the output file. This is needed for strace, + # which shows system call flags in symbolic form. + self.variables = VariableContext() + # defines is a symbol table that is used to + # create preprocessor defines, effectively evaluating + # all flag defines in the .run file. + self.defines = VariableContext() + program = \ + '#include \n' + \ + '#include \n' + \ + '#include \n' + \ + 'int main() {\n' + for define in defines_used: + program += '#ifdef ' + define + '\n' + # output is Python that sets attributes of 'o'. + program += ' printf("o.' + define + '=%d\\n", ' + \ + define + ');\n' + program += '#endif\n' + program += \ + ' return(0);\n' + \ + '}\n' + probe_c = os.path.join(self.exetopdir, "syscall_probe.c") + probe_exe = os.path.join(self.exetopdir, "syscall_probe") + with open(probe_c, "w") as f: + f.write(program) + ccargs = ['cc', '-o', probe_exe] + ccargs.append('-I' + self.incdir1) + ccargs.append('-I' + self.incdir2) + if self.args.systype == 'Linux': + ccargs.append('-D_GNU_SOURCE') + ccargs.append(probe_c) + subret = subprocess.call(ccargs) + if subret != 0: + msg("probe compilation returned " + str(subret)) + return False + proc = subprocess.Popen([probe_exe], stdout=subprocess.PIPE) + out, err = proc.communicate() + subret = proc.returncode + if subret != 0 or err: + msg("probe run returned " + str(subret) + ", error=" + str(err)) + return False + if self.args.verbose: + print('probe output:\n' + out) + o = self.defines # The 'o' object will be modified. + exec(out) # Run the produced Python. + o = self.variables # Set these in variables too, so strace + exec(out) # symbolic output is evaluated. + if not self.args.preserve: + os.remove(probe_c) + os.remove(probe_exe) + return True + + def execute(self): + args = self.args + result = True + if not self.build_system_defines(): + die('cannot build system defines') + if not self.dorun: + for testname in args.tests: + result = self.runone(testname, None, None, args) and result + else: + if len(args.tests) > 0: + tests = [] + for arg in args.tests: + abspath = os.path.abspath(arg) + tests.append([os.path.dirname(abspath), [], + [os.path.basename(abspath)]]) + else: + tests = os.walk(syscalldir) + os.chdir(self.exetopdir) + for path, subdirs, files in tests: + testnum = -1 if len(files) <= 1 else 0 + for name in files: + if fnmatch.fnmatch(name, '*.run'): + testname = os.path.basename(os.path.normpath(path)) + runfilename = os.path.join(path, name) + testexe = os.path.join(self.exetopdir, + 'test_' + testname) + exedir = os.path.join(self.exetopdir, + 'WT_TEST.' + testname) + # If there are multiple tests in this directory, + # give each one its own execution dir. + if testnum >= 0: + exedir += '.' + str(testnum) + testnum += 1 + result = self.runone(runfilename, exedir, + testexe, args) and result + return result + +# Set paths, determining the top of the build. +syscalldir = sys.path[0] +wt_disttop = os.path.dirname(os.path.dirname(syscalldir)) + +# Note: this code is borrowed from test/suite/run.py +# Check for a local build that contains the wt utility. First check in +# current working directory, then in build_posix and finally in the disttop +# directory. This isn't ideal - if a user has multiple builds in a tree we +# could pick the wrong one. +if os.path.isfile(os.path.join(os.getcwd(), 'wt')): + wt_builddir = os.getcwd() +elif os.path.isfile(os.path.join(wt_disttop, 'wt')): + wt_builddir = wt_disttop +elif os.path.isfile(os.path.join(wt_disttop, 'build_posix', 'wt')): + wt_builddir = os.path.join(wt_disttop, 'build_posix') +elif os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): + wt_builddir = wt_disttop +else: + die('unable to find useable WiredTiger build') + +cmd = SyscallCommand(wt_disttop, wt_builddir) +if not cmd.parse_args(sys.argv): + die('bad usage') +if not cmd.execute(): + sys.exit(1) +sys.exit(0) diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run new file mode 100644 index 00000000000..7d2c42ce64e --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run @@ -0,0 +1,180 @@ +// Public Domain 2014-2016 MongoDB, Inc. +// Public Domain 2008-2014 WiredTiger, Inc. +// +// This is free and unencumbered software released into the public domain. +// +// Anyone is free to copy, modify, publish, use, compile, sell, or +// distribute this software, either in source code form or as a compiled +// binary, for any purpose, commercial or non-commercial, and by any +// means. +// +// In jurisdictions that recognize copyright laws, the author or authors +// of this software dedicate any and all copyright interest in the +// software to the public domain. We make this dedication for the benefit +// of the public at large and to the detriment of our heirs and +// successors. We intend this dedication to be an overt act of +// relinquishment in perpetuity of all present and future rights to this +// software under copyright law. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// base.run +// Command line syscall test runner +// +#ifdef __linux__ +SYSTEM("Linux"); +#define OPEN_EXISTING(name, flags) open(name, flags) +#else /* __linux__ */ +SYSTEM("Darwin"); +#define O_NOATIME 0 +#define OPEN_EXISTING(name, flags) open(name, flags, 0) +#endif /* __linux__ */ + +#ifdef HAVE_FTRUNCATE +#ifdef __linux__ +#define FTRUNCATE(fd, len) ftruncate(fd, len) +#else +#define FTRUNCATE(fd, len) ftruncate(fd, len, len) +#endif +#else +#define FTRUNCATE(fd, len) /* do nothing */ +#endif + +TRACE("close,fdatasync,fsync,ftruncate,open,pwrite64,rename"); +RUN(""); +... +OUTPUT("--------------wiredtiger_open"); +// lock == 3 +lock = open("./WiredTiger.lock", O_RDWR|O_CREAT|O_CLOEXEC, 0666); +pwrite64(lock, "WiredTiger lock file\n", 0x15, 0x0); +fd = open("./WiredTiger", O_RDWR|O_CREAT|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); + +... // On Linux, there are calls to open and read "/proc/meminfo" here. + +fd = open("./WiredTiger.basecfg.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "# Do not modify this file."..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.basecfg.set", "./WiredTiger.basecfg"); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif + +fd = open("./WiredTiger.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ + +pwrite64(fd, ""..., 0x1000, 0x0); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); + +wt = OPEN_EXISTING("./WiredTiger.wt\0", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(wt, 0x1000); + +fd = open("./WiredTiger.turtle.set\0", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); + +... // There is a second open of turtle here, is it important? + +fd = open("./WiredTigerLAS.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ + +pwrite64(fd, ""..., 0x1000, 0x0); + +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ + +close(fd); +fd = OPEN_EXISTING("./WiredTigerLAS.wt", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(fd, 0x1000); +fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(fd); +pwrite64(wt, ""..., 0x1000, 0x1000); +pwrite64(wt, ""..., 0x1000, 0x2000); +pwrite64(wt, ""..., 0x1000, 0x3000); +#ifdef __linux__ +fdatasync(wt); +#endif /* __linux__ */ +fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); + +close(fd); +fd = open("./WiredTiger.turtle.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +fdatasync(wt); +#endif /* __linux__ */ + +OUTPUT("--------------open_session"); +OUTPUT("--------------create"); + +hello = open("./hello.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ +pwrite64(hello, "A\330\001"..., 0x1000, 0x0); +#ifdef __linux__ +fdatasync(hello); +#endif /* __linux__ */ +close(hello); +hello = OPEN_EXISTING("./hello.wt", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(hello, 0x1000); +turtle = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(turtle); +... // more changes to turtle +OUTPUT("--------------rename"); +#ifdef __linux__ +FTRUNCATE(hello, 0x1000); +#endif /* __linux__ */ +close(hello); +rename("./hello.wt", "./world.wt"); +// Changes to turtle, more writes to wt. +turtle = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(turtle); +... +OUTPUT("--------------drop"); +... +OUTPUT("--------------WT_CONNECTION::close"); +... diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c new file mode 100644 index 00000000000..22420371dd0 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c @@ -0,0 +1,69 @@ +#include + +#include // TODO +#include // TODO +#include + +static void fail(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + +static void +fail(int ret) { + fprintf(stderr, + "%s: %d (%s)\n", + "wt2336_fileop_basic", ret, wiredtiger_strerror(ret)); + exit(ret); +} + +#define SEPARATOR "--------------" + +int +main(int argc, char *argv[]) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + (void)argc; + (void)argv; + fprintf(stderr, SEPARATOR "wiredtiger_open\n"); + if ((ret = wiredtiger_open(".", NULL, "create", &conn)) != 0) + fail(ret); + + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "open_session\n"); + fflush(stderr); + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fail(ret); + + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "create\n"); + fflush(stderr); + + if ((ret = session->create( + session, "table:hello", "key_format=S,value_format=S")) != 0) + fail(ret); + + usleep(100); + fprintf(stderr, SEPARATOR "rename\n"); + + if ((ret = session->rename( + session, "table:hello", "table:world", NULL)) != 0) + fail(ret); + + fflush(stdout); + fprintf(stderr, SEPARATOR "drop\n"); + fflush(stdout); + + if ((ret = session->drop(session, "table:world", NULL)) != 0) + fail(ret); + + fprintf(stderr, SEPARATOR "WT_CONNECTION::close\n"); + + if ((ret = conn->close(conn, NULL)) != 0) + fail(ret); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/thread/file.c b/src/third_party/wiredtiger/test/thread/file.c index 81ec6ad44f8..7a7d16c4cd6 100644 --- a/src/third_party/wiredtiger/test/thread/file.c +++ b/src/third_party/wiredtiger/test/thread/file.c @@ -33,20 +33,18 @@ file_create(const char *name) { WT_SESSION *session; int ret; - char *p, *end, config[128]; + char config[128]; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s," "internal_page_max=%d," - "leaf_page_max=%d,", - ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024); - if (ftype == FIX) - (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + "leaf_page_max=%d," + "%s", + ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024, + ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) @@ -62,9 +60,10 @@ load(const char *name) WT_CURSOR *cursor; WT_ITEM *key, _key, *value, _value; WT_SESSION *session; - char keybuf[64], valuebuf[64]; - u_int keyno; + uint64_t keyno; + size_t len; int ret; + char keybuf[64], valuebuf[64]; file_create(name); @@ -79,18 +78,22 @@ load(const char *name) value = &_value; for (keyno = 1; keyno <= nkeys; ++keyno) { if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), + &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); - value->data = valuebuf; + cursor->set_key(cursor, keyno); if (ftype == FIX) cursor->set_value(cursor, 0x01); else { - value->size = (uint32_t) - snprintf(valuebuf, sizeof(valuebuf), "%37u", keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "%37" PRIu64, keyno)); + value->data = valuebuf; + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c index c6107a06c49..e8a2650ca51 100644 --- a/src/third_party/wiredtiger/test/thread/rw.c +++ b/src/third_party/wiredtiger/test/thread/rw.c @@ -66,7 +66,8 @@ rw_start(u_int readers, u_int writers) for (i = 0; i < writers; ++i) { if (i == 0 || multiple_files) { run_info[i].name = dmalloc(64); - snprintf(run_info[i].name, 64, FNAME, i); + testutil_check(__wt_snprintf( + run_info[i].name, 64, FNAME, i)); /* Vary by orders of magnitude */ if (vary_nops) @@ -88,8 +89,8 @@ rw_start(u_int readers, u_int writers) run_info[offset].name = dmalloc(64); /* Have readers read from tables with writes. */ name_index = i % writers; - snprintf( - run_info[offset].name, 64, FNAME, name_index); + testutil_check(__wt_snprintf( + run_info[offset].name, 64, FNAME, name_index)); /* Vary by orders of magnitude */ if (vary_nops) @@ -158,7 +159,8 @@ static inline void reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *key, _key; - u_int keyno; + size_t len; + uint64_t keyno; int ret; char keybuf[64]; @@ -166,17 +168,18 @@ reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) keyno = __wt_random(&s->rnd) % nkeys + 1; if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); + cursor->set_key(cursor, keyno); if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) testutil_die(ret, "cursor.search"); if (log_print) testutil_check(session->log_printf(session, - "Reader Thread %p key %017u", pthread_self(), keyno)); + "Reader Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* @@ -195,7 +198,7 @@ reader(void *arg) id = (int)(uintptr_t)arg; s = &run_info[id]; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf(" read thread %2d starting: tid: %s, file: %s\n", @@ -242,7 +245,8 @@ static inline void writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *key, _key, *value, _value; - u_int keyno; + uint64_t keyno; + size_t len; int ret; char keybuf[64], valuebuf[64]; @@ -251,12 +255,13 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) keyno = __wt_random(&s->rnd) % nkeys + 1; if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); + cursor->set_key(cursor, keyno); if (keyno % 5 == 0) { ++s->remove; if ((ret = @@ -268,8 +273,10 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) if (ftype == FIX) cursor->set_value(cursor, 0x10); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "XXX %37u", keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->update(cursor)) != 0) @@ -277,7 +284,7 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) } if (log_print) testutil_check(session->log_printf(session, - "Writer Thread %p key %017u", pthread_self(), keyno)); + "Writer Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* @@ -296,7 +303,7 @@ writer(void *arg) id = (int)(uintptr_t)arg; s = &run_info[id]; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf("write thread %2d starting: tid: %s, file: %s\n", diff --git a/src/third_party/wiredtiger/test/thread/stats.c b/src/third_party/wiredtiger/test/thread/stats.c index 67a2c02719b..839d65e8a4d 100644 --- a/src/third_party/wiredtiger/test/thread/stats.c +++ b/src/third_party/wiredtiger/test/thread/stats.c @@ -65,7 +65,8 @@ stats(void) /* File statistics. */ if (!multiple_files) { - (void)snprintf(name, sizeof(name), "statistics:" FNAME, 0); + testutil_check(__wt_snprintf( + name, sizeof(name), "statistics:" FNAME, 0)); if ((ret = session->open_cursor( session, name, NULL, NULL, &cursor)) != 0) testutil_die(ret, "session.open_cursor"); diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index baadbf2adb9..d2ed4c74bb7 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -37,7 +37,6 @@ int multiple_files; /* File per thread */ int session_per_op; /* New session per operation */ static char home[512]; /* Program working dir */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); @@ -59,10 +58,7 @@ main(int argc, char *argv[]) int ch, cnt, runs; char *config_open, *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); config_open = NULL; working_dir = NULL; @@ -189,19 +185,15 @@ wt_connect(char *config_open) }; int ret; char config[512]; - size_t print_count; testutil_clean_work_dir(home); testutil_make_work_dir(home); - print_count = (size_t)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); - - if (print_count >= sizeof(config)) - testutil_die(EINVAL, "Config string too long"); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 1491c9a6938..934dac86a7b 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -28,6 +28,7 @@ #include "test_util.h" void (*custom_die)(void) = NULL; +const char *progname = "program name not set"; /* * die -- @@ -42,7 +43,9 @@ testutil_die(int e, const char *fmt, ...) if (custom_die != NULL) (*custom_die)(); + fprintf(stderr, "%s: FAILED", progname); if (fmt != NULL) { + fprintf(stderr, ": "); va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); @@ -54,6 +57,20 @@ testutil_die(int e, const char *fmt, ...) exit(EXIT_FAILURE); } +/* + * testutil_set_progname -- + * Set the global program name for error handling. + */ +const char * +testutil_set_progname(char * const *argv) +{ + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + return (progname); +} + /* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. @@ -78,7 +95,7 @@ testutil_work_dir_from_path(char *buffer, size_t len, const char *dir) * Remove the work directory. */ void -testutil_clean_work_dir(char *dir) +testutil_clean_work_dir(const char *dir) { size_t len; int ret; @@ -91,14 +108,14 @@ testutil_clean_work_dir(char *dir) if ((buf = malloc(len)) == NULL) testutil_die(ENOMEM, "Failed to allocate memory"); - snprintf(buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, - RM_COMMAND, dir); + testutil_check(__wt_snprintf( + buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, RM_COMMAND, dir)); #else len = strlen(dir) + strlen(RM_COMMAND) + 1; if ((buf = malloc(len)) == NULL) testutil_die(ENOMEM, "Failed to allocate memory"); - snprintf(buf, len, "%s%s", RM_COMMAND, dir); + testutil_check(__wt_snprintf(buf, len, "%s%s", RM_COMMAND, dir)); #endif if ((ret = system(buf)) != 0 && ret != ENOENT) @@ -125,7 +142,7 @@ testutil_make_work_dir(char *dir) testutil_die(ENOMEM, "Failed to allocate memory"); /* mkdir shares syntax between Windows and Linux */ - snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir); + testutil_check(__wt_snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir)); if ((ret = system(buf)) != 0) testutil_die(ret, "%s", buf); free(buf); @@ -149,20 +166,25 @@ testutil_cleanup(TEST_OPTS *opts) } /* - * testutil_disable_long_tests -- - * Return if TESTUTIL_DISABLE_LONG_TESTS is set. + * testutil_enable_long_tests -- + * Return if TESTUTIL_ENABLE_LONG_TESTS is set. */ bool -testutil_disable_long_tests(void) +testutil_enable_long_tests(void) { const char *res; + bool enable_long_tests; if (__wt_getenv(NULL, - "TESTUTIL_DISABLE_LONG_TESTS", &res) == WT_NOTFOUND) + "TESTUTIL_ENABLE_LONG_TESTS", &res) == WT_NOTFOUND) return (false); + /* Accept anything other than "TESTUTIL_ENABLE_LONG_TESTS=0". */ + enable_long_tests = res[0] != '0'; + free((void *)res); - return (true); + + return (enable_long_tests); } /* diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c index 74a1c021d5d..c3eff3360de 100644 --- a/src/third_party/wiredtiger/test/utility/parse_opts.c +++ b/src/third_party/wiredtiger/test/utility/parse_opts.c @@ -43,10 +43,7 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) opts->running = true; opts->verbose = false; - if ((opts->progname = strrchr(argv[0], DIR_DELIM)) == NULL) - opts->progname = argv[0]; - else - ++opts->progname; + opts->progname = testutil_set_progname(argv); while ((ch = __wt_getopt(opts->progname, argc, argv, "A:h:n:o:pR:T:t:vW:")) != EOF) @@ -118,13 +115,15 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) if (opts->home == NULL) { len = strlen("WT_TEST.") + strlen(opts->progname) + 10; opts->home = dmalloc(len); - snprintf(opts->home, len, "WT_TEST.%s", opts->progname); + testutil_check(__wt_snprintf( + opts->home, len, "WT_TEST.%s", opts->progname)); } /* Setup the default URI string */ len = strlen("table:") + strlen(opts->progname) + 10; opts->uri = dmalloc(len); - snprintf(opts->uri, len, "table:%s", opts->progname); + testutil_check(__wt_snprintf( + opts->uri, len, "table:%s", opts->progname)); return (0); } diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index f6a9cd68e02..406ed2c4961 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -48,7 +48,7 @@ /* Generic option parsing structure shared by all test cases. */ typedef struct { char *home; - char *progname; + const char *progname; enum { TABLE_COL=1, /* Fixed-length column store */ TABLE_FIX=2, /* Variable-length column store */ TABLE_ROW=3 /* Row-store */ @@ -183,12 +183,15 @@ void *dmalloc(size_t); void *drealloc(void *, size_t); void *dstrdup(const void *); void *dstrndup(const char *, size_t); -void testutil_clean_work_dir(char *); +void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); -bool testutil_disable_long_tests(void); +bool testutil_enable_long_tests(void); void testutil_make_work_dir(char *); int testutil_parse_opts(int, char * const *, TEST_OPTS *); void testutil_work_dir_from_path(char *, size_t, const char *); void *thread_append(void *); void *thread_insert_append(void *); void *thread_prev(void *); + +extern const char *progname; +const char *testutil_set_progname(char * const *); diff --git a/src/third_party/wiredtiger/test/utility/thread.c b/src/third_party/wiredtiger/test/utility/thread.c index 38465b2f02b..122ad554442 100644 --- a/src/third_party/wiredtiger/test/utility/thread.c +++ b/src/third_party/wiredtiger/test/utility/thread.c @@ -57,8 +57,8 @@ thread_append(void *arg) if (opts->table_type == TABLE_FIX) cursor->set_value(cursor, buf[0]); else { - snprintf(buf, sizeof(buf), - "%" PRIu64 " VALUE ------", recno); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "%" PRIu64 " VALUE ------", recno)); cursor->set_value(cursor, buf); } testutil_check(cursor->insert(cursor)); @@ -94,7 +94,8 @@ thread_insert_append(void *arg) session, opts->uri, NULL, NULL, &cursor)); for (i = 0; i < opts->nrecords; ++i) { - snprintf(kbuf, sizeof(kbuf), "%010d KEY------", (int)i); + testutil_check(__wt_snprintf( + kbuf, sizeof(kbuf), "%010d KEY------", (int)i)); cursor->set_key(cursor, kbuf); cursor->set_value(cursor, "========== VALUE ======="); testutil_check(cursor->insert(cursor)); diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index a6050e92bca..8985904fb19 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -36,6 +36,8 @@ #include #include +#include "wt_internal.h" + #define inline __inline /* Define some POSIX types */ @@ -52,12 +54,7 @@ typedef int u_int; /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 -#define snprintf _wt_snprintf - -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...); +#define snprintf __wt_snprintf #endif /* @@ -87,7 +84,7 @@ int usleep(useconds_t useconds); /* - * Emulate the support we need for the tests + * Emulate the support we need for tests and example code. */ typedef CRITICAL_SECTION pthread_mutex_t; typedef CONDITION_VARIABLE pthread_cond_t; diff --git a/src/third_party/wiredtiger/test/wtperf/test_conf_dump.py b/src/third_party/wiredtiger/test/wtperf/test_conf_dump.py new file mode 100644 index 00000000000..ef7f276a1d0 --- /dev/null +++ b/src/third_party/wiredtiger/test/wtperf/test_conf_dump.py @@ -0,0 +1,296 @@ +# Usage: python test_conf_dump.py +# +# This script tests if the config file dumped in the test directory corresponds +# correctly to the wtperf config file used. Command line options to wtperf are +# also taken into account. +# +# Following expectations are checked for: +# 1. If provided through multiple sources, "conn_config" and "table_config" +# configuration options are appended to each other. All other options get +# replaced by a higher precedent source. +# 2. The precedence order for the options in an increasing order is as follows: +# default option, +# provided through config file, +# provided through option -o +# provided through option -C (for conn_config) or -T (for table_config) +# +# Test fails if any config option is missing or has a wrong value. Test also +# fails if the value for the option is not replaced/appended in the correct +# order of precedence as stated above. + +import os, re, subprocess, sys + +OP_FILE = "WT_TEST/CONFIG.wtperf" +TMP_CONF = "__tmp.wtperf" +WTPERF_BIN = "./wtperf" +WTPERF_DIR = "../../build_posix/bench/wtperf/" + +CONF_NOT_PROVIDED = -2 + +# Generate a wtperf conf file to use +def generate_conf_file(file_name): + f = open(file_name, 'w') + f.write( +'''conn_config="cache_size=16GB,eviction=(threads_max=4),log=(enabled=false),session_max=33" +table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" +close_conn=false +icount=1500 +create=true +compression="snappy" +checkpoint_interval=5 +checkpoint_threads=1 +populate_threads=1 +report_interval=5 +session_count_idle=50 +session_count_idle=60 +session_count_idle=70 +session_count_idle=80 +run_time=5 +sample_interval=5 +sample_rate=1 +table_count=2 +threads=((count=6,updates=1)) +value_sz=1000 +warmup=2 +''') + f.close() + +# Build a command from the given options and execute wtperf +def execute_wtperf(conf_file, option_C = "", option_T = "", option_o = ""): + # Generate the command to run, execute wtperf + cmd = WTPERF_BIN + " -O " + conf_file + if option_C: + cmd += " -C " + option_C + if option_T: + cmd += " -T " + option_T + if option_o: + # Any quotes in option_o need to be escaped before providing it as part + # of the command + option_o_cmd_str = option_o.replace('"', '\\"') + cmd += " -o " + option_o_cmd_str + + print "Running: ", cmd + subprocess.check_call(cmd, shell=True) + print "=========================\n" + +# Build a dictionary of config key and it's value from the given config file. +# Optionally take -C, -T and -o and overwrite/append values as per correct +# precedence +def build_dict_from_conf( + conf_file, option_C = "", option_T = "", option_o = ""): + # Open given conf file and make a dictionary of passed arguments and values + with open(conf_file) as f: + lines = f.read().splitlines() + + # Maintain precedence order of config file, -o, -C/-T + # Build a dict of config options, appending values for table_config and + # conn_config, if specified multiple times. Replace with the latest in + # case of all other configuration keys. + key_val_dict = {} + for line in lines: + if re.match('^\s*#', line) is None: + key_val_pair = line.split('=', 1) + if ((key_val_pair[0] == 'table_config' or + key_val_pair[0] == 'conn_config') and + key_val_pair[0] in key_val_dict): + tmp_val = key_val_dict[key_val_pair[0]][:-1] + tmp_val += "," + tmp_val += key_val_pair[1][1:] + key_val_dict[key_val_pair[0]] = tmp_val + else: + key_val_dict[key_val_pair[0]] = key_val_pair[1] + + # If provided, put option o in the dict + if option_o: + opt_o_key_val_list = option_o.split(',') + for op_o_key_val in opt_o_key_val_list: + key_val_pair = op_o_key_val.split('=', 1) + if ((key_val_pair[0] == 'table_config' or + key_val_pair[0] == 'conn_config') and + key_val_pair[0] in key_val_dict): + tmp_val = key_val_dict[key_val_pair[0]][:-1] + tmp_val += "," + tmp_val += key_val_pair[1][1:] + key_val_dict[key_val_pair[0]] = tmp_val + else: + key_val_dict[key_val_pair[0]] = key_val_pair[1] + + # If provided, put option C in the dict + if option_C: + tmp_val = key_val_dict["conn_config"][:-1] + tmp_val += "," + tmp_val += option_C[1:] + key_val_dict["conn_config"] = tmp_val + + # If provided, put option T in the dict + if option_T: + tmp_val = key_val_dict["table_config"][:-1] + tmp_val += "," + tmp_val += option_T[1:] + key_val_dict["table_config"] = tmp_val + + return key_val_dict + +# Extract configuration value for the given key from the given config file +def extract_config_from_file(conf_file, key): + ret_val = "" + with open(conf_file) as f: + lines = f.read().splitlines() + for line in lines: + if re.match('^\s*#', line) is None: + key_val_pair = line.split('=', 1) + if key_val_pair[0] == key: + ret_val = key_val_pair[1] + return ret_val + +# Extract configuration value for the given key from the given "-o" string +def extract_config_from_opt_o(option_o, key): + ret_val = "" + opt_o_key_val_list = option_o.split(',') + for op_o_key_val in opt_o_key_val_list: + key_val_pair = op_o_key_val.split('=', 1) + if key_val_pair[0] == key: + ret_val = key_val_pair[1] + return ret_val + +# Execute test: +# Run wtperf with given config and check if the dumped config file matches the +# given inputs +def run_test(conf_file, option_C = "", option_T = "", option_o = ""): + # Run wtperf + execute_wtperf(conf_file, option_C, option_T, option_o) + + key_val_dict_ip = build_dict_from_conf( + conf_file, option_C, option_T, option_o) + key_val_dict_op = build_dict_from_conf(OP_FILE) + + conn_config_from_file = extract_config_from_file(conf_file, "conn_config") + table_config_from_file = extract_config_from_file(conf_file, "table_config") + conn_config_from_opt_o = "" + table_config_from_opt_o = "" + if option_o: + conn_config_from_opt_o = extract_config_from_opt_o( + option_o, "conn_config") + table_config_from_opt_o = extract_config_from_opt_o( + option_o, "table_config") + + # Check if dumped output conf matches with input file and options + match = True + for key in key_val_dict_ip: + match_itr = True + + # Check if we see this config key in the dumped file + if not key in key_val_dict_op: + print "Key '", key, "' not found in dumped file ", OP_FILE + match = match_itr = False + continue + + # Check if values from all sources of conn_config are presented in the + # conn_config in dumped file. Also check of their relative ordering as + # per precedence rules defined. + if (key == 'conn_config' and + (conn_config_from_file or conn_config_from_opt_o or option_C)): + # Should find these config in order: file < option o < option C + file_loc = CONF_NOT_PROVIDED + option_o_loc = CONF_NOT_PROVIDED + option_C_loc = CONF_NOT_PROVIDED + op_conn_config = key_val_dict_op['conn_config'] + + if conn_config_from_file: + file_loc = op_conn_config.find(conn_config_from_file[1:-1]) + if conn_config_from_opt_o: + option_o_loc = op_conn_config.find(conn_config_from_opt_o[1:-1]) + if option_C: + option_C_loc = op_conn_config.find(option_C[1:-1]) + + # Check if value from any of the sources is missing + if ((conn_config_from_file and file_loc == -1) or + (conn_config_from_opt_o and option_o_loc == -1) or + (option_C and option_C_loc == -1)): + print "Part of conn_config missing in dumped file ", OP_FILE + match_itr = False + + # Check if the values got appended in the correct order + if match_itr: + if ((option_o_loc != CONF_NOT_PROVIDED and + option_o_loc < file_loc) or + (option_C_loc != CONF_NOT_PROVIDED and + (option_C_loc < file_loc or option_C_loc < option_o_loc))): + print "Detected incorrect config append order:" + match_itr = False + + # Check if values from all sources of table_config are presented in the + # table_config in dumped file. Also check of their relative ordering as + # per precedence rules defined. + if (key == 'table_config' and + (table_config_from_file or table_config_from_opt_o or option_T)): + # Should find these config in order: file < option o < option T + file_loc = CONF_NOT_PROVIDED + option_o_loc = CONF_NOT_PROVIDED + option_T_loc = CONF_NOT_PROVIDED + op_table_config = key_val_dict_op['table_config'] + + if table_config_from_file: + file_loc = op_table_config.find(table_config_from_file[1:-1]) + if table_config_from_opt_o: + option_o_loc = op_table_config.find( + table_config_from_opt_o[1:-1]) + if option_T: + option_T_loc = op_table_config.find(option_T[1:-1]) + + # Check if value from any of the sources is missing + if ((table_config_from_file and file_loc == -1) or + (table_config_from_opt_o and option_o_loc == -1) or + (option_T and option_T_loc == -1)): + print "Part of table_config missing in dumped file ", OP_FILE + match_itr = False + + # Check if the values got appended in the correct order + if match_itr: + if ((option_o_loc != CONF_NOT_PROVIDED and + option_o_loc < file_loc) or + (option_T_loc != CONF_NOT_PROVIDED and + (option_T_loc < file_loc or option_T_loc < option_o_loc))): + print "Detected incorrect config append order:" + match_itr = False + + if (key != 'table_config' and key != 'conn_config' and + key_val_dict_ip[key] != key_val_dict_op[key]): + print "Config mismatch between:" + match_itr = False + + if match_itr is False: + print "Input Config:", key, '=', key_val_dict_ip[key] + print "Dumped Config:", key, '=', key_val_dict_op[key] + print "\n" + + match = match and match_itr + + return match + +# ----------------- Execute Test -------------- +# If a wtperf conf file is provided use it, else generate a temp conf file +os.chdir(WTPERF_DIR) +if len(sys.argv) == 2: + conf_file = sys.argv[1] +else: + conf_file = TMP_CONF + generate_conf_file(conf_file) + +# Run a test with no options +if not run_test(conf_file): + exit(-1) + +# Run a test with -C, -T, -o provided +option_o = "verbose=2,conn_config=\"session_max=135\",table_config=\"type=lsm\",sample_interval=2,run_time=0,sample_rate=2,readonly=false" +option_C = "\"cache_size=10GB,session_max=115\"" +option_T = "\"allocation_size=8k,split_pct=92\"" +if not run_test(conf_file, option_C, option_T, option_o): + exit(-1) + +# Cleanup generated temp files +subprocess.check_call("rm -rf WT_TEST/", shell=True) +if len(sys.argv) == 1 and conf_file == TMP_CONF: + subprocess.check_call("rm " + TMP_CONF, shell=True) + +print "All tests succeeded" diff --git a/src/third_party/wiredtiger/tools/wtstats/stat_data.py b/src/third_party/wiredtiger/tools/wtstats/stat_data.py index 635e710c469..a94ce524ae3 100644 --- a/src/third_party/wiredtiger/tools/wtstats/stat_data.py +++ b/src/third_party/wiredtiger/tools/wtstats/stat_data.py @@ -34,6 +34,9 @@ no_scale_per_second_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'session: table alter failed calls', + 'session: table alter successful calls', + 'session: table alter unchanged and skipped', 'session: table compact failed calls', 'session: table compact successful calls', 'session: table create failed calls', @@ -91,6 +94,7 @@ no_scale_per_second_list = [ 'btree: row-store leaf pages', 'cache: bytes currently in the cache', 'cache: overflow values cached in memory', + 'cache: tracked dirty bytes in the cache', 'cache_walk: Average difference between current eviction generation when the page was last considered', 'cache_walk: Average on-disk page image size seen', 'cache_walk: Clean pages currently in cache', @@ -124,6 +128,8 @@ no_clear_list = [ 'cache: eviction currently operating in aggressive mode', 'cache: eviction empty score', 'cache: eviction state', + 'cache: eviction worker thread active', + 'cache: eviction worker thread stable number', 'cache: files with active eviction walks', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', @@ -147,6 +153,9 @@ no_clear_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'session: table alter failed calls', + 'session: table alter successful calls', + 'session: table alter unchanged and skipped', 'session: table compact failed calls', 'session: table compact successful calls', 'session: table create failed calls', @@ -180,6 +189,7 @@ no_clear_list = [ 'transaction: transaction range of IDs currently pinned by named snapshots', 'btree: btree checkpoint generation', 'cache: bytes currently in the cache', + 'cache: tracked dirty bytes in the cache', 'cache_walk: Average difference between current eviction generation when the page was last considered', 'cache_walk: Average on-disk page image size seen', 'cache_walk: Clean pages currently in cache', diff --git a/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template b/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template index ef0f60bb4f2..8e26e519456 100644 --- a/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template +++ b/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template @@ -10011,19 +10011,18 @@ svg.multiline .line { }
\ No newline at end of file diff --git a/version.json b/version.json index a0b32133df4..ad5f47b99da 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "009580ad490190ba33d1c6253ebd8d91808923e4", - "version": "3.2.11" + "githash": "056bf45128114e44c5358c7a8776fb582363e094", + "version": "3.2.16" } \ No newline at end of file -- cgit v1.2.3 From 930687d86a670280fe7e5f98534183a571eaecd1 Mon Sep 17 00:00:00 2001 From: Apollon Oikonomopoulos Date: Sat, 7 Oct 2017 23:39:33 +0300 Subject: New upstream version 3.2.17 --- etc/evergreen.yml | 86 +---- etc/longevity.yml | 11 +- etc/system_perf.yml | 175 +++------- jstests/aggregation/bugs/server6118.js | 17 +- jstests/aggregation/bugs/server6179.js | 19 +- jstests/aggregation/bugs/server7781.js | 17 +- jstests/aggregation/bugs/server9444.js | 154 +++++---- jstests/auth/mongos_cache_invalidation.js | 11 +- jstests/core/group9.js | 20 ++ jstests/core/mr4.js | 6 +- jstests/gle/gle_sharded_write.js | 376 ++++++++++----------- jstests/multiVersion/libs/multi_rs.js | 7 +- jstests/noPassthrough/backup_restore.js | 11 +- jstests/noPassthrough/initial_sync_cloner_dups.js | 3 +- .../no_balance_collection.js | 8 +- .../apply_ops_insert_write_conflict_nonatomic.js | 8 + jstests/replsets/election_not_blocked.js | 2 +- jstests/replsets/initial_sync1.js | 6 +- jstests/replsets/initial_sync2.js | 20 +- .../libs/apply_ops_insert_write_conflict.js | 88 +++++ jstests/replsets/maintenance.js | 2 +- jstests/replsets/oplog_truncated_on_recovery.js | 2 +- .../priority_takeover_one_node_higher_priority.js | 14 +- .../read_committed_with_catalog_changes.js | 4 +- jstests/replsets/replsetadd_profile.js | 2 +- jstests/replsets/replsetprio1.js | 8 +- jstests/replsets/request_primary_stepdown.js | 2 +- jstests/replsets/resync_with_write_load.js | 2 +- jstests/replsets/rollback.js | 2 +- jstests/replsets/rollback2.js | 2 +- jstests/replsets/rollback3.js | 2 +- jstests/replsets/rollback5.js | 2 +- jstests/replsets/rollback_auth.js | 2 +- jstests/replsets/rollback_cmd_unrollbackable.js | 2 +- jstests/replsets/rollback_collMod_fatal.js | 2 +- jstests/replsets/rollback_different_h.js | 2 +- jstests/replsets/rollback_dropdb.js | 2 +- jstests/replsets/rollback_fake_cmd.js | 2 +- jstests/replsets/rollback_index.js | 2 +- jstests/replsets/rslib.js | 2 +- jstests/replsets/stepdown.js | 2 +- jstests/replsets/stepdown_kill_other_ops.js | 2 +- jstests/replsets/stepdown_killop.js | 2 +- jstests/replsets/stepdown_long_wait_time.js | 2 +- jstests/replsets/sync_passive.js | 2 +- jstests/replsets/two_nodes_priority_take_over.js | 2 +- jstests/sharding/balance_repl.js | 21 +- jstests/sharding/balance_tags2.js | 15 +- jstests/sharding/explain_cmd.js | 357 +++++++++---------- .../sharding/explain_find_and_modify_sharded.js | 2 +- jstests/sharding/hash_shard_unique_compound.js | 60 ++-- jstests/sharding/mapReduce_inSharded_outSharded.js | 128 +++---- jstests/sharding/migrateBig.js | 46 ++- jstests/sharding/migrateBig_balancer.js | 19 +- jstests/sharding/printShardingStatus.js | 2 +- jstests/sharding/shard3.js | 10 +- jstests/sharding/split_with_force_small.js | 99 +++--- jstests/sharding/stale_version_write.js | 46 +-- jstests/slow2/mr_during_migrate.js | 159 +++++---- src/mongo/base/validate_locale.cpp | 13 +- src/mongo/db/commands/mr.cpp | 6 +- src/mongo/db/dbhelpers.cpp | 5 +- src/mongo/db/exec/group.cpp | 3 +- src/mongo/db/ops/update_driver.cpp | 12 +- src/mongo/db/repl/oplog.cpp | 6 +- src/mongo/dbtests/jstests.cpp | 44 +++ src/mongo/s/SConscript | 2 +- src/mongo/s/catalog/catalog_manager.h | 6 +- src/mongo/s/chunk_manager.cpp | 3 + src/mongo/scripting/deadline_monitor.cpp | 2 +- src/mongo/scripting/deadline_monitor.h | 12 +- src/mongo/scripting/engine.cpp | 3 + src/mongo/scripting/engine.h | 2 + src/mongo/scripting/mozjs/bson.cpp | 10 +- src/mongo/scripting/mozjs/implscope.cpp | 10 + src/mongo/scripting/mozjs/implscope.h | 5 + src/mongo/scripting/mozjs/objectwrapper.cpp | 7 +- src/mongo/scripting/mozjs/proxyscope.cpp | 4 + src/mongo/scripting/mozjs/proxyscope.h | 2 + src/mongo/shell/replsettest.js | 2 +- src/third_party/wiredtiger/dist/s_string.ok | 1 + src/third_party/wiredtiger/dist/stat_data.py | 1 + src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/btree/row_key.c | 2 + .../wiredtiger/src/docs/programming.dox | 15 +- src/third_party/wiredtiger/src/docs/spell.ok | 1 + .../wiredtiger/src/docs/tune-build-options.dox | 9 + src/third_party/wiredtiger/src/evict/evict_lru.c | 7 + src/third_party/wiredtiger/src/include/extern.h | 1 + src/third_party/wiredtiger/src/include/gcc.h | 2 +- src/third_party/wiredtiger/src/include/lint.h | 1 + src/third_party/wiredtiger/src/include/misc.i | 25 ++ src/third_party/wiredtiger/src/include/msvc.h | 4 +- src/third_party/wiredtiger/src/include/session.h | 7 + src/third_party/wiredtiger/src/include/stat.h | 1 + src/third_party/wiredtiger/src/include/txn.h | 2 + .../wiredtiger/src/include/wiredtiger.in | 346 ++++++++++--------- .../wiredtiger/src/os_common/os_alloc.c | 2 + .../wiredtiger/src/os_common/os_getopt.c | 8 +- src/third_party/wiredtiger/src/os_posix/os_dir.c | 6 + src/third_party/wiredtiger/src/os_posix/os_time.c | 22 +- src/third_party/wiredtiger/src/os_win/os_time.c | 9 +- .../wiredtiger/src/schema/schema_list.c | 31 ++ .../wiredtiger/src/session/session_api.c | 17 +- src/third_party/wiredtiger/src/support/stat.c | 3 + src/third_party/wiredtiger/src/txn/txn.c | 22 +- src/third_party/wiredtiger/src/txn/txn_log.c | 11 + version.json | 4 +- 108 files changed, 1528 insertions(+), 1274 deletions(-) create mode 100644 jstests/core/group9.js create mode 100644 jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js create mode 100644 jstests/replsets/libs/apply_ops_insert_write_conflict.js create mode 100644 src/third_party/wiredtiger/src/docs/tune-build-options.dox diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 9bef88f9a0e..97b0f49b755 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -90,7 +90,6 @@ variables: - ubuntu1404-rocksdb - debian71 - debian81 - - solaris-64-bit - osx-107-ssl - osx-107 @@ -201,7 +200,7 @@ functions: rm -rf rocksdb git clone https://github.com/facebook/rocksdb.git cd rocksdb - make static_lib + make USE_RTTI=1 static_lib fi "build new tools" : @@ -5394,89 +5393,6 @@ buildvariants: - ubuntu1604-packer - name: push -########################################### -# Solaris buildvariants # -########################################### - -- name: solaris-64-bit - display_name: "* Solaris" - modules: - - mongo-tools - run_on: - - solaris - expansions: - push_path: sunos5 - push_bucket: downloads.mongodb.org - push_name: sunos5 - push_arch: x86_64 - gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH - tooltags: -gccgoflags "-lsocket -lnsl" - compile_flags: CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ -j$(kstat cpu | sort -u | grep -c "^module") --release CCFLAGS="-m64" LINKFLAGS="-m64 -static-libstdc++ -static-libgcc" OBJCOPY=/opt/mongodbtoolchain/bin/objcopy - num_jobs_available: $(kstat cpu | sort -u | grep -c "^module") - tasks: - - name: compile - - name: aggregation - - name: aggregation_WT - - name: auth - - name: auth_WT - - name: bulk_gle_passthrough - - name: bulk_gle_passthrough_WT - - name: concurrency - - name: concurrency_WT - - name: concurrency_replication - - name: concurrency_replication_WT - - name: concurrency_sharded - - name: concurrency_sharded_WT - - name: concurrency_sharded_sccc - - name: concurrency_sharded_sccc_WT - - name: dbtest - - name: dbtest_WT - - name: disk - - name: durability - - name: failpoints - - name: httpinterface - - name: jsCore - - name: jsCore_compatibility - - name: jsCore_compatibility_WT - - name: jsCore_small_oplog - - name: jsCore_small_oplog_rs - - name: jsCore_small_oplog_rs_WT - - name: jsCore_small_oplog_WT - - name: jsCore_WT - - name: mmap - - name: mongosTest - - name: noPassthrough - - name: noPassthroughWithMongod - - name: noPassthroughWithMongod_WT - - name: noPassthrough_WT - - name: parallel - - name: parallel_compatibility - - name: parallel_compatibility_WT - - name: parallel_WT - - name: replicasets - - name: replicasets_WT - - name: replication - - name: replication_WT - - name: sharding - - name: sharding_csrs_upgrade - - name: sharded_collections_jscore_passthrough - - name: sharded_collections_jscore_passthrough_WT - - name: sharding_jscore_passthrough - - name: sharding_jscore_passthrough_WT - - name: sharding_jscore_passthrough_wire_ops_WT - - name: sharding_WT - - name: sharding_csrs_upgrade_WT - - name: slow1 - - name: slow1_WT - - name: slow2 - - name: slow2_WT - - name: tool - - name: tool_WT - - name: unittests - - name: push - distros: - - rhel70 - ########################################### # Debian buildvariants # ########################################### diff --git a/etc/longevity.yml b/etc/longevity.yml index b5cb674a354..af830baf318 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -141,15 +141,15 @@ functions: "infrastructure provisioning": - command: shell.exec - # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances. + # call infrastructure-provisioning.py. This will either create a cluster, or update tags on existing instances. params: working_dir: work script: | set -e set -v source ./dsienv.sh - export PRODUCTION=true - $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster} + source ./venv/bin/activate + $DSI_PATH/bin/infrastructure_provisioning.py "configure mongodb cluster": - command: shell.exec @@ -190,10 +190,11 @@ functions: set -o verbose source ./dsienv.sh # Longevity runs so rarely, we simply teardown the cluster when done. - # Note that nowadays infrastructure_teardown.sh is actually copying the terraform.tfstate into /data/infrastructure_provisioning + # Note that nowadays infrastructure_teardown.py is actually copying the terraform.tfstate into /data/infrastructure_provisioning # but as of this writing the rhel70-perf-longevity distro didn't actually use the teardown hook. source ./dsienv.sh - $DSI_PATH/bin/infrastructure_teardown.sh + source ./venv/bin/activate + $DSI_PATH/bin/infrastructure_teardown.py echo "Cluster DESTROYED." echo echo "All perf results" diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 641b34bac49..e654824801c 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -25,8 +25,8 @@ post: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: work/reports/graphs/timeseries-p1.html - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-p1-${task_name}-${build_id}.html + local_file: work/reports/graphs/timeseries-mongod.0.html + remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-mongod.0-${task_name}-${build_id}.html bucket: mciuploads permissions: public-read content_type: text/html @@ -103,7 +103,6 @@ functions: ext: ${ext} script_flags : ${script_flags} dsi_rev: ${dsi_rev} - compare_task: ${compare_task} workloads_rev: ${workloads_rev} # compositions of expansions @@ -151,15 +150,15 @@ functions: "infrastructure provisioning": - command: shell.exec - # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances. + # call infrastructure-provisioning.py. This will either create a cluster, or update tags on existing instances. params: working_dir: work script: | set -e set -v source ./dsienv.sh - export PRODUCTION=true - $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster} + source ./venv/bin/activate + $DSI_PATH/bin/infrastructure_provisioning.py "configure mongodb cluster": - command: shell.exec @@ -237,45 +236,6 @@ functions: OVERRIDEFILE="../src/dsi/dsi/analysis/v3.2/system_perf_override.json" python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file reports/perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} - "compare": - - command: shell.exec - params: - script: | - set -o verbose - rm -rf ./src ./work - mkdir src - mkdir work - - command: manifest.load - - command: git.get_project - params: - directory: src - revisions: # for each module include revision as : ${_rev} - dsi: ${dsi_rev} - - command: json.get - params: - task: ${compare_task} - variant : ${variant1} - file: "work/standalone.json" - name: "perf" - - command: json.get - params: - task: ${compare_task} - variant : ${variant2} - file: "work/oplog.json" - name: "perf" - - command: shell.exec - type : test - params: - working_dir: work - script: | - set -o errexit - set -o verbose - python -u ../src/dsi/dsi/analysis/compare.py -b standalone.json -c oplog.json - - command: "json.send" - params: - name: "perf" - file: "work/perf.json" - ####################################### # Tasks # ####################################### @@ -357,6 +317,50 @@ tasks: vars: script_flags: --ycsb-throughput-analysis reports +- name: industry_benchmarks_wmajority_WT + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" + vars: + storageEngine: "wiredTiger" + test: "ycsb-wmajority" + - func: "infrastructure provisioning" + - func: "configure mongodb cluster" + vars: + storageEngine: "wiredTiger" + - func: "run test" + vars: + storageEngine: "wiredTiger" + test: "ycsb-wmajority" + - func: "make test log artifact" + - func: "analyze" + vars: + script_flags: --ycsb-throughput-analysis reports + +- name: industry_benchmarks_wmajority_MMAPv1 + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" + vars: + storageEngine: "mmapv1" + test: "ycsb-wmajority" + - func: "infrastructure provisioning" + - func: "configure mongodb cluster" + vars: + storageEngine: "mmapv1" + - func: "run test" + vars: + storageEngine: "mmapv1" + test: "ycsb-wmajority" + - func: "make test log artifact" + - func: "analyze" + vars: + script_flags: --ycsb-throughput-analysis reports + - name: core_workloads_WT depends_on: - name: compile @@ -518,70 +522,6 @@ tasks: - func: "make test log artifact" - func: "analyze" -- name: industry_benchmarks_WT_oplog_comp - depends_on: - - name: industry_benchmarks_WT - variant: linux-standalone - status : "*" - - name: industry_benchmarks_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: industry_benchmarks_MMAPv1_oplog_comp - depends_on: - - name: industry_benchmarks_MMAPv1 - variant: linux-standalone - status: "*" - - name: industry_benchmarks_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_WT_oplog_comp - depends_on: - - name: core_workloads_WT - variant: linux-standalone - status: "*" - - name: core_workloads_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_MMAPv1_oplog_comp - depends_on: - - name: core_workloads_MMAPv1 - variant: linux-standalone - status: "*" - - name: core_workloads_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - - name: initialsync_WT depends_on: - name: compile @@ -706,6 +646,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: mongos_workloads_WT - name: mongos_workloads_MMAPv1 @@ -728,6 +670,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: non_sharded_workloads_WT - name: non_sharded_workloads_MMAPv1 @@ -748,16 +692,3 @@ buildvariants: - name: initialsync_WT - name: initialsync_MMAPv1 -- name: linux-oplog-compare - display_name: Linux Oplog Compare - batchtime: 10080 # 7 days - modules: *modules - expansions: - project: *project - run_on: - - "rhel70-perf-single" - tasks: - - name: industry_benchmarks_WT_oplog_comp - - name: core_workloads_WT_oplog_comp - - name: industry_benchmarks_MMAPv1_oplog_comp - - name: core_workloads_MMAPv1_oplog_comp diff --git a/jstests/aggregation/bugs/server6118.js b/jstests/aggregation/bugs/server6118.js index f891135de72..3c55ae5ce33 100644 --- a/jstests/aggregation/bugs/server6118.js +++ b/jstests/aggregation/bugs/server6118.js @@ -1,12 +1,12 @@ // SERVER-6118: support for sharded sorts (function() { + 'use strict'; - var s = new ShardingTest({name: "aggregation_sort1", shards: 2, mongos: 1}); - s.stopBalancer(); + var s = new ShardingTest({shards: 2}); - s.adminCommand({enablesharding: "test"}); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); - s.adminCommand({shardcollection: "test.data", key: {_id: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.data", key: {_id: 1}})); var d = s.getDB("test"); @@ -20,12 +20,12 @@ bulkOp.execute(); // Split the data into 3 chunks - s.adminCommand({split: "test.data", middle: {_id: 33}}); - s.adminCommand({split: "test.data", middle: {_id: 66}}); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 33}})); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 66}})); // Migrate the middle chunk to another shard - s.adminCommand( - {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name})); // Check that the results are in order. var result = d.data.aggregate({$sort: {_id: 1}}).toArray(); @@ -36,5 +36,4 @@ } s.stop(); - })(); diff --git a/jstests/aggregation/bugs/server6179.js b/jstests/aggregation/bugs/server6179.js index 1109ddaa67e..503e91a70d1 100644 --- a/jstests/aggregation/bugs/server6179.js +++ b/jstests/aggregation/bugs/server6179.js @@ -1,12 +1,12 @@ // SERVER-6179: support for two $groups in sharded agg (function() { + 'use strict'; - var s = new ShardingTest({name: "aggregation_multiple_group", shards: 2, mongos: 1}); - s.stopBalancer(); + var s = new ShardingTest({shards: 2}); - s.adminCommand({enablesharding: "test"}); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); - s.adminCommand({shardcollection: "test.data", key: {_id: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.data", key: {_id: 1}})); var d = s.getDB("test"); @@ -20,18 +20,18 @@ bulkOp.execute(); // Split the data into 3 chunks - s.adminCommand({split: "test.data", middle: {_id: 33}}); - s.adminCommand({split: "test.data", middle: {_id: 66}}); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 33}})); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 66}})); // Migrate the middle chunk to another shard - s.adminCommand( - {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name})); // Check that we get results rather than an error var result = d.data.aggregate({$group: {_id: '$_id', i: {$first: '$i'}}}, {$group: {_id: '$i', avg_id: {$avg: '$_id'}}}, {$sort: {_id: 1}}).toArray(); - expected = [ + var expected = [ {"_id": 0, "avg_id": 45}, {"_id": 1, "avg_id": 46}, {"_id": 2, "avg_id": 47}, @@ -47,5 +47,4 @@ assert.eq(result, expected); s.stop(); - })(); diff --git a/jstests/aggregation/bugs/server7781.js b/jstests/aggregation/bugs/server7781.js index 230a8a64c9f..c3918aeb8d2 100644 --- a/jstests/aggregation/bugs/server7781.js +++ b/jstests/aggregation/bugs/server7781.js @@ -1,5 +1,6 @@ // SERVER-7781 $geoNear pipeline stage (function() { + 'use strict'; load('jstests/libs/geo_near_random.js'); load('jstests/aggregation/extras/utils.js'); @@ -59,10 +60,12 @@ shards.push(shard._id); }); - db.adminCommand({shardCollection: db[coll].getFullName(), key: {rand: 1}}); + assert.commandWorked( + db.adminCommand({shardCollection: db[coll].getFullName(), key: {rand: 1}})); for (var i = 1; i < 10; i++) { // split at 0.1, 0.2, ... 0.9 - db.adminCommand({split: db[coll].getFullName(), middle: {rand: i / 10}}); + assert.commandWorked( + db.adminCommand({split: db[coll].getFullName(), middle: {rand: i / 10}})); db.adminCommand({ moveChunk: db[coll].getFullName(), find: {rand: i / 10}, @@ -87,13 +90,13 @@ // test with defaults var queryPoint = pointMaker.mkPt(0.25); // stick to center of map - geoCmd = { + var geoCmd = { geoNear: coll, near: queryPoint, includeLocs: true, spherical: true }; - aggCmd = { + var aggCmd = { $geoNear: { near: queryPoint, includeLocs: 'stats.loc', @@ -134,7 +137,7 @@ geoCmd.num = 40; geoCmd.near = queryPoint; aggCmd.$geoNear.near = queryPoint; - aggArr = [aggCmd, {$limit: 50}, {$limit: 60}, {$limit: 40}]; + var aggArr = [aggCmd, {$limit: 50}, {$limit: 60}, {$limit: 40}]; checkOutput(db.runCommand(geoCmd), db[coll].aggregate(aggArr), 40); // Test $geoNear with an initial batchSize of 0. Regression test for SERVER-20935. @@ -157,13 +160,11 @@ test(db, false, '2dsphere'); var sharded = new ShardingTest({shards: 3, mongos: 1}); - sharded.stopBalancer(); - sharded.adminCommand({enablesharding: "test"}); + assert.commandWorked(sharded.s0.adminCommand({enablesharding: "test"})); sharded.ensurePrimaryShard('test', 'shard0001'); test(sharded.getDB('test'), true, '2d'); test(sharded.getDB('test'), true, '2dsphere'); sharded.stop(); - })(); diff --git a/jstests/aggregation/bugs/server9444.js b/jstests/aggregation/bugs/server9444.js index ad5f4b03ca6..f3dc2748b0a 100644 --- a/jstests/aggregation/bugs/server9444.js +++ b/jstests/aggregation/bugs/server9444.js @@ -1,76 +1,80 @@ // server-9444 support disk storage of intermediate results in aggregation - -var t = db.server9444; -t.drop(); - -var sharded = (typeof(RUNNING_IN_SHARDED_AGG_TEST) != 'undefined'); // see end of testshard1.js -if (sharded) { - db.adminCommand({shardcollection: t.getFullName(), key: {"_id": 'hashed'}}); -} - -var memoryLimitMB = sharded ? 200 : 100; - -function loadData() { - var bigStr = Array(1024 * 1024 + 1).toString(); // 1MB of ',' - for (var i = 0; i < memoryLimitMB + 1; i++) - t.insert({_id: i, bigStr: i + bigStr, random: Math.random()}); - - assert.gt(t.stats().size, memoryLimitMB * 1024 * 1024); -} -loadData(); - -function test(pipeline, outOfMemoryCode) { - // ensure by default we error out if exceeding memory limit - var res = t.runCommand('aggregate', {pipeline: pipeline}); - assert.commandFailed(res); - assert.eq(res.code, outOfMemoryCode); - - // ensure allowDiskUse: false does what it says - var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: false}); - assert.commandFailed(res); - assert.eq(res.code, outOfMemoryCode); - - // allowDiskUse only supports bool. In particular, numbers aren't allowed. - var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: 1}); - assert.commandFailed(res); - assert.eq(res.code, 16949); - - // ensure we work when allowDiskUse === true - var res = t.aggregate(pipeline, {allowDiskUse: true}); - assert.eq(res.itcount(), t.count()); // all tests output one doc per input doc -} - -var groupCode = 16945; -var sortCode = 16819; -var sortLimitCode = 16820; - -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], groupCode); - -// sorting with _id would use index which doesn't require extsort -test([{$sort: {random: 1}}], sortCode); -test([{$sort: {bigStr: 1}}], sortCode); // big key and value - -// make sure sort + large limit won't crash the server (SERVER-10136) -test([{$sort: {bigStr: 1}}, {$limit: 1000 * 1000 * 1000}], sortLimitCode); - -// test combining two extSorts in both same and different orders -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: 1}}], groupCode); -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: -1}}], groupCode); -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {random: 1}}], groupCode); -test([{$sort: {random: 1}}, {$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], sortCode); - -var origDB = db; -if (sharded) { - // Stop balancer first before dropping so there will be no contention on the ns lock. - // It's alright to modify the global db variable since sharding tests never run in parallel. - db = db.getSiblingDB('config'); - sh.stopBalancer(); -} - -// don't leave large collection laying around -t.drop(); - -if (sharded) { - sh.startBalancer(); - db = origDB; -} +(function() { + 'use strict'; + + var t = db.server9444; + t.drop(); + + var sharded = (typeof(RUNNING_IN_SHARDED_AGG_TEST) != 'undefined'); // see end of testshard1.js + if (sharded) { + assert.commandWorked( + db.adminCommand({shardcollection: t.getFullName(), key: {"_id": 'hashed'}})); + } + + var memoryLimitMB = sharded ? 200 : 100; + + function loadData() { + var bigStr = Array(1024 * 1024 + 1).toString(); // 1MB of ',' + for (var i = 0; i < memoryLimitMB + 1; i++) + t.insert({_id: i, bigStr: i + bigStr, random: Math.random()}); + + assert.gt(t.stats().size, memoryLimitMB * 1024 * 1024); + } + loadData(); + + function test(pipeline, outOfMemoryCode) { + // ensure by default we error out if exceeding memory limit + var res = t.runCommand('aggregate', {pipeline: pipeline}); + assert.commandFailed(res); + assert.eq(res.code, outOfMemoryCode); + + // ensure allowDiskUse: false does what it says + var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: false}); + assert.commandFailed(res); + assert.eq(res.code, outOfMemoryCode); + + // allowDiskUse only supports bool. In particular, numbers aren't allowed. + var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: 1}); + assert.commandFailed(res); + assert.eq(res.code, 16949); + + // ensure we work when allowDiskUse === true + var res = t.aggregate(pipeline, {allowDiskUse: true}); + assert.eq(res.itcount(), t.count()); // all tests output one doc per input doc + } + + var groupCode = 16945; + var sortCode = 16819; + var sortLimitCode = 16820; + + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], groupCode); + + // sorting with _id would use index which doesn't require extsort + test([{$sort: {random: 1}}], sortCode); + test([{$sort: {bigStr: 1}}], sortCode); // big key and value + + // make sure sort + large limit won't crash the server (SERVER-10136) + test([{$sort: {bigStr: 1}}, {$limit: 1000 * 1000 * 1000}], sortLimitCode); + + // test combining two extSorts in both same and different orders + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: 1}}], groupCode); + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: -1}}], groupCode); + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {random: 1}}], groupCode); + test([{$sort: {random: 1}}, {$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], sortCode); + + var origDB = db; + if (sharded) { + // Stop balancer first before dropping so there will be no contention on the ns lock. + // It's alright to modify the global db variable since sharding tests never run in parallel. + db = db.getSiblingDB('config'); + sh.stopBalancer(); + } + + // don't leave large collection laying around + t.drop(); + + if (sharded) { + sh.startBalancer(); + db = origDB; + } +})(); diff --git a/jstests/auth/mongos_cache_invalidation.js b/jstests/auth/mongos_cache_invalidation.js index 60700956e39..4594a18f8c4 100644 --- a/jstests/auth/mongos_cache_invalidation.js +++ b/jstests/auth/mongos_cache_invalidation.js @@ -209,13 +209,10 @@ db3.auth('spencer', 'pwd'); // s0/db1 should update its cache instantly assert.commandFailedWithCode(db1.foo.runCommand("collStats"), authzErrorCode); - // s1/db2 should update its cache in 5 seconds. - assert.soon( - function() { - return db2.foo.runCommand("collStats").code == authzErrorCode; - }, - "Mongos did not update its user cache after 5 seconds", - 6 * 1000); // Give an extra 1 second to avoid races + // s1/db2 should update its cache in 10 seconds. + assert.soon(function() { + return db2.foo.runCommand("collStats").code == authzErrorCode; + }, "Mongos did not update its user cache after 10 seconds", 10 * 1000); // We manually invalidate the cache on s2/db3. db3.adminCommand("invalidateUserCache"); diff --git a/jstests/core/group9.js b/jstests/core/group9.js new file mode 100644 index 00000000000..a7d9a0d128e --- /dev/null +++ b/jstests/core/group9.js @@ -0,0 +1,20 @@ +(function() { + 'use strict'; + var t = db.group_owned; + t.drop(); + + assert.writeOK(t.insert({_id: 1, subdoc: {id: 1}})); + assert.writeOK(t.insert({_id: 2, subdoc: {id: 2}})); + + var result = t.group({ + key: {'subdoc.id': 1}, + reduce: function(doc, value) { + value.subdoc = doc.subdoc; + return value; + }, + initial: {}, + finalize: function(res) {} + }); + + assert(result.length == 2); +}()); diff --git a/jstests/core/mr4.js b/jstests/core/mr4.js index ae5e11528af..2b8e93c3b35 100644 --- a/jstests/core/mr4.js +++ b/jstests/core/mr4.js @@ -9,7 +9,7 @@ t.save({x: 4, tags: ["b", "c"]}); m = function() { this.tags.forEach(function(z) { - emit(z, {count: xx}); + emit(z, {count: xx.val}); }); }; @@ -23,7 +23,7 @@ r = function(key, values) { }; }; -res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: 1}}); +res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: {val: 1}}}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); @@ -33,7 +33,7 @@ assert.eq(3, z.c.count, "A4"); res.drop(); -res = t.mapReduce(m, r, {scope: {xx: 2}, out: "mr4_out"}); +res = t.mapReduce(m, r, {scope: {xx: {val: 2}}, out: "mr4_out"}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); diff --git a/jstests/gle/gle_sharded_write.js b/jstests/gle/gle_sharded_write.js index f1feffed5b2..8d2a21cd758 100644 --- a/jstests/gle/gle_sharded_write.js +++ b/jstests/gle/gle_sharded_write.js @@ -2,192 +2,192 @@ // Ensures GLE correctly reports basic write stats and failures // Note that test should work correctly with and without write commands. // - -var st = new ShardingTest({shards: 2, mongos: 1}); -st.stopBalancer(); - -var mongos = st.s0; -var admin = mongos.getDB("admin"); -var config = mongos.getDB("config"); -var coll = mongos.getCollection(jsTestName() + ".coll"); -var shards = config.shards.find().toArray(); - -assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().toString()})); -printjson(admin.runCommand({movePrimary: coll.getDB().toString(), to: shards[0]._id})); -assert.commandWorked(admin.runCommand({shardCollection: coll.toString(), key: {_id: 1}})); -assert.commandWorked(admin.runCommand({split: coll.toString(), middle: {_id: 0}})); -assert.commandWorked( - admin.runCommand({moveChunk: coll.toString(), find: {_id: 0}, to: shards[1]._id})); - -st.printShardingStatus(); - -var gle = null; - -// -// Successful insert -coll.remove({}); -coll.insert({_id: -1}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert.eq(coll.count(), 1); - -// -// Successful update -coll.remove({}); -coll.insert({_id: 1}); -coll.update({_id: 1}, {$set: {foo: "bar"}}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(coll.count(), 1); - -// -// Successful multi-update -coll.remove({}); -coll.insert({_id: 1}); -coll.update({}, {$set: {foo: "bar"}}, false, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(coll.count(), 1); - -// -// Successful upsert -coll.remove({}); -coll.update({_id: 1}, {_id: 1}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(!gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(gle.upserted, 1); -assert.eq(coll.count(), 1); - -// -// Successful upserts -coll.remove({}); -coll.update({_id: -1}, {_id: -1}, true); -coll.update({_id: 1}, {_id: 1}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(!gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(gle.upserted, 1); -assert.eq(coll.count(), 2); - -// -// Successful remove -coll.remove({}); -coll.insert({_id: 1}); -coll.remove({_id: 1}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert.eq(gle.n, 1); -assert.eq(coll.count(), 0); - -// -// Error on one host during update -coll.remove({}); -coll.update({_id: 1}, {$invalid: "xxx"}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.singleShard); -assert.eq(coll.count(), 0); - -// -// Error on two hosts during remove -coll.remove({}); -coll.remove({$invalid: 'remove'}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.shards); -assert.eq(coll.count(), 0); - -// -// Repeated calls to GLE should work -coll.remove({}); -coll.update({_id: 1}, {$invalid: "xxx"}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.singleShard); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.singleShard); -assert.eq(coll.count(), 0); - -// -// Geo $near is not supported on mongos -coll.ensureIndex({loc: "2dsphere"}); -coll.remove({}); -var query = { - loc: { - $near: { - $geometry: {type: "Point", coordinates: [0, 0]}, - $maxDistance: 1000, +(function() { + 'use strict'; + + var st = new ShardingTest({shards: 2, mongos: 1}); + + var mongos = st.s0; + var admin = mongos.getDB("admin"); + var config = mongos.getDB("config"); + var coll = mongos.getCollection(jsTestName() + ".coll"); + var shards = config.shards.find().toArray(); + + assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().toString()})); + printjson(admin.runCommand({movePrimary: coll.getDB().toString(), to: shards[0]._id})); + assert.commandWorked(admin.runCommand({shardCollection: coll.toString(), key: {_id: 1}})); + assert.commandWorked(admin.runCommand({split: coll.toString(), middle: {_id: 0}})); + assert.commandWorked( + admin.runCommand({moveChunk: coll.toString(), find: {_id: 0}, to: shards[1]._id})); + + st.printShardingStatus(); + + var gle = null; + + // + // Successful insert + coll.remove({}); + coll.insert({_id: -1}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert.eq(coll.count(), 1); + + // + // Successful update + coll.remove({}); + coll.insert({_id: 1}); + coll.update({_id: 1}, {$set: {foo: "bar"}}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(coll.count(), 1); + + // + // Successful multi-update + coll.remove({}); + coll.insert({_id: 1}); + coll.update({}, {$set: {foo: "bar"}}, false, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(coll.count(), 1); + + // + // Successful upsert + coll.remove({}); + coll.update({_id: 1}, {_id: 1}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(!gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(gle.upserted, 1); + assert.eq(coll.count(), 1); + + // + // Successful upserts + coll.remove({}); + coll.update({_id: -1}, {_id: -1}, true); + coll.update({_id: 1}, {_id: 1}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(!gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(gle.upserted, 1); + assert.eq(coll.count(), 2); + + // + // Successful remove + coll.remove({}); + coll.insert({_id: 1}); + coll.remove({_id: 1}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert.eq(gle.n, 1); + assert.eq(coll.count(), 0); + + // + // Error on one host during update + coll.remove({}); + coll.update({_id: 1}, {$invalid: "xxx"}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.singleShard); + assert.eq(coll.count(), 0); + + // + // Error on two hosts during remove + coll.remove({}); + coll.remove({$invalid: 'remove'}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.shards); + assert.eq(coll.count(), 0); + + // + // Repeated calls to GLE should work + coll.remove({}); + coll.update({_id: 1}, {$invalid: "xxx"}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.singleShard); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.singleShard); + assert.eq(coll.count(), 0); + + // + // Geo $near is not supported on mongos + coll.ensureIndex({loc: "2dsphere"}); + coll.remove({}); + var query = { + loc: { + $near: { + $geometry: {type: "Point", coordinates: [0, 0]}, + $maxDistance: 1000, + } } - } -}; -printjson(coll.remove(query)); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.shards); -assert.eq(coll.count(), 0); - -// -// First shard down -// - -// -// Successful bulk insert on two hosts, host dies before gle (error contacting host) -coll.remove({}); -coll.insert([{_id: 1}, {_id: -1}]); -// Wait for write to be written to shards before shutting it down. -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -MongoRunner.stopMongod(st.shard0); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -// Should get an error about contacting dead host. -assert(!gle.ok); -assert(gle.errmsg); - -// -// Failed insert on two hosts, first host dead -// NOTE: This is DIFFERENT from 2.4, since we don't need to contact a host we didn't get -// successful writes from. -coll.remove({_id: 1}); -coll.insert([{_id: 1}, {_id: -1}]); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert.eq(coll.count({_id: 1}), 1); - -jsTest.log("DONE!"); - -st.stop(); + }; + printjson(coll.remove(query)); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.shards); + assert.eq(coll.count(), 0); + + // + // First shard down + // + + // + // Successful bulk insert on two hosts, host dies before gle (error contacting host) + coll.remove({}); + coll.insert([{_id: 1}, {_id: -1}]); + // Wait for write to be written to shards before shutting it down. + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + MongoRunner.stopMongod(st.shard0); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + // Should get an error about contacting dead host. + assert(!gle.ok); + assert(gle.errmsg); + + // + // Failed insert on two hosts, first host dead + // NOTE: This is DIFFERENT from 2.4, since we don't need to contact a host we didn't get + // successful writes from. + coll.remove({_id: 1}); + coll.insert([{_id: 1}, {_id: -1}]); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert.eq(coll.count({_id: 1}), 1); + + st.stop(); +})(); diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index 673be42d3df..da976c7f4dc 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -50,7 +50,12 @@ ReplSetTest.prototype.upgradeNode = function(node, opts, user, pwd) { var isMaster = node.getDB('admin').runCommand({isMaster: 1}); if (!isMaster.arbiterOnly) { - assert.commandWorked(node.adminCommand("replSetMaintenance")); + // Must retry this command, as it might return "currently running for election" and fail. + // Node might still be running for an election that will fail because it lost the election + // race with another node, at test initialization. See SERVER-23133. + assert.soon(function() { + return (node.adminCommand("replSetMaintenance").ok); + }); this.waitForState(node, ReplSetTest.State.RECOVERING); } diff --git a/jstests/noPassthrough/backup_restore.js b/jstests/noPassthrough/backup_restore.js index 95d111e2cd3..0074c8c1f8d 100644 --- a/jstests/noPassthrough/backup_restore.js +++ b/jstests/noPassthrough/backup_restore.js @@ -226,8 +226,8 @@ rst.start(secondary.nodeId, {}, true); } - // Wait up to 60 seconds until restarted node is in state secondary - rst.waitForState(rst.getSecondaries(), ReplSetTest.State.SECONDARY, 60 * 1000); + // Wait up to 5 minutes until restarted node is in state secondary. + rst.waitForState(rst.getSecondaries(), ReplSetTest.State.SECONDARY); // Add new hidden node to replSetTest var hiddenCfg = { @@ -263,8 +263,7 @@ // Wait up to 60 seconds until the new hidden node is in state RECOVERING. rst.waitForState(rst.nodes[numNodes], - [ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY], - 60 * 1000); + [ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY]); // Stop CRUD client and FSM client. assert(checkProgram(crudPid), testName + ' CRUD client was not running at end of test'); @@ -272,8 +271,8 @@ stopMongoProgramByPid(crudPid); stopMongoProgramByPid(fsmPid); - // Wait up to 60 seconds until the new hidden node is in state SECONDARY. - rst.waitForState(rst.nodes[numNodes], ReplSetTest.State.SECONDARY, 60 * 1000); + // Wait up to 5 minutes until the new hidden node is in state SECONDARY. + rst.waitForState(rst.nodes[numNodes], ReplSetTest.State.SECONDARY); // Wait for secondaries to finish catching up before shutting down. assert.writeOK(primary.getDB("test").foo.insert( diff --git a/jstests/noPassthrough/initial_sync_cloner_dups.js b/jstests/noPassthrough/initial_sync_cloner_dups.js index 1208dc8a16e..c0a3e71c0c1 100644 --- a/jstests/noPassthrough/initial_sync_cloner_dups.js +++ b/jstests/noPassthrough/initial_sync_cloner_dups.js @@ -79,8 +79,7 @@ // Wait for the secondary to get ReplSetInitiate command. replTest.waitForState( secondary, - [ReplSetTest.State.STARTUP_2, ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY], - 60 * 1000); + [ReplSetTest.State.STARTUP_2, ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY]); // This fail point will cause the first intial sync to fail, and leave an op in the buffer to // verify the fix from SERVER-17807 diff --git a/jstests/noPassthroughWithMongod/no_balance_collection.js b/jstests/noPassthroughWithMongod/no_balance_collection.js index cfec6199ca2..1c2f1aae009 100644 --- a/jstests/noPassthroughWithMongod/no_balance_collection.js +++ b/jstests/noPassthroughWithMongod/no_balance_collection.js @@ -1,14 +1,11 @@ // Tests whether the noBalance flag disables balancing for collections -var st = new ShardingTest({shards: 2, mongos: 1, verbose: 1}); +var st = new ShardingTest({shards: 2, mongos: 1}); // First, test that shell helpers require an argument assert.throws(sh.disableBalancing, [], "sh.disableBalancing requires a collection"); assert.throws(sh.enableBalancing, [], "sh.enableBalancing requires a collection"); -// Initially stop balancing -st.stopBalancer(); - var shardAName = st._shardNames[0]; var shardBName = st._shardNames[1]; @@ -70,10 +67,11 @@ jsTest.log("Chunks for " + collB + " are balanced."); // Re-disable balancing for collB sh.disableBalancing(collB); + // Wait for the balancer to fully finish the last migration and write the changelog // MUST set db var here, ugly but necessary db = st.s0.getDB("config"); -sh.waitForBalancer(true); +st.waitForBalancerRound(); // Make sure auto-migrates on insert don't move chunks var lastMigration = sh._lastMigration(collB); diff --git a/jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js b/jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js new file mode 100644 index 00000000000..4ef394f5682 --- /dev/null +++ b/jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js @@ -0,0 +1,8 @@ +(function() { + 'use strict'; + + load("jstests/replsets/libs/apply_ops_insert_write_conflict.js"); + + new ApplyOpsInsertWriteConflictTest( + {testName: 'apply_ops_insert_write_conflict_nonatomic', atomic: false}).run(); +}()); diff --git a/jstests/replsets/election_not_blocked.js b/jstests/replsets/election_not_blocked.js index 95b53be1ebc..88d66715929 100644 --- a/jstests/replsets/election_not_blocked.js +++ b/jstests/replsets/election_not_blocked.js @@ -25,7 +25,7 @@ // so it cannot vote while fsync locked in PV1. Use PV0 explicitly here. protocolVersion: 0 }); - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); // do a write diff --git a/jstests/replsets/initial_sync1.js b/jstests/replsets/initial_sync1.js index 55a454b765c..51d355d80c1 100644 --- a/jstests/replsets/initial_sync1.js +++ b/jstests/replsets/initial_sync1.js @@ -77,9 +77,7 @@ wait(function() { return config2.version == config.version && (config3 && config3.version == config.version); }); -replTest.waitForState(slave2, - [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING], - 60 * 1000); +replTest.waitForState(slave2, [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING]); print("7. Kill the secondary in the middle of syncing"); replTest.stop(slave1); @@ -91,7 +89,7 @@ replTest.waitForState(slave2, ReplSetTest.State.SECONDARY, 60 * 1000); print("9. Bring the secondary back up"); replTest.start(slave1, {}, true); reconnect(slave1); -replTest.waitForState(slave1, [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); +replTest.waitForState(slave1, [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); print("10. Insert some stuff"); master = replTest.getPrimary(); diff --git a/jstests/replsets/initial_sync2.js b/jstests/replsets/initial_sync2.js index bab1063c072..69c91d30f04 100644 --- a/jstests/replsets/initial_sync2.js +++ b/jstests/replsets/initial_sync2.js @@ -25,7 +25,7 @@ var doTest = function() { var conns = replTest.startSet(); replTest.initiate(); - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 5 * 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var foo = master.getDB("foo"); @@ -73,27 +73,27 @@ var doTest = function() { }); admin_s2.runCommand({replSetFreeze: 999999}); - replTest.waitForState( - replTest.nodes[2], [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING], 60 * 1000); + replTest.waitForState(replTest.nodes[2], + [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING]); jsTest.log("7. Kill #1 in the middle of syncing"); replTest.stop(0); jsTest.log("8. Check that #3 makes it into secondary state"); - replTest.waitForState( - replTest.nodes[2], [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); + replTest.waitForState(replTest.nodes[2], + [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); jsTest.log("9. Bring #1 back up"); replTest.start(0, {}, true); - replTest.waitForState( - replTest.nodes[0], [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); + replTest.waitForState(replTest.nodes[0], + [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); jsTest.log("10. Initial sync should succeed"); - replTest.waitForState( - replTest.nodes[2], [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); + replTest.waitForState(replTest.nodes[2], + [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); jsTest.log("11. Ensure #1 becomes primary"); - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); jsTest.log("12. Everyone happy eventually"); replTest.awaitReplication(2 * 60 * 1000); diff --git a/jstests/replsets/libs/apply_ops_insert_write_conflict.js b/jstests/replsets/libs/apply_ops_insert_write_conflict.js new file mode 100644 index 00000000000..9bbdea7e08e --- /dev/null +++ b/jstests/replsets/libs/apply_ops_insert_write_conflict.js @@ -0,0 +1,88 @@ +/** + * Sets up a test for WriteConflictException handling in applyOps with an insert workload. + */ +var ApplyOpsInsertWriteConflictTest = function(options) { + 'use strict'; + + if (!(this instanceof ApplyOpsInsertWriteConflictTest)) { + return new ApplyOpsInsertWriteConflictTest(options); + } + + // Capture the 'this' reference + var self = this; + + self.options = options; + + /** + * Runs the test. + */ + this.run = function() { + var options = this.options; + + var replTest = new ReplSetTest({nodes: 1}); + replTest.startSet(); + replTest.initiate(); + + var primary = replTest.getPrimary(); + var primaryDB = primary.getDB('test'); + + var t = primaryDB.getCollection(options.testName); + t.drop(); + + assert.commandWorked(primaryDB.createCollection(t.getName())); + + var numOps = 1000; + var ops = Array(numOps).fill('ignored').map((unused, i) => { + return { + op: 'i', + ns: t.getFullName(), + o: {_id: i} + }; + }); + + if (!options.atomic) { + // Adding a command to the list of operations to prevent the applyOps command from + // applying + // all the operations atomically. + ops.push({ns: "test.$cmd", op: "c", o: {applyOps: []}}); + numOps++; + } + + // Probabilities for WCE are chosen based on empirical testing. + // The probability for WCE during an atomic applyOps should be much smaller than that for + // the non-atomic case because we have to attempt to re-apply the entire batch of 'numOps' + // operations on WCE in the atomic case. + var probability = (options.atomic ? 0.1 : 5.0) / numOps; + + // Set up failpoint to trigger WriteConflictException during write operations. + assert.commandWorked( + primaryDB.adminCommand({setParameter: 1, traceWriteConflictExceptions: true})); + assert.commandWorked(primaryDB.adminCommand({ + configureFailPoint: 'WTWriteConflictException', + mode: {activationProbability: probability} + })); + + // This logs each operation being applied. + var previousLogLevel = + assert.commandWorked(primaryDB.setLogLevel(3, 'replication')).was.replication.verbosity; + + var applyOpsResult = primaryDB.adminCommand({applyOps: ops}); + + // Reset log level. + primaryDB.setLogLevel(previousLogLevel, 'replication'); + + assert.eq( + numOps, + applyOpsResult.applied, + 'number of operations applied did not match list of generated insert operations. ' + + 'applyOps result: ' + tojson(applyOpsResult)); + applyOpsResult.results.forEach((operationSucceeded, i) => { + assert(operationSucceeded, + 'applyOps failed: operation with index ' + i + ' failed: operation: ' + + tojson(ops[i], '', true) + '. applyOps result: ' + tojson(applyOpsResult)); + }); + assert.commandWorked(applyOpsResult); + + replTest.stopSet(); + }; +}; diff --git a/jstests/replsets/maintenance.js b/jstests/replsets/maintenance.js index b1fe94efc0e..8b4765212b8 100644 --- a/jstests/replsets/maintenance.js +++ b/jstests/replsets/maintenance.js @@ -5,7 +5,7 @@ var conns = replTest.startSet({verbose: 1}); var config = replTest.getReplSetConfig(); config.members[0].priority = 2; replTest.initiate(config); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // Make sure we have a master var master = replTest.getPrimary(); diff --git a/jstests/replsets/oplog_truncated_on_recovery.js b/jstests/replsets/oplog_truncated_on_recovery.js index 4d469178691..477da7b4d92 100644 --- a/jstests/replsets/oplog_truncated_on_recovery.js +++ b/jstests/replsets/oplog_truncated_on_recovery.js @@ -74,7 +74,7 @@ log(assert.commandWorked(localDB.adminCommand("replSetGetStatus"))); log("restart primary"); replTest.restart(master); - replTest.waitForState(master, ReplSetTest.State.RECOVERING, 90000); + replTest.waitForState(master, ReplSetTest.State.RECOVERING); assert.soon(function() { var mv; diff --git a/jstests/replsets/priority_takeover_one_node_higher_priority.js b/jstests/replsets/priority_takeover_one_node_higher_priority.js index 81f7717a0ee..de20f71c854 100644 --- a/jstests/replsets/priority_takeover_one_node_higher_priority.js +++ b/jstests/replsets/priority_takeover_one_node_higher_priority.js @@ -13,7 +13,7 @@ replSet.startSet(); replSet.initiate(); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var primary = replSet.getPrimary(); replSet.awaitSecondaryNodes(); @@ -21,19 +21,19 @@ // Primary should step down long enough for election to occur on secondary. var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - var electionTimeoutMillis = config.settings.electionTimeoutMillis; - var stepDownGuardMillis = electionTimeoutMillis * 2; var stepDownException = assert.throws(function() { - primary.adminCommand({replSetStepDown: stepDownGuardMillis / 1000}); + primary.adminCommand({replSetStepDown: replSet.kDefaultTimeoutMS / 1000}); }); assert.neq(-1, tojson(stepDownException).indexOf('error doing query'), 'replSetStepDown did not disconnect client'); // Step down primary and wait for node 1 to be promoted to primary. - replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY); + + // Unfreeze node 0 so it can seek election. + assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); // Eventually node 0 will stand for election again because it has a higher priorty. - replSet.waitForState( - replSet.nodes[0], ReplSetTest.State.PRIMARY, stepDownGuardMillis + 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); })(); diff --git a/jstests/replsets/read_committed_with_catalog_changes.js b/jstests/replsets/read_committed_with_catalog_changes.js index dae14da31de..4290391bd73 100644 --- a/jstests/replsets/read_committed_with_catalog_changes.js +++ b/jstests/replsets/read_committed_with_catalog_changes.js @@ -208,13 +208,13 @@ load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. // they may be passed in to a ScopedThread. function assertReadsBlock(coll) { var res = - coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": 1000}); + coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": 5000}); assert.commandFailedWithCode(res, ErrorCodes.ExceededTimeLimit, "Expected read of " + coll.getFullName() + " to block"); } - function assertReadsSucceed(coll, timeoutMs = 1000) { + function assertReadsSucceed(coll, timeoutMs = 20000) { var res = coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": timeoutMs}); assert.commandWorked(res, 'reading from ' + coll.getFullName()); diff --git a/jstests/replsets/replsetadd_profile.js b/jstests/replsets/replsetadd_profile.js index 641e7ca7cfd..2e396a61eb7 100644 --- a/jstests/replsets/replsetadd_profile.js +++ b/jstests/replsets/replsetadd_profile.js @@ -19,7 +19,7 @@ masterCollection.save({a: 1}); var newNode = replTest.add(); replTest.reInitiate(); -replTest.waitForState(replTest.nodes[1], ReplSetTest.State.SECONDARY, 60 * 1000); +replTest.waitForState(replTest.nodes[1], ReplSetTest.State.SECONDARY); // Allow documents to propagate to new replica set member. replTest.awaitReplication(); diff --git a/jstests/replsets/replsetprio1.js b/jstests/replsets/replsetprio1.js index 16beb851b81..5aee0a33a92 100644 --- a/jstests/replsets/replsetprio1.js +++ b/jstests/replsets/replsetprio1.js @@ -16,10 +16,10 @@ }); // 2 should be master (give this a while to happen, as other nodes might first be elected) - replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY, 120000); + replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY); // wait for 1 to not appear to be master (we are about to make it master and need a clean slate // here) - replTest.waitForState(nodes[1], ReplSetTest.State.SECONDARY, 60000); + replTest.waitForState(nodes[1], ReplSetTest.State.SECONDARY); // Wait for election oplog entry to be replicated, to ensure 0 will vote for 1 after stopping 2. replTest.awaitReplication(); @@ -28,7 +28,7 @@ replTest.stop(2); // 1 should eventually be master - replTest.waitForState(nodes[1], ReplSetTest.State.PRIMARY, 60000); + replTest.waitForState(nodes[1], ReplSetTest.State.PRIMARY); // do some writes on 1 var master = replTest.getPrimary(); @@ -42,7 +42,7 @@ // bring 2 back up, 2 should wait until caught up and then become master replTest.restart(2); - replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY, 60000); + replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY); // make sure nothing was rolled back master = replTest.getPrimary(); diff --git a/jstests/replsets/request_primary_stepdown.js b/jstests/replsets/request_primary_stepdown.js index 02050bd55f4..3e56946397b 100644 --- a/jstests/replsets/request_primary_stepdown.js +++ b/jstests/replsets/request_primary_stepdown.js @@ -15,7 +15,7 @@ conf.protocolVersion = 0; replSet.initiate(conf); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); replSet.awaitSecondaryNodes(); replSet.awaitReplication(); var primary = replSet.getPrimary(); diff --git a/jstests/replsets/resync_with_write_load.js b/jstests/replsets/resync_with_write_load.js index 1a782ffacbe..13b6042ab58 100644 --- a/jstests/replsets/resync_with_write_load.js +++ b/jstests/replsets/resync_with_write_load.js @@ -19,7 +19,7 @@ var config = { ] }; var r = replTest.initiate(config); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // Make sure we have a master var master = replTest.getPrimary(); var a_conn = conns[0]; diff --git a/jstests/replsets/rollback.js b/jstests/replsets/rollback.js index 56aaed37d6f..6baeb88666a 100644 --- a/jstests/replsets/rollback.js +++ b/jstests/replsets/rollback.js @@ -46,7 +46,7 @@ load("jstests/replsets/rslib.js"); }); // Make sure we have a master - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; var A = a_conn.getDB("admin"); diff --git a/jstests/replsets/rollback2.js b/jstests/replsets/rollback2.js index cd4f5a049b7..1cbf8194880 100644 --- a/jstests/replsets/rollback2.js +++ b/jstests/replsets/rollback2.js @@ -42,7 +42,7 @@ load("jstests/replsets/rslib.js"); }); // Make sure we have a master and that that master is node A - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; a_conn.setSlaveOk(); diff --git a/jstests/replsets/rollback3.js b/jstests/replsets/rollback3.js index 59b685c7bad..5eb9ba574d1 100644 --- a/jstests/replsets/rollback3.js +++ b/jstests/replsets/rollback3.js @@ -47,7 +47,7 @@ load("jstests/replsets/rslib.js"); }); // Make sure we have a master and that that master is node A - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; a_conn.setSlaveOk(); diff --git a/jstests/replsets/rollback5.js b/jstests/replsets/rollback5.js index cc0007822dd..81614447607 100644 --- a/jstests/replsets/rollback5.js +++ b/jstests/replsets/rollback5.js @@ -23,7 +23,7 @@ var r = replTest.initiate({ }); // Make sure we have a master -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; var b_conn = conns[1]; diff --git a/jstests/replsets/rollback_auth.js b/jstests/replsets/rollback_auth.js index 0c0b35b91ed..4b31a0e527e 100644 --- a/jstests/replsets/rollback_auth.js +++ b/jstests/replsets/rollback_auth.js @@ -39,7 +39,7 @@ }); // Make sure we have a master - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; var b_conn = conns[1]; diff --git a/jstests/replsets/rollback_cmd_unrollbackable.js b/jstests/replsets/rollback_cmd_unrollbackable.js index 41b8f77f74f..01c0bc5bf38 100644 --- a/jstests/replsets/rollback_cmd_unrollbackable.js +++ b/jstests/replsets/rollback_cmd_unrollbackable.js @@ -26,7 +26,7 @@ var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); // get master and do an initial write -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); assert(master === conns[0], "conns[0] assumed to be master"); assert(a_conn.host === master.host, "a_conn assumed to be master"); diff --git a/jstests/replsets/rollback_collMod_fatal.js b/jstests/replsets/rollback_collMod_fatal.js index 61af67b15d0..76a43c8cd4c 100644 --- a/jstests/replsets/rollback_collMod_fatal.js +++ b/jstests/replsets/rollback_collMod_fatal.js @@ -25,7 +25,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_different_h.js b/jstests/replsets/rollback_different_h.js index 4b9aede1bbc..37f6f0a71cd 100644 --- a/jstests/replsets/rollback_different_h.js +++ b/jstests/replsets/rollback_different_h.js @@ -36,7 +36,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_dropdb.js b/jstests/replsets/rollback_dropdb.js index b8f7d8d09ee..5853f7c47ce 100644 --- a/jstests/replsets/rollback_dropdb.js +++ b/jstests/replsets/rollback_dropdb.js @@ -25,7 +25,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_fake_cmd.js b/jstests/replsets/rollback_fake_cmd.js index 6eba30d2c16..fcc5fbaf39b 100644 --- a/jstests/replsets/rollback_fake_cmd.js +++ b/jstests/replsets/rollback_fake_cmd.js @@ -36,7 +36,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_index.js b/jstests/replsets/rollback_index.js index d3bf747680c..66b2be66f63 100644 --- a/jstests/replsets/rollback_index.js +++ b/jstests/replsets/rollback_index.js @@ -38,7 +38,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js index b97f79c00d1..3b41531d760 100644 --- a/jstests/replsets/rslib.js +++ b/jstests/replsets/rslib.js @@ -138,7 +138,7 @@ var startSetIfSupportsReadMajority; } printjson(state); return true; - }, "not all members ready", timeout || 60000); + }, "not all members ready", timeout || 10 * 60 * 1000); print("All members are now in state PRIMARY, SECONDARY, or ARBITER"); }; diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js index 5a8388da4d4..bed42cff3e5 100644 --- a/jstests/replsets/stepdown.js +++ b/jstests/replsets/stepdown.js @@ -20,7 +20,7 @@ var replTest = new ReplSetTest({ }); var nodes = replTest.startSet(); replTest.initiate(); -replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); // do a write diff --git a/jstests/replsets/stepdown_kill_other_ops.js b/jstests/replsets/stepdown_kill_other_ops.js index 930206046c1..d360624bc7a 100644 --- a/jstests/replsets/stepdown_kill_other_ops.js +++ b/jstests/replsets/stepdown_kill_other_ops.js @@ -15,7 +15,7 @@ ] }); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var primary = replSet.getPrimary(); assert.eq(primary.host, nodes[0], "primary assumed to be node 0"); diff --git a/jstests/replsets/stepdown_killop.js b/jstests/replsets/stepdown_killop.js index 5c0e0ffae91..1b86d5fb8d4 100644 --- a/jstests/replsets/stepdown_killop.js +++ b/jstests/replsets/stepdown_killop.js @@ -23,7 +23,7 @@ ] }); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var secondary = replSet.getSecondary(); jsTestLog('Disable replication on the SECONDARY ' + secondary.host); diff --git a/jstests/replsets/stepdown_long_wait_time.js b/jstests/replsets/stepdown_long_wait_time.js index 60e0fdb4247..eb1bf7007d9 100644 --- a/jstests/replsets/stepdown_long_wait_time.js +++ b/jstests/replsets/stepdown_long_wait_time.js @@ -22,7 +22,7 @@ ] }); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var primary = replSet.getPrimary(); var secondary = replSet.getSecondary(); diff --git a/jstests/replsets/sync_passive.js b/jstests/replsets/sync_passive.js index 4899385563f..c0be375b98b 100644 --- a/jstests/replsets/sync_passive.js +++ b/jstests/replsets/sync_passive.js @@ -29,7 +29,7 @@ config.members[0].priority = 2; config.members[2].priority = 0; replTest.initiate(config); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary().getDB("test"); var server0 = master; diff --git a/jstests/replsets/two_nodes_priority_take_over.js b/jstests/replsets/two_nodes_priority_take_over.js index f6e62fe681d..ae0d53e92b6 100644 --- a/jstests/replsets/two_nodes_priority_take_over.js +++ b/jstests/replsets/two_nodes_priority_take_over.js @@ -29,7 +29,7 @@ if (false) { }); // The first node will be the primary at the beginning. - rst.waitForState(rst.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + rst.waitForState(rst.nodes[0], ReplSetTest.State.PRIMARY); // Get the term when replset is stable. var res = rst.getPrimary().adminCommand("replSetGetStatus"); diff --git a/jstests/sharding/balance_repl.js b/jstests/sharding/balance_repl.js index 46404646995..59cc694fc42 100644 --- a/jstests/sharding/balance_repl.js +++ b/jstests/sharding/balance_repl.js @@ -3,7 +3,7 @@ // (function() { - "use strict"; + 'use strict'; // The mongod secondaries are set to priority 0 and votes 0 to prevent the primaries // from stepping down during migrations on slow evergreen builders. @@ -27,14 +27,16 @@ } assert.writeOK(bulk.execute()); - s.adminCommand({enablesharding: "test"}); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'test-rs0'); - s.adminCommand({shardcollection: "test.foo", key: {_id: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo", key: {_id: 1}})); - for (i = 0; i < 20; i++) - s.adminCommand({split: "test.foo", middle: {_id: i * 100}}); + for (i = 0; i < 20; i++) { + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {_id: i * 100}})); + } assert.eq(2100, db.foo.find().itcount()); + var coll = db.foo; coll.setSlaveOk(); @@ -42,10 +44,9 @@ var other = s.config.shards.findOne({_id: {$ne: serverName}}); for (i = 0; i < 20; i++) { - // Needs to waitForDelete because we'll be performing a slaveOk query, - // and secondaries don't have a chunk manager so it doesn't know how to - // filter out docs it doesn't own. - assert(s.adminCommand({ + // Needs to waitForDelete because we'll be performing a slaveOk query, and secondaries don't + // have a chunk manager so it doesn't know how to filter out docs it doesn't own. + assert.commandWorked(s.s0.adminCommand({ moveChunk: "test.foo", find: {_id: i * 100}, to: other._id, @@ -53,9 +54,9 @@ writeConcern: {w: 2}, _waitForDelete: true })); + assert.eq(2100, coll.find().itcount()); } s.stop(); - }()); diff --git a/jstests/sharding/balance_tags2.js b/jstests/sharding/balance_tags2.js index e4bf370d1cd..a7f1161d6dc 100644 --- a/jstests/sharding/balance_tags2.js +++ b/jstests/sharding/balance_tags2.js @@ -1,27 +1,26 @@ // Test balancing all chunks to one shard by tagging the full shard-key range on that collection -var s = new ShardingTest( - {name: "balance_tags2", shards: 3, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); +var s = new ShardingTest({shards: 3, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); -s.adminCommand({enablesharding: "test"}); +assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); var db = s.getDB("test"); var bulk = db.foo.initializeUnorderedBulkOp(); -for (i = 0; i < 21; i++) { +for (var i = 0; i < 21; i++) { bulk.insert({_id: i, x: i}); } assert.writeOK(bulk.execute()); -sh.shardCollection("test.foo", {_id: 1}); +assert.commandWorked(s.s0.adminCommand({shardCollection: "test.foo", key: {_id: 1}})); -sh.stopBalancer(); +s.stopBalancer(); -for (i = 0; i < 20; i++) { +for (var i = 0; i < 20; i++) { sh.splitAt("test.foo", {_id: i}); } -sh.startBalancer(); +s.startBalancer(); sh.status(true); diff --git a/jstests/sharding/explain_cmd.js b/jstests/sharding/explain_cmd.js index 3b8a8ef1240..b4ec0db35e9 100644 --- a/jstests/sharding/explain_cmd.js +++ b/jstests/sharding/explain_cmd.js @@ -1,174 +1,185 @@ // Tests for the mongos explain command. - -// Create a cluster with 3 shards. -var st = new ShardingTest({shards: 2}); -st.stopBalancer(); - -var db = st.s.getDB("test"); -var explain; - -// Setup a collection that will be sharded. The shard key will be 'a'. There's also an index on 'b'. -var collSharded = db.getCollection("mongos_explain_cmd"); -collSharded.drop(); -collSharded.ensureIndex({a: 1}); -collSharded.ensureIndex({b: 1}); - -// Enable sharding. -assert.commandWorked(db.adminCommand({enableSharding: db.getName()})); -st.ensurePrimaryShard(db.getName(), 'shard0001'); -db.adminCommand({shardCollection: collSharded.getFullName(), key: {a: 1}}); - -// Pre-split the collection to ensure that both shards have chunks. Explicitly -// move chunks since the balancer is disabled. -for (var i = 1; i <= 2; i++) { - assert.commandWorked(db.adminCommand({split: collSharded.getFullName(), middle: {a: i}})); - - var shardName = "shard000" + (i - 1); - printjson(db.adminCommand({moveChunk: collSharded.getFullName(), find: {a: i}, to: shardName})); -} - -// Put data on each shard. -for (var i = 0; i < 3; i++) { - collSharded.insert({_id: i, a: i, b: 1}); -} - -printjson(sh.status()); - -// Test a scatter-gather count command. -assert.eq(3, collSharded.count({b: 1})); - -// Explain the scatter-gather count. -explain = db.runCommand( - {explain: {count: collSharded.getName(), query: {b: 1}}, verbosity: "allPlansExecution"}); - -// Validate some basic properties of the result. -printjson(explain); -assert.commandWorked(explain); -assert("queryPlanner" in explain); -assert("executionStats" in explain); -assert.eq(2, explain.queryPlanner.winningPlan.shards.length); -assert.eq(2, explain.executionStats.executionStages.shards.length); - -// An explain of a command that doesn't exist should fail gracefully. -explain = db.runCommand({ - explain: {nonexistent: collSharded.getName(), query: {b: 1}}, - verbosity: "allPlansExecution" -}); -printjson(explain); -assert.commandFailed(explain); - -// ------- - -// Setup a collection that is not sharded. -var collUnsharded = db.getCollection("mongos_explain_cmd_unsharded"); -collUnsharded.drop(); -collUnsharded.ensureIndex({a: 1}); -collUnsharded.ensureIndex({b: 1}); - -for (var i = 0; i < 3; i++) { - collUnsharded.insert({_id: i, a: i, b: 1}); -} -assert.eq(3, collUnsharded.count({b: 1})); - -explain = db.runCommand({ - explain: { - group: { - ns: collUnsharded.getName(), - key: "a", - cond: "b", - $reduce: function(curr, result) {}, - initial: {} - } - }, - verbosity: "allPlansExecution" -}); - -// Basic validation: a group command can only be passed through to an unsharded collection, -// so we should confirm that the mongos stage is always SINGLE_SHARD. -printjson(explain); -assert.commandWorked(explain); -assert("queryPlanner" in explain); -assert("executionStats" in explain); -assert.eq("SINGLE_SHARD", explain.queryPlanner.winningPlan.stage); - -// The same group should fail over the sharded collection, because group is only supported -// if it is passed through to an unsharded collection. -explain = db.runCommand({ - explain: { - group: { - ns: collSharded.getName(), - key: "a", - cond: "b", - $reduce: function(curr, result) {}, - initial: {} - } - }, - verbosity: "allPlansExecution" -}); -printjson(explain); -assert.commandFailed(explain); - -// ------- - -// Explain a delete operation and verify that it hits all shards without the shard key -explain = db.runCommand({ - explain: {delete: collSharded.getName(), deletes: [{q: {b: 1}, limit: 0}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); -assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "DELETE"); -assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "DELETE"); -// Check that the deletes didn't actually happen. -assert.eq(3, collSharded.count({b: 1})); - -// Explain a delete operation and verify that it hits only one shard with the shard key -explain = db.runCommand({ - explain: {delete: collSharded.getName(), deletes: [{q: {a: 1}, limit: 0}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); -// Check that the deletes didn't actually happen. -assert.eq(3, collSharded.count({b: 1})); - -// Check that we fail gracefully if we try to do an explain of a write batch that has more -// than one operation in it. -explain = db.runCommand({ - explain: - {delete: collSharded.getName(), deletes: [{q: {a: 1}, limit: 1}, {q: {a: 2}, limit: 1}]}, - verbosity: "allPlansExecution" -}); -assert.commandFailed(explain, tojson(explain)); - -// Explain a multi upsert operation and verify that it hits all shards -explain = db.runCommand({ - explain: {update: collSharded.getName(), updates: [{q: {}, u: {$set: {b: 10}}, multi: true}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); -assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); -assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "UPDATE"); -assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "UPDATE"); -// Check that the update didn't actually happen. -assert.eq(0, collSharded.count({b: 10})); - -// Explain an upsert operation and verify that it hits only a single shard -explain = db.runCommand({ - explain: {update: collSharded.getName(), updates: [{q: {a: 10}, u: {a: 10}, upsert: true}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); -// Check that the upsert didn't actually happen. -assert.eq(0, collSharded.count({a: 10})); - -// Explain an upsert operation which cannot be targeted, ensure an error is thrown -explain = db.runCommand({ - explain: {update: collSharded.getName(), updates: [{q: {b: 10}, u: {b: 10}, upsert: true}]}, - verbosity: "allPlansExecution" -}); -assert.commandFailed(explain, tojson(explain)); +(function() { + 'use strict'; + + // Create a cluster with 3 shards. + var st = new ShardingTest({shards: 2}); + + var db = st.s.getDB("test"); + var explain; + + // Setup a collection that will be sharded. The shard key will be 'a'. There's also an index on + // 'b'. + var collSharded = db.getCollection("mongos_explain_cmd"); + collSharded.drop(); + collSharded.ensureIndex({a: 1}); + collSharded.ensureIndex({b: 1}); + + // Enable sharding. + assert.commandWorked(db.adminCommand({enableSharding: db.getName()})); + st.ensurePrimaryShard(db.getName(), 'shard0001'); + db.adminCommand({shardCollection: collSharded.getFullName(), key: {a: 1}}); + + // Pre-split the collection to ensure that both shards have chunks. Explicitly + // move chunks since the balancer is disabled. + for (var i = 1; i <= 2; i++) { + assert.commandWorked(db.adminCommand({split: collSharded.getFullName(), middle: {a: i}})); + + var shardName = "shard000" + (i - 1); + printjson( + db.adminCommand({moveChunk: collSharded.getFullName(), find: {a: i}, to: shardName})); + } + + // Put data on each shard. + for (var i = 0; i < 3; i++) { + collSharded.insert({_id: i, a: i, b: 1}); + } + + st.printShardingStatus(); + + // Test a scatter-gather count command. + assert.eq(3, collSharded.count({b: 1})); + + // Explain the scatter-gather count. + explain = db.runCommand( + {explain: {count: collSharded.getName(), query: {b: 1}}, verbosity: "allPlansExecution"}); + + // Validate some basic properties of the result. + printjson(explain); + assert.commandWorked(explain); + assert("queryPlanner" in explain); + assert("executionStats" in explain); + assert.eq(2, explain.queryPlanner.winningPlan.shards.length); + assert.eq(2, explain.executionStats.executionStages.shards.length); + + // An explain of a command that doesn't exist should fail gracefully. + explain = db.runCommand({ + explain: {nonexistent: collSharded.getName(), query: {b: 1}}, + verbosity: "allPlansExecution" + }); + printjson(explain); + assert.commandFailed(explain); + + // ------- + + // Setup a collection that is not sharded. + var collUnsharded = db.getCollection("mongos_explain_cmd_unsharded"); + collUnsharded.drop(); + collUnsharded.ensureIndex({a: 1}); + collUnsharded.ensureIndex({b: 1}); + + for (var i = 0; i < 3; i++) { + collUnsharded.insert({_id: i, a: i, b: 1}); + } + assert.eq(3, collUnsharded.count({b: 1})); + + explain = db.runCommand({ + explain: { + group: { + ns: collUnsharded.getName(), + key: "a", + cond: "b", + $reduce: function(curr, result) {}, + initial: {} + } + }, + verbosity: "allPlansExecution" + }); + + // Basic validation: a group command can only be passed through to an unsharded collection, + // so we should confirm that the mongos stage is always SINGLE_SHARD. + printjson(explain); + assert.commandWorked(explain); + assert("queryPlanner" in explain); + assert("executionStats" in explain); + assert.eq("SINGLE_SHARD", explain.queryPlanner.winningPlan.stage); + + // The same group should fail over the sharded collection, because group is only supported + // if it is passed through to an unsharded collection. + explain = db.runCommand({ + explain: { + group: { + ns: collSharded.getName(), + key: "a", + cond: "b", + $reduce: function(curr, result) {}, + initial: {} + } + }, + verbosity: "allPlansExecution" + }); + printjson(explain); + assert.commandFailed(explain); + + // ------- + + // Explain a delete operation and verify that it hits all shards without the shard key + explain = db.runCommand({ + explain: {delete: collSharded.getName(), deletes: [{q: {b: 1}, limit: 0}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); + assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "DELETE"); + assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "DELETE"); + // Check that the deletes didn't actually happen. + assert.eq(3, collSharded.count({b: 1})); + + // Explain a delete operation and verify that it hits only one shard with the shard key + explain = db.runCommand({ + explain: {delete: collSharded.getName(), deletes: [{q: {a: 1}, limit: 0}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); + // Check that the deletes didn't actually happen. + assert.eq(3, collSharded.count({b: 1})); + + // Check that we fail gracefully if we try to do an explain of a write batch that has more + // than one operation in it. + explain = db.runCommand({ + explain: { + delete: collSharded.getName(), + deletes: [{q: {a: 1}, limit: 1}, {q: {a: 2}, limit: 1}] + }, + verbosity: "allPlansExecution" + }); + assert.commandFailed(explain, tojson(explain)); + + // Explain a multi upsert operation and verify that it hits all shards + explain = db.runCommand({ + explain: + {update: collSharded.getName(), updates: [{q: {}, u: {$set: {b: 10}}, multi: true}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); + assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); + assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "UPDATE"); + assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "UPDATE"); + // Check that the update didn't actually happen. + assert.eq(0, collSharded.count({b: 10})); + + // Explain an upsert operation and verify that it hits only a single shard + explain = db.runCommand({ + explain: + {update: collSharded.getName(), updates: [{q: {a: 10}, u: {a: 10}, upsert: true}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); + // Check that the upsert didn't actually happen. + assert.eq(0, collSharded.count({a: 10})); + + // Explain an upsert operation which cannot be targeted, ensure an error is thrown + explain = db.runCommand({ + explain: + {update: collSharded.getName(), updates: [{q: {b: 10}, u: {b: 10}, upsert: true}]}, + verbosity: "allPlansExecution" + }); + assert.commandFailed(explain, tojson(explain)); + + st.stop(); +})(); diff --git a/jstests/sharding/explain_find_and_modify_sharded.js b/jstests/sharding/explain_find_and_modify_sharded.js index 40af14f6265..e8c69adc222 100644 --- a/jstests/sharding/explain_find_and_modify_sharded.js +++ b/jstests/sharding/explain_find_and_modify_sharded.js @@ -9,7 +9,6 @@ // Create a cluster with 2 shards. var st = new ShardingTest({shards: 2}); - st.stopBalancer(); var testDB = st.s.getDB('test'); var shardKey = { @@ -85,4 +84,5 @@ assert.commandWorked(res); assertExplainResult(res, 'executionStats', 'executionStages', 'shard0001', 'DELETE'); + st.stop(); })(); diff --git a/jstests/sharding/hash_shard_unique_compound.js b/jstests/sharding/hash_shard_unique_compound.js index 5d6d466c1f9..abaf45260b9 100644 --- a/jstests/sharding/hash_shard_unique_compound.js +++ b/jstests/sharding/hash_shard_unique_compound.js @@ -2,44 +2,42 @@ // Does 2 things and checks for consistent error: // 1.) shard collection on hashed "a", ensure unique index {a:1, b:1} // 2.) reverse order +(function() { + 'use strict'; -var s = new ShardingTest({name: jsTestName(), shards: 1, mongos: 1, verbose: 1}); -var dbName = "test"; -var collName = "foo"; -var ns = dbName + "." + collName; -var db = s.getDB(dbName); -var coll = db.getCollection(collName); + var s = new ShardingTest({shards: 1, mongos: 1}); + var dbName = "test"; + var collName = "foo"; + var ns = dbName + "." + collName; + var db = s.getDB(dbName); + var coll = db.getCollection(collName); -// Enable sharding on DB -var res = db.adminCommand({enablesharding: dbName}); + // Enable sharding on DB + assert.commandWorked(db.adminCommand({enablesharding: dbName})); -// for simplicity start by turning off balancer -var res = s.stopBalancer(); + // Shard a fresh collection using a hashed shard key + assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}})); -// shard a fresh collection using a hashed shard key -coll.drop(); -assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}})); -db.printShardingStatus(); + // Create unique index + assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); -// Create unique index -assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); + jsTest.log("------ indexes -------"); + jsTest.log(tojson(coll.getIndexes())); -jsTest.log("------ indexes -------"); -jsTest.log(tojson(coll.getIndexes())); + // Second Part + jsTest.log("------ dropping sharded collection to start part 2 -------"); + coll.drop(); -// Second Part -jsTest.log("------ dropping sharded collection to start part 2 -------"); -coll.drop(); + // Create unique index + assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); -// Create unique index -assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); + // shard a fresh collection using a hashed shard key + assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}}), + "shardcollection didn't worked 2"); -// shard a fresh collection using a hashed shard key -assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}}), - "shardcollection didn't worked 2"); + s.printShardingStatus(); + jsTest.log("------ indexes 2-------"); + jsTest.log(tojson(coll.getIndexes())); -db.printShardingStatus(); -jsTest.log("------ indexes 2-------"); -jsTest.log(tojson(coll.getIndexes())); - -s.stop(); + s.stop(); +})(); diff --git a/jstests/sharding/mapReduce_inSharded_outSharded.js b/jstests/sharding/mapReduce_inSharded_outSharded.js index d1aba2599f0..5190a1fe4ba 100644 --- a/jstests/sharding/mapReduce_inSharded_outSharded.js +++ b/jstests/sharding/mapReduce_inSharded_outSharded.js @@ -1,60 +1,70 @@ -var verifyOutput = function(out) { - printjson(out); - assert.eq(out.counts.input, 51200, "input count is wrong"); - assert.eq(out.counts.emit, 51200, "emit count is wrong"); - assert.gt(out.counts.reduce, 99, "reduce count is wrong"); - assert.eq(out.counts.output, 512, "output count is wrong"); -}; - -var st = new ShardingTest( - {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); - -st.adminCommand({enablesharding: "mrShard"}); -st.ensurePrimaryShard('mrShard', 'shard0001'); -st.adminCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}}); - -var db = st.getDB("mrShard"); - -var bulk = db.srcSharded.initializeUnorderedBulkOp(); -for (j = 0; j < 100; j++) { - for (i = 0; i < 512; i++) { - bulk.insert({j: j, i: i}); +(function() { + "use strict"; + + var verifyOutput = function(out) { + printjson(out); + assert.eq(out.counts.input, 51200, "input count is wrong"); + assert.eq(out.counts.emit, 51200, "emit count is wrong"); + assert.gt(out.counts.reduce, 99, "reduce count is wrong"); + assert.eq(out.counts.output, 512, "output count is wrong"); + }; + + var st = new ShardingTest( + {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); + + var admin = st.s0.getDB('admin'); + + assert.commandWorked(admin.runCommand({enablesharding: "mrShard"})); + st.ensurePrimaryShard('mrShard', 'shard0001'); + assert.commandWorked( + admin.runCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}})); + + var db = st.s0.getDB("mrShard"); + + var bulk = db.srcSharded.initializeUnorderedBulkOp(); + for (var j = 0; j < 100; j++) { + for (var i = 0; i < 512; i++) { + bulk.insert({j: j, i: i}); + } + } + assert.writeOK(bulk.execute()); + + function map() { + emit(this.i, 1); + } + function reduce(key, values) { + return Array.sum(values); } -} -assert.writeOK(bulk.execute()); - -function map() { - emit(this.i, 1); -} -function reduce(key, values) { - return Array.sum(values); -} - -// sharded src sharded dst -var suffix = "InShardedOutSharded"; - -var out = - db.srcSharded.mapReduce(map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); -verifyOutput(out); -assert(out.results != 'undefined', "no results for inline"); - -out = db.srcSharded.mapReduce( - map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); -verifyOutput(out); - -out = db.runCommand({ - mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce - map: map, - reduce: reduce, - out: "mrBasic" + "srcSharded", -}); -verifyOutput(out); + + // sharded src sharded dst + var suffix = "InShardedOutSharded"; + + var out = db.srcSharded.mapReduce( + map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); + verifyOutput(out); + assert(out.results != 'undefined', "no results for inline"); + + out = db.srcSharded.mapReduce( + map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); + verifyOutput(out); + + out = db.runCommand({ + mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce + map: map, + reduce: reduce, + out: "mrBasic" + "srcSharded", + }); + verifyOutput(out); + + st.stop(); + +})(); diff --git a/jstests/sharding/migrateBig.js b/jstests/sharding/migrateBig.js index e11782baed2..6e6be382795 100644 --- a/jstests/sharding/migrateBig.js +++ b/jstests/sharding/migrateBig.js @@ -1,64 +1,63 @@ (function() { + 'use strict'; var s = new ShardingTest({name: "migrateBig", shards: 2, other: {chunkSize: 1}}); - s.config.settings.update({_id: "balancer"}, {$set: {_waitForDelete: true}}, true); - s.adminCommand({enablesharding: "test"}); + assert.writeOK( + s.config.settings.update({_id: "balancer"}, {$set: {_waitForDelete: true}}, true)); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); - s.adminCommand({shardcollection: "test.foo", key: {x: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo", key: {x: 1}})); - db = s.getDB("test"); - coll = db.foo; + var db = s.getDB("test"); + var coll = db.foo; - big = ""; + var big = ""; while (big.length < 10000) big += "eliot"; var bulk = coll.initializeUnorderedBulkOp(); - for (x = 0; x < 100; x++) { + for (var x = 0; x < 100; x++) { bulk.insert({x: x, big: big}); } assert.writeOK(bulk.execute()); - db.printShardingStatus(); - - s.adminCommand({split: "test.foo", middle: {x: 30}}); - s.adminCommand({split: "test.foo", middle: {x: 66}}); - s.adminCommand( - {movechunk: "test.foo", find: {x: 90}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {x: 30}})); + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {x: 66}})); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.foo", find: {x: 90}, to: s.getOther(s.getServer("test")).name})); db.printShardingStatus(); print("YO : " + s.getServer("test").host); - direct = new Mongo(s.getServer("test").host); + var direct = new Mongo(s.getServer("test").host); print("direct : " + direct); - directDB = direct.getDB("test"); + var directDB = direct.getDB("test"); - for (done = 0; done < 2 * 1024 * 1024; done += big.length) { + for (var done = 0; done < 2 * 1024 * 1024; done += big.length) { assert.writeOK(directDB.foo.insert({x: 50 + Math.random(), big: big})); } db.printShardingStatus(); assert.throws(function() { - s.adminCommand( - {movechunk: "test.foo", find: {x: 50}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.foo", find: {x: 50}, to: s.getOther(s.getServer("test")).name})); }, [], "move should fail"); - for (i = 0; i < 20; i += 2) { + for (var i = 0; i < 20; i += 2) { try { - s.adminCommand({split: "test.foo", middle: {x: i}}); + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {x: i}})); } catch (e) { - // we may have auto split on some of these - // which is ok + // We may have auto split on some of these, which is ok print(e); } } db.printShardingStatus(); - s.config.settings.update({_id: "balancer"}, {$set: {stopped: false}}, true); + s.startBalancer(); assert.soon(function() { var x = s.chunkDiff("foo", "test"); @@ -73,5 +72,4 @@ assert.eq(coll.count(), coll.find().itcount()); s.stop(); - })(); diff --git a/jstests/sharding/migrateBig_balancer.js b/jstests/sharding/migrateBig_balancer.js index cd44a225a62..906ed341c7c 100644 --- a/jstests/sharding/migrateBig_balancer.js +++ b/jstests/sharding/migrateBig_balancer.js @@ -1,11 +1,12 @@ (function() { + 'use strict'; var st = new ShardingTest({name: 'migrateBig_balancer', shards: 2, other: {enableBalancer: true}}); var mongos = st.s; var admin = mongos.getDB("admin"); - db = mongos.getDB("test"); + var db = mongos.getDB("test"); var coll = db.getCollection("stuff"); assert.commandWorked(admin.runCommand({enablesharding: coll.getDB().getName()})); @@ -18,7 +19,7 @@ for (var i = 0; i < nsq; i++) data += data; - dataObj = {}; + var dataObj = {}; for (var i = 0; i < n; i++) dataObj["data-" + i] = data; @@ -30,19 +31,16 @@ assert.eq(40, coll.count(), "prep1"); - printjson(coll.stats()); - - admin.printShardingStatus(); - - admin.runCommand({shardcollection: "" + coll, key: {_id: 1}}); + assert.commandWorked(admin.runCommand({shardcollection: "" + coll, key: {_id: 1}})); + st.printShardingStatus(); assert.lt( 5, mongos.getDB("config").chunks.find({ns: "test.stuff"}).count(), "not enough chunks"); assert.soon(function() { // On *extremely* slow or variable systems, we've seen migrations fail in the critical - // section and - // kill the server. Do an explicit check for this. SERVER-8781 + // section and kill the server. Do an explicit check for this. SERVER-8781 + // // TODO: Remove once we can better specify what systems to run what tests on. try { assert.commandWorked(st.shard0.getDB("admin").runCommand({ping: 1})); @@ -53,7 +51,7 @@ throw e; } - res = mongos.getDB("config").chunks.group({ + var res = mongos.getDB("config").chunks.group({ cond: {ns: "test.stuff"}, key: {shard: 1}, reduce: function(doc, out) { @@ -68,5 +66,4 @@ }, "never migrated", 10 * 60 * 1000, 1000); st.stop(); - })(); diff --git a/jstests/sharding/printShardingStatus.js b/jstests/sharding/printShardingStatus.js index 05e6eca0d4f..5bfa70c2d8f 100644 --- a/jstests/sharding/printShardingStatus.js +++ b/jstests/sharding/printShardingStatus.js @@ -3,6 +3,7 @@ // headings and the names of sharded collections and their shard keys. (function() { + 'use strict'; var st = new ShardingTest({shards: 1, mongos: 2, config: 1, other: {smallfiles: true}}); @@ -233,5 +234,4 @@ assert(mongos.getDB("test").dropDatabase()); st.stop(); - })(); diff --git a/jstests/sharding/shard3.js b/jstests/sharding/shard3.js index 6800e3f4370..290a9f79719 100644 --- a/jstests/sharding/shard3.js +++ b/jstests/sharding/shard3.js @@ -1,5 +1,4 @@ (function() { - // Include helpers for analyzing explain output. load("jstests/libs/analyze_plan.js"); @@ -17,11 +16,14 @@ } assert(sh.getBalancerState(), "A1"); - sh.setBalancerState(false); + + sh.stopBalancer(); assert(!sh.getBalancerState(), "A2"); - sh.setBalancerState(true); + + sh.startBalancer(); assert(sh.getBalancerState(), "A3"); - sh.setBalancerState(false); + + sh.stopBalancer(); assert(!sh.getBalancerState(), "A4"); s.config.databases.find().forEach(printjson); diff --git a/jstests/sharding/split_with_force_small.js b/jstests/sharding/split_with_force_small.js index 0148c924993..be21049650e 100644 --- a/jstests/sharding/split_with_force_small.js +++ b/jstests/sharding/split_with_force_small.js @@ -1,70 +1,69 @@ // // Tests autosplit locations with force : true, for small collections // +(function() { + 'use strict'; -var options = { - chunkSize: 1 // MB -}; + var st = new ShardingTest( + {shards: 1, mongos: 1, other: {chunkSize: 1, mongosOptions: {noAutoSplit: ""}}}); -var st = new ShardingTest({shards: 1, mongos: 1, other: options}); -st.stopBalancer(); + var mongos = st.s0; + var admin = mongos.getDB("admin"); + var config = mongos.getDB("config"); + var shardAdmin = st.shard0.getDB("admin"); + var coll = mongos.getCollection("foo.bar"); -var mongos = st.s0; -var admin = mongos.getDB("admin"); -var config = mongos.getDB("config"); -var shardAdmin = st.shard0.getDB("admin"); -var coll = mongos.getCollection("foo.bar"); + assert.commandWorked(admin.runCommand({enableSharding: coll.getDB() + ""})); + assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); + assert.commandWorked(admin.runCommand({split: coll + "", middle: {_id: 0}})); -assert(admin.runCommand({enableSharding: coll.getDB() + ""}).ok); -assert(admin.runCommand({shardCollection: coll + "", key: {_id: 1}}).ok); -assert(admin.runCommand({split: coll + "", middle: {_id: 0}}).ok); + jsTest.log("Insert a bunch of data into the low chunk of a collection," + + " to prevent relying on stats."); -jsTest.log("Insert a bunch of data into the low chunk of a collection," + - " to prevent relying on stats."); + var data128k = "x"; + for (var i = 0; i < 7; i++) + data128k += data128k; -var data128k = "x"; -for (var i = 0; i < 7; i++) - data128k += data128k; + var bulk = coll.initializeUnorderedBulkOp(); + for (var i = 0; i < 1024; i++) { + bulk.insert({_id: -(i + 1)}); + } + assert.writeOK(bulk.execute()); -var bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < 1024; i++) { - bulk.insert({_id: -(i + 1)}); -} -assert.writeOK(bulk.execute()); + jsTest.log("Insert 32 docs into the high chunk of a collection"); -jsTest.log("Insert 32 docs into the high chunk of a collection"); + bulk = coll.initializeUnorderedBulkOp(); + for (var i = 0; i < 32; i++) { + bulk.insert({_id: i}); + } + assert.writeOK(bulk.execute()); -bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < 32; i++) { - bulk.insert({_id: i}); -} -assert.writeOK(bulk.execute()); + jsTest.log("Split off MaxKey chunk..."); -jsTest.log("Split off MaxKey chunk..."); + assert.commandWorked(admin.runCommand({split: coll + "", middle: {_id: 32}})); -assert(admin.runCommand({split: coll + "", middle: {_id: 32}}).ok); + jsTest.log("Keep splitting chunk multiple times..."); -jsTest.log("Keep splitting chunk multiple times..."); - -st.printShardingStatus(); - -for (var i = 0; i < 5; i++) { - assert(admin.runCommand({split: coll + "", find: {_id: 0}}).ok); st.printShardingStatus(); -} -// Make sure we can't split further than 5 (2^5) times -assert(!admin.runCommand({split: coll + "", find: {_id: 0}}).ok); + for (var i = 0; i < 5; i++) { + assert.commandWorked(admin.runCommand({split: coll + "", find: {_id: 0}})); + st.printShardingStatus(); + } + + // Make sure we can't split further than 5 (2^5) times + assert.commandFailed(admin.runCommand({split: coll + "", find: {_id: 0}})); -var chunks = config.chunks.find({'min._id': {$gte: 0, $lt: 32}}).sort({min: 1}).toArray(); -printjson(chunks); + var chunks = config.chunks.find({'min._id': {$gte: 0, $lt: 32}}).sort({min: 1}).toArray(); + printjson(chunks); -// Make sure the chunks grow by 2x (except the first) -var nextSize = 1; -for (var i = 0; i < chunks.size; i++) { - assert.eq(coll.count({_id: {$gte: chunks[i].min._id, $lt: chunks[i].max._id}}), nextSize); - if (i != 0) - nextSize += nextSize; -} + // Make sure the chunks grow by 2x (except the first) + var nextSize = 1; + for (var i = 0; i < chunks.size; i++) { + assert.eq(coll.count({_id: {$gte: chunks[i].min._id, $lt: chunks[i].max._id}}), nextSize); + if (i != 0) + nextSize += nextSize; + } -st.stop(); + st.stop(); +})(); diff --git a/jstests/sharding/stale_version_write.js b/jstests/sharding/stale_version_write.js index e5885dcfa41..bd603124548 100644 --- a/jstests/sharding/stale_version_write.js +++ b/jstests/sharding/stale_version_write.js @@ -1,37 +1,37 @@ // Tests whether a reset sharding version triggers errors +(function() { + 'use strict'; -jsTest.log("Starting sharded cluster..."); + var st = new ShardingTest({shards: 1, mongos: 2}); -var st = new ShardingTest({shards: 1, mongos: 2, verbose: 2}); + var mongosA = st.s0; + var mongosB = st.s1; -st.stopBalancer(); + jsTest.log("Adding new collections..."); -var mongosA = st.s0; -var mongosB = st.s1; + var collA = mongosA.getCollection(jsTestName() + ".coll"); + assert.writeOK(collA.insert({hello: "world"})); -jsTest.log("Adding new collections..."); + var collB = mongosB.getCollection("" + collA); + assert.writeOK(collB.insert({hello: "world"})); -var collA = mongosA.getCollection(jsTestName() + ".coll"); -assert.writeOK(collA.insert({hello: "world"})); + jsTest.log("Enabling sharding..."); -var collB = mongosB.getCollection("" + collA); -assert.writeOK(collB.insert({hello: "world"})); + assert.commandWorked(mongosA.getDB("admin").adminCommand({enableSharding: "" + collA.getDB()})); + assert.commandWorked( + mongosA.getDB("admin").adminCommand({shardCollection: "" + collA, key: {_id: 1}})); -jsTest.log("Enabling sharding..."); + // MongoD doesn't know about the config shard version *until* MongoS tells it + collA.findOne(); -printjson(mongosA.getDB("admin").runCommand({enableSharding: "" + collA.getDB()})); -printjson(mongosA.getDB("admin").runCommand({shardCollection: "" + collA, key: {_id: 1}})); + jsTest.log("Trigger shard version mismatch..."); -// MongoD doesn't know about the config shard version *until* MongoS tells it -collA.findOne(); + assert.writeOK(collB.insert({goodbye: "world"})); -jsTest.log("Trigger shard version mismatch..."); + print("Inserted..."); -assert.writeOK(collB.insert({goodbye: "world"})); + assert.eq(3, collA.find().itcount()); + assert.eq(3, collB.find().itcount()); -print("Inserted..."); - -assert.eq(3, collA.find().itcount()); -assert.eq(3, collB.find().itcount()); - -st.stop(); + st.stop(); +})(); diff --git a/jstests/slow2/mr_during_migrate.js b/jstests/slow2/mr_during_migrate.js index cb439aeb241..1b3f55721f4 100644 --- a/jstests/slow2/mr_during_migrate.js +++ b/jstests/slow2/mr_during_migrate.js @@ -1,113 +1,112 @@ // Do parallel ops with migrates occurring +(function() { + 'use strict'; -var st = new ShardingTest({shards: 10, mongos: 2, verbose: 2}); + var st = new ShardingTest({shards: 10, mongos: 2, verbose: 2}); -jsTest.log("Doing parallel operations..."); + var mongos = st.s0; + var admin = mongos.getDB("admin"); + var coll = st.s.getCollection(jsTest.name() + ".coll"); -// Stop balancer, since it'll just get in the way of these -st.stopBalancer(); + var numDocs = 1024 * 1024; + var dataSize = 1024; // bytes, must be power of 2 -var mongos = st.s0; -var admin = mongos.getDB("admin"); -var coll = st.s.getCollection(jsTest.name() + ".coll"); + var data = "x"; + while (data.length < dataSize) + data += data; -var numDocs = 1024 * 1024; -var dataSize = 1024; // bytes, must be power of 2 + var bulk = coll.initializeUnorderedBulkOp(); + for (var i = 0; i < numDocs; i++) { + bulk.insert({_id: i, data: data}); + } + assert.writeOK(bulk.execute()); -var data = "x"; -while (data.length < dataSize) - data += data; + // Make sure everything got inserted + assert.eq(numDocs, coll.find().itcount()); -var bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < numDocs; i++) { - bulk.insert({_id: i, data: data}); -} -assert.writeOK(bulk.execute()); + jsTest.log("Inserted " + sh._dataFormat(dataSize * numDocs) + " of data."); -// Make sure everything got inserted -assert.eq(numDocs, coll.find().itcount()); + // Shard collection + st.shardColl(coll, {_id: 1}, false); -jsTest.log("Inserted " + sh._dataFormat(dataSize * numDocs) + " of data."); + st.printShardingStatus(); -// Shard collection -st.shardColl(coll, {_id: 1}, false); + jsTest.log("Sharded collection now initialized, starting migrations..."); -st.printShardingStatus(); - -jsTest.log("Sharded collection now initialized, starting migrations..."); + var checkMigrate = function() { + print("Result of migrate : "); + printjson(this); + }; -var checkMigrate = function() { - print("Result of migrate : "); - printjson(this); -}; + // Creates a number of migrations of random chunks to diff shard servers + var ops = []; + for (var i = 0; i < st._connections.length; i++) { + ops.push({ + op: "command", + ns: "admin", + command: { + moveChunk: "" + coll, + find: {_id: {"#RAND_INT": [0, numDocs]}}, + to: st._connections[i].shardName, + _waitForDelete: true + }, + showResult: true + }); + } -// Creates a number of migrations of random chunks to diff shard servers -var ops = []; -for (var i = 0; i < st._connections.length; i++) { - ops.push({ - op: "command", - ns: "admin", - command: { - moveChunk: "" + coll, - find: {_id: {"#RAND_INT": [0, numDocs]}}, - to: st._connections[i].shardName, - _waitForDelete: true - }, - showResult: true - }); -} + // TODO: Also migrate output collection -// TODO: Also migrate output collection + jsTest.log("Starting migrations now..."); -jsTest.log("Starting migrations now..."); + var bid = benchStart({ops: ops, host: st.s.host, parallel: 1, handleErrors: false}); -var bid = benchStart({ops: ops, host: st.s.host, parallel: 1, handleErrors: false}); + //####################### + // Tests during migration -//####################### -// Tests during migration + var numTests = 5; -var numTests = 5; + for (var t = 0; t < numTests; t++) { + jsTest.log("Test #" + t); -for (var t = 0; t < numTests; t++) { - jsTest.log("Test #" + t); + var mongos = st.s1; // use other mongos so we get stale shard versions + var coll = mongos.getCollection(coll + ""); + var outputColl = mongos.getCollection(coll + "_output"); - var mongos = st.s1; // use other mongos so we get stale shard versions - var coll = mongos.getCollection(coll + ""); - var outputColl = mongos.getCollection(coll + "_output"); + var numTypes = 32; + var map = function() { + emit(this._id % 32 /* must be hardcoded */, {c: 1}); + }; - var numTypes = 32; - var map = function() { - emit(this._id % 32 /* must be hardcoded */, {c: 1}); - }; - var reduce = function(k, vals) { - var total = 0; - for (var i = 0; i < vals.length; i++) - total += vals[i].c; - return { - c: total + var reduce = function(k, vals) { + var total = 0; + for (var i = 0; i < vals.length; i++) + total += vals[i].c; + return { + c: total + }; }; - }; - printjson(coll.find({_id: 0}).itcount()); + printjson(coll.find({_id: 0}).itcount()); - jsTest.log("Starting new mapReduce run #" + t); + jsTest.log("Starting new mapReduce run #" + t); - // assert.eq( coll.find().itcount(), numDocs ) + // assert.eq( coll.find().itcount(), numDocs ) - coll.getMongo().getDB("admin").runCommand({setParameter: 1, traceExceptions: true}); + coll.getMongo().getDB("admin").runCommand({setParameter: 1, traceExceptions: true}); - printjson(coll.mapReduce( - map, reduce, {out: {replace: outputColl.getName(), db: outputColl.getDB() + ""}})); + printjson(coll.mapReduce( + map, reduce, {out: {replace: outputColl.getName(), db: outputColl.getDB() + ""}})); - jsTest.log("MapReduce run #" + t + " finished."); + jsTest.log("MapReduce run #" + t + " finished."); - assert.eq(outputColl.find().itcount(), numTypes); + assert.eq(outputColl.find().itcount(), numTypes); - outputColl.find().forEach(function(x) { - assert.eq(x.value.c, numDocs / numTypes); - }); -} + outputColl.find().forEach(function(x) { + assert.eq(x.value.c, numDocs / numTypes); + }); + } -printjson(benchFinish(bid)); + printjson(benchFinish(bid)); -st.stop(); + st.stop(); +})(); diff --git a/src/mongo/base/validate_locale.cpp b/src/mongo/base/validate_locale.cpp index 5a4320c34c5..81207d4d89d 100644 --- a/src/mongo/base/validate_locale.cpp +++ b/src/mongo/base/validate_locale.cpp @@ -30,6 +30,7 @@ #include #include "mongo/base/init.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { @@ -38,13 +39,15 @@ MONGO_INITIALIZER_GENERAL(ValidateLocale, MONGO_NO_PREREQUISITES, MONGO_DEFAULT_ try { // Validate that boost can correctly load the user's locale boost::filesystem::path("/").has_root_directory(); - } catch (const std::runtime_error&) { - return Status(ErrorCodes::BadValue, - "Invalid or no user locale set." + } catch (const std::runtime_error& e) { + return Status( + ErrorCodes::BadValue, + str::stream() + << "Invalid or no user locale set. " #ifndef _WIN32 - " Please ensure LANG and/or LC_* environment variables are set correctly." + << " Please ensure LANG and/or LC_* environment variables are set correctly. " #endif - ); + << e.what()); } return Status::OK(); } diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index f93dee2b554..ced9f2bbdb7 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -308,7 +308,7 @@ Config::Config(const string& _dbname, const BSONObj& cmdObj) { // scope and code if (cmdObj["scope"].type() == Object) - scopeSetup = cmdObj["scope"].embeddedObjectUserCheck(); + scopeSetup = cmdObj["scope"].embeddedObjectUserCheck().getOwned(); mapper.reset(new JSMapper(cmdObj["map"])); reducer.reset(new JSReducer(cmdObj["reduce"])); @@ -316,7 +316,7 @@ Config::Config(const string& _dbname, const BSONObj& cmdObj) { finalizer.reset(new JSFinalizer(cmdObj["finalize"])); if (cmdObj["mapparams"].type() == Array) { - mapParams = cmdObj["mapparams"].embeddedObjectUserCheck(); + mapParams = cmdObj["mapparams"].embeddedObjectUserCheck().getOwned(); } } @@ -787,6 +787,7 @@ void State::init() { AuthorizationSession::get(ClientBasic::getCurrent())->getAuthenticatedUserNamesToken(); _scope.reset(globalScriptEngine->newScopeForCurrentThread()); _scope->registerOperation(_txn); + _scope->requireOwnedObjects(); _scope->setLocalDB(_config.dbname); _scope->loadStored(_txn, true); @@ -1435,6 +1436,7 @@ public: BSONObj o; PlanExecutor::ExecState execState; while (PlanExecutor::ADVANCED == (execState = exec->getNext(&o, NULL))) { + o = o.getOwned(); // we will be accessing outside of the lock // check to see if this is a new object we don't own yet // because of a chunk migration if (collMetadata) { diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index d3c62ea0ef8..ced194addbf 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -448,11 +448,12 @@ long long Helpers::removeRange(OperationContext* txn, txn, repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), writeConcern); - if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit) { + if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || + replStatus.status.code() == ErrorCodes::WriteConcernFailed) { warning(LogComponent::kSharding) << "replication to secondaries for removeRange at " "least 60 seconds behind"; } else { - massertStatusOK(replStatus.status); + uassertStatusOK(replStatus.status); } millisWaitingForReplication += replStatus.duration; } diff --git a/src/mongo/db/exec/group.cpp b/src/mongo/db/exec/group.cpp index 4ce1d4031db..31a72df4772 100644 --- a/src/mongo/db/exec/group.cpp +++ b/src/mongo/db/exec/group.cpp @@ -148,7 +148,8 @@ Status GroupStage::processObject(const BSONObj& obj) { } } - _scope->setObject("obj", obj, true); + BSONObj objCopy = obj.getOwned(); + _scope->setObject("obj", objCopy, true); _scope->setNumber("n", n - 1); try { diff --git a/src/mongo/db/ops/update_driver.cpp b/src/mongo/db/ops/update_driver.cpp index c0c11082a2e..a99376174a8 100644 --- a/src/mongo/db/ops/update_driver.cpp +++ b/src/mongo/db/ops/update_driver.cpp @@ -183,6 +183,13 @@ Status UpdateDriver::populateDocumentWithQueryFields(const BSONObj& query, return populateDocumentWithQueryFields(*cq, immutablePaths, doc); } +namespace { + +const FieldRef idPath("_id"); +const vector emptyImmutablePaths; + +} // namespace + Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query, const vector* immutablePathsPtr, mutablebson::Document& doc) const { @@ -192,13 +199,12 @@ Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query if (isDocReplacement()) { FieldRefSet pathsToExtract; - // TODO: Refactor update logic, make _id just another immutable field - static const FieldRef idPath("_id"); - static const vector emptyImmutablePaths; const vector& immutablePaths = immutablePathsPtr ? *immutablePathsPtr : emptyImmutablePaths; pathsToExtract.fillFrom(immutablePaths); + + // TODO: Refactor update logic, make _id just another immutable field pathsToExtract.insert(&idPath); // Extract only immutable fields from replacement-style diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 63858050014..2c764baf9e3 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -825,11 +825,7 @@ Status applyOperation_inlock(OperationContext* txn, Status status{ErrorCodes::NotYetInitialized, ""}; { WriteUnitOfWork wuow(txn); - try { - status = collection->insertDocument(txn, o, true); - } catch (DBException dbe) { - status = dbe.toStatus(); - } + status = collection->insertDocument(txn, o, true); if (status.isOK()) { wuow.commit(); } diff --git a/src/mongo/dbtests/jstests.cpp b/src/mongo/dbtests/jstests.cpp index 53cead63a3a..04a9180dbbd 100644 --- a/src/mongo/dbtests/jstests.cpp +++ b/src/mongo/dbtests/jstests.cpp @@ -2361,6 +2361,49 @@ public: } }; +class RequiresOwnedObjects { +public: + void run() { + char buf[] = {5, 0, 0, 0, 0}; + BSONObj unowned(buf); + BSONObj owned = unowned.getOwned(); + + ASSERT(!unowned.isOwned()); + ASSERT(owned.isOwned()); + + // Ensure that by default we can bind owned and unowned + { + unique_ptr s(globalScriptEngine->newScope()); + s->setObject("unowned", unowned, true); + s->setObject("owned", owned, true); + } + + // After we set the flag, we should only be able to set owned + { + unique_ptr s(globalScriptEngine->newScope()); + s->requireOwnedObjects(); + s->setObject("owned", owned, true); + + bool threwException = false; + try { + s->setObject("unowned", unowned, true); + } catch (...) { + threwException = true; + + auto status = exceptionToStatus(); + + ASSERT_EQUALS(status.code(), ErrorCodes::BadValue); + } + + ASSERT(threwException); + + // after resetting, we can set unowned's again + s->reset(); + s->setObject("unowned", unowned, true); + } + } +}; + class All : public Suite { public: All() : Suite("js") { @@ -2422,6 +2465,7 @@ public: add(); add(); + add(); add(); add(); diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 34efb20cf88..cf8d04827a6 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -75,9 +75,9 @@ env.Library( '$BUILD_DIR/mongo/s/catalog/forwarding_catalog_manager', '$BUILD_DIR/mongo/s/catalog/replset/catalog_manager_replica_set', '$BUILD_DIR/mongo/s/coreshard', - '$BUILD_DIR/mongo/s/mongoscore', '$BUILD_DIR/mongo/util/clock_source_mock', '$BUILD_DIR/mongo/util/net/message_port_mock', + 'mongoscore', ], LIBDEPS_TAGS=[ # Depends on coreshard, but that would be circular diff --git a/src/mongo/s/catalog/catalog_manager.h b/src/mongo/s/catalog/catalog_manager.h index 5257f53f7db..2f75ba2da69 100644 --- a/src/mongo/s/catalog/catalog_manager.h +++ b/src/mongo/s/catalog/catalog_manager.h @@ -451,9 +451,6 @@ public: */ virtual bool isMetadataConsistentFromLastCheck(OperationContext* txn) = 0; -protected: - CatalogManager() = default; - /** * Obtains a reference to the distributed lock manager instance to use for synchronizing * system-wide changes. @@ -462,6 +459,9 @@ protected: * be cached. */ virtual DistLockManager* getDistLockManager() = 0; + +protected: + CatalogManager() = default; }; } // namespace mongo diff --git a/src/mongo/s/chunk_manager.cpp b/src/mongo/s/chunk_manager.cpp index f785e748c54..29e596948d4 100644 --- a/src/mongo/s/chunk_manager.cpp +++ b/src/mongo/s/chunk_manager.cpp @@ -98,6 +98,9 @@ public: string shardFor(OperationContext* txn, const string& hostName) const final { const auto shard = grid.shardRegistry()->getShard(txn, hostName); + uassert(ErrorCodes::ShardNotFound, + str::stream() << "Shard " << hostName << " not found.", + shard); return shard->getId(); } diff --git a/src/mongo/scripting/deadline_monitor.cpp b/src/mongo/scripting/deadline_monitor.cpp index 5eb0f52e5de..f75c34a0cc2 100644 --- a/src/mongo/scripting/deadline_monitor.cpp +++ b/src/mongo/scripting/deadline_monitor.cpp @@ -34,7 +34,7 @@ namespace mongo { -MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 1000); +MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 0); int getScriptingEngineInterruptInterval() { return scriptingEngineInterruptIntervalMS.load(); diff --git a/src/mongo/scripting/deadline_monitor.h b/src/mongo/scripting/deadline_monitor.h index b1c2855dd46..05acd5c349c 100644 --- a/src/mongo/scripting/deadline_monitor.h +++ b/src/mongo/scripting/deadline_monitor.h @@ -138,7 +138,7 @@ private: const Date_t now = Date_t::now(); const auto interruptInterval = Milliseconds{getScriptingEngineInterruptInterval()}; - if (now - lastInterruptCycle > interruptInterval) { + if ((interruptInterval.count() > 0) && (now - lastInterruptCycle > interruptInterval)) { for (const auto& task : _tasks) { if (task.second > now) task.first->interrupt(); @@ -148,9 +148,13 @@ private: // wait for a task to be added or a deadline to expire if (_nearestDeadlineWallclock > now) { - if (_nearestDeadlineWallclock == Date_t::max() || - _nearestDeadlineWallclock - now > interruptInterval) { - _newDeadlineAvailable.wait_for(lk, interruptInterval); + if (_nearestDeadlineWallclock == Date_t::max()) { + if ((interruptInterval.count() > 0) && + (_nearestDeadlineWallclock - now > interruptInterval)) { + _newDeadlineAvailable.wait_for(lk, interruptInterval); + } else { + _newDeadlineAvailable.wait(lk); + } } else { _newDeadlineAvailable.wait_until(lk, _nearestDeadlineWallclock.toSystemTimePoint()); diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index 7e6628344ab..71c42f2fe74 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -422,6 +422,9 @@ public: void advanceGeneration() { _real->advanceGeneration(); } + void requireOwnedObjects() override { + _real->requireOwnedObjects(); + } bool isKillPending() const { return _real->isKillPending(); } diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index d31311f0114..2fbae5c347e 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -105,6 +105,8 @@ public: virtual void advanceGeneration() = 0; + virtual void requireOwnedObjects() = 0; + virtual ScriptingFunction createFunction(const char* code); /** diff --git a/src/mongo/scripting/mozjs/bson.cpp b/src/mongo/scripting/mozjs/bson.cpp index 1e7e4b7c190..3783bc0755a 100644 --- a/src/mongo/scripting/mozjs/bson.cpp +++ b/src/mongo/scripting/mozjs/bson.cpp @@ -59,13 +59,17 @@ namespace { * the appearance of mutable state on the read/write versions. */ struct BSONHolder { - BSONHolder(const BSONObj& obj, const BSONObj* parent, std::size_t generation, bool ro) + BSONHolder(const BSONObj& obj, const BSONObj* parent, const MozJSImplScope* scope, bool ro) : _obj(obj), - _generation(generation), + _generation(scope->getGeneration()), _isOwned(obj.isOwned() || (parent && parent->isOwned())), _resolved(false), _readOnly(ro), _altered(false) { + uassert( + ErrorCodes::BadValue, + "Attempt to bind an unowned BSON Object to a JS scope marked as requiring ownership", + _isOwned || (!scope->requiresOwnedObjects())); if (parent) { _parent.emplace(*parent); } @@ -107,7 +111,7 @@ void BSONInfo::make( auto scope = getScope(cx); scope->getProto().newObject(obj); - JS_SetPrivate(obj, new BSONHolder(bson, parent, scope->getGeneration(), ro)); + JS_SetPrivate(obj, new BSONHolder(bson, parent, scope, ro)); } void BSONInfo::finalize(JSFreeOp* fop, JSObject* obj) { diff --git a/src/mongo/scripting/mozjs/implscope.cpp b/src/mongo/scripting/mozjs/implscope.cpp index 3b260c1558c..e85efd97505 100644 --- a/src/mongo/scripting/mozjs/implscope.cpp +++ b/src/mongo/scripting/mozjs/implscope.cpp @@ -322,6 +322,7 @@ MozJSImplScope::MozJSImplScope(MozJSScriptEngine* engine) _status(Status::OK()), _quickExit(false), _generation(0), + _requireOwnedObjects(false), _hasOutOfMemoryException(false), _binDataProto(_context), _bsonProto(_context), @@ -749,6 +750,7 @@ void MozJSImplScope::reset() { unregisterOperation(); _pendingKill.store(false); _pendingGC.store(false); + _requireOwnedObjects = false; advanceGeneration(); } @@ -869,6 +871,14 @@ void MozJSImplScope::advanceGeneration() { _generation++; } +void MozJSImplScope::requireOwnedObjects() { + _requireOwnedObjects = true; +} + +bool MozJSImplScope::requiresOwnedObjects() const { + return _requireOwnedObjects; +} + const std::string& MozJSImplScope::getParentStack() const { return _parentStack; } diff --git a/src/mongo/scripting/mozjs/implscope.h b/src/mongo/scripting/mozjs/implscope.h index bfff70a93b0..edfc236e90c 100644 --- a/src/mongo/scripting/mozjs/implscope.h +++ b/src/mongo/scripting/mozjs/implscope.h @@ -305,6 +305,10 @@ public: void advanceGeneration() override; + void requireOwnedObjects() override; + + bool requiresOwnedObjects() const; + JS::HandleId getInternedStringId(InternedString name) { return _internedStrings.getInternedString(name); } @@ -373,6 +377,7 @@ private: bool _quickExit; std::string _parentStack; std::size_t _generation; + bool _requireOwnedObjects; bool _hasOutOfMemoryException; WrapType _binDataProto; diff --git a/src/mongo/scripting/mozjs/objectwrapper.cpp b/src/mongo/scripting/mozjs/objectwrapper.cpp index 77ea9a1eca8..8781f792484 100644 --- a/src/mongo/scripting/mozjs/objectwrapper.cpp +++ b/src/mongo/scripting/mozjs/objectwrapper.cpp @@ -564,7 +564,12 @@ ObjectWrapper::WriteFieldRecursionFrame::WriteFieldRecursionFrame(JSContext* cx, ids.infallibleAppend(rid); } } else { - JS::AutoIdArray rids(cx, JS_Enumerate(cx, thisv)); + auto ridArrayPtr = JS_Enumerate(cx, thisv); + if (!ridArrayPtr) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to enumerate object"); + } + JS::AutoIdArray rids(cx, ridArrayPtr); if (!ids.reserve(rids.length())) { throwCurrentJSException( diff --git a/src/mongo/scripting/mozjs/proxyscope.cpp b/src/mongo/scripting/mozjs/proxyscope.cpp index bb4cd5c06ff..9bb11ce400b 100644 --- a/src/mongo/scripting/mozjs/proxyscope.cpp +++ b/src/mongo/scripting/mozjs/proxyscope.cpp @@ -120,6 +120,10 @@ void MozJSProxyScope::advanceGeneration() { run([&] { _implScope->advanceGeneration(); }); } +void MozJSProxyScope::requireOwnedObjects() { + run([&] { _implScope->requireOwnedObjects(); }); +} + double MozJSProxyScope::getNumber(const char* field) { double out; run([&] { out = _implScope->getNumber(field); }); diff --git a/src/mongo/scripting/mozjs/proxyscope.h b/src/mongo/scripting/mozjs/proxyscope.h index 451981330a1..4dd69a3ebe9 100644 --- a/src/mongo/scripting/mozjs/proxyscope.h +++ b/src/mongo/scripting/mozjs/proxyscope.h @@ -129,6 +129,8 @@ public: void advanceGeneration() override; + void requireOwnedObjects() override; + double getNumber(const char* field) override; int getNumberInt(const char* field) override; long long getNumberLongLong(const char* field) override; diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index 4a64f5e50a0..89ab91da580 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -89,7 +89,7 @@ var ReplSetTest = function(opts) { var _unbridgedPorts; var _unbridgedNodes; - this.kDefaultTimeoutMS = 5 * 60 * 1000; + this.kDefaultTimeoutMS = 10 * 60 * 1000; // Publicly exposed variables diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 1f7f7d9fd3a..f3852d00ac8 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -731,6 +731,7 @@ fsyncLock fsyncs ftruncate func +fvisibility gcc gdb ge diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index ac79ffd029a..512892eb44d 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -150,6 +150,7 @@ connection_stats = [ ConnStat('read_io', 'total read I/Os'), ConnStat('rwlock_read', 'pthread mutex shared lock read-lock calls'), ConnStat('rwlock_write', 'pthread mutex shared lock write-lock calls'), + ConnStat('time_travel', 'detected system time went backwards'), ConnStat('write_io', 'total write I/Os'), ########################################## diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index abf3fe5cb9c..8bd00db3aa2 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "b8f590dea0400666ef26e21adf11c5997bb5ef1b", + "commit": "827b48a34227243c809d41fac3dc909ed46b0c5e", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.2" diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index 032fdf7d897..5bb09832eed 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -471,6 +471,8 @@ __wt_row_ikey_alloc(WT_SESSION_IMPL *session, { WT_IKEY *ikey; + WT_ASSERT(session, key != NULL); /* quiet clang scan-build */ + /* * Allocate memory for the WT_IKEY structure and the key, then copy * the key into place. diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index aa76bef4614..205e7544c6c 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -65,19 +65,20 @@ each of which is ordered by one or more columns. - @subpage_single wtperf - @subpage_single wtstats

-- @subpage_single tune_memory_allocator -- @subpage_single tune_page_size_and_comp -- @subpage_single tune_cache +- @subpage_single tune_build_options - @subpage_single tune_bulk_load +- @subpage_single tune_cache +- @subpage_single tune_checksum +- @subpage_single tune_close - @subpage_single tune_cursor_persist -- @subpage_single tune_read_only - @subpage_single tune_durability -- @subpage_single tune_checksum - @subpage_single tune_file_alloc +- @subpage_single tune_memory_allocator +- @subpage_single tune_mutex +- @subpage_single tune_page_size_and_comp +- @subpage_single tune_read_only - @subpage_single tune_system_buffer_cache - @subpage_single tune_transparent_huge_pages -- @subpage_single tune_close -- @subpage_single tune_mutex - @subpage_single tune_zone_reclaim */ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index bc2e16b1122..5d629f4c49f 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -237,6 +237,7 @@ fput freelist fsync ftruncate +fvisibility gcc gdbm ge diff --git a/src/third_party/wiredtiger/src/docs/tune-build-options.dox b/src/third_party/wiredtiger/src/docs/tune-build-options.dox new file mode 100644 index 00000000000..79cd60b1105 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/tune-build-options.dox @@ -0,0 +1,9 @@ +/*! @page tune_build_options gcc/clang build options + +WiredTiger can be built using the gcc/clang \c -fvisibility=hidden flag, +which may significantly reduce the size and load time of the WiredTiger +library when built as a dynamic shared object, and allow the optimizer +to produce better code (for example, by eliminating most lookups in the +procedure linkage table). + + */ diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 26bbf9f679b..cc3c5a5c824 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -941,6 +941,13 @@ __evict_tune_workers(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + /* + * If we have a fixed number of eviction threads, there is no value in + * calculating if we should do any tuning. + */ + if (conn->evict_threads_max == conn->evict_threads_min) + return (0); + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); pgs_evicted_cur = pgs_evicted_persec_cur = 0; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index bf3279d0f94..dfd2d03707f 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -570,6 +570,7 @@ extern int __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) extern int __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_sweep_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, size_t len, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index 22d78fc165a..6c712813d29 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -9,7 +9,7 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ -/* Add GCC-specific attributes to types and function declarations. */ +/* GCC-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ struct __attribute__ ((__packed__)) name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 2d0f47988b7..813b9182683 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -9,6 +9,7 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ +/* Lint-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ struct name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index 7040886cf82..fad10f01103 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -54,6 +54,31 @@ __wt_seconds(WT_SESSION_IMPL *session, time_t *timep) *timep = t.tv_sec; } +/* + * __wt_time_check_monotonic -- + * Check and prevent time running backward. If we detect that it has, we + * set the time structure to the previous values, making time stand still + * until we see a time in the future of the highest value seen so far. + */ +static inline void +__wt_time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) +{ + /* + * Detect time going backward. If so, use the last + * saved timestamp. + */ + if (session == NULL) + return; + + if (tsp->tv_sec < session->last_epoch.tv_sec || + (tsp->tv_sec == session->last_epoch.tv_sec && + tsp->tv_nsec < session->last_epoch.tv_nsec)) { + WT_STAT_CONN_INCR(session, time_travel); + *tsp = session->last_epoch; + } else + session->last_epoch = *tsp; +} + /* * __wt_verbose -- * Verbose message. diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index 6c5c8b67647..c9399be3185 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -16,9 +16,7 @@ #define WT_PTRDIFFT_FMT "Id" /* ptrdiff_t format string */ #define WT_SIZET_FMT "Iu" /* size_t format string */ -/* - * Add MSVC-specific attributes and pragmas to types and function declarations. - */ +/* MSVC-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ __pragma(pack(push,1)) \ struct name { diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 1b2dfd1ed2b..d05dee68641 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -66,6 +66,7 @@ struct __wt_session_impl { /* Session handle reference list */ TAILQ_HEAD(__dhandles, __wt_data_handle_cache) dhandles; time_t last_sweep; /* Last sweep for dead handles */ + struct timespec last_epoch; /* Last epoch time returned */ /* Cursors closed with the session */ TAILQ_HEAD(__cursors, __wt_cursor) cursors; @@ -97,6 +98,12 @@ struct __wt_session_impl { */ TAILQ_HEAD(__tables, __wt_table) tables; + /* + * Updated when the table cache is swept of all tables older than the + * current schema generation. + */ + uint64_t table_sweep_gen; + /* Current rwlock for callback. */ WT_RWLOCK *current_rwlock; uint8_t current_rwticket; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 6c274484bcb..db48a841571 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -361,6 +361,7 @@ struct __wt_connection_stats { int64_t cache_eviction_clean; int64_t cond_auto_wait_reset; int64_t cond_auto_wait; + int64_t time_travel; int64_t file_open; int64_t memory_allocation; int64_t memory_free; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 7e802c188ab..fdf9c714afa 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -93,6 +93,8 @@ struct __wt_txn_global { * the global transaction state. */ WT_RWLOCK scan_rwlock; + /* Protects logging, checkpoints and transaction visibility. */ + WT_RWLOCK visibility_rwlock; /* * Track information about the running checkpoint. The transaction diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index ddecb2ac765..821efdf5fa1 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -39,6 +39,16 @@ extern "C" { #define __F(func) (*(func)) #endif +/* + * We support configuring WiredTiger with the gcc/clang -fvisibility=hidden + * flags, but that requires public APIs be specifically marked. + */ +#if defined(DOXYGEN) || defined(SWIG) || !defined(__GNUC__) +#define WT_ATTRIBUTE_LIBRARY_VISIBLE +#else +#define WT_ATTRIBUTE_LIBRARY_VISIBLE __attribute__((visibility("default"))) +#endif + #ifdef SWIG %{ #include @@ -2553,7 +2563,7 @@ struct __wt_connection { */ int wiredtiger_open(const char *home, WT_EVENT_HANDLER *errhandler, const char *config, - WT_CONNECTION **connectionp); + WT_CONNECTION **connectionp) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Return information about a WiredTiger error as a string (see @@ -2564,7 +2574,7 @@ int wiredtiger_open(const char *home, * @param error a return value from a WiredTiger, ISO C, or POSIX standard API * @returns a string representation of the error */ -const char *wiredtiger_strerror(int error); +const char *wiredtiger_strerror(int error) WT_ATTRIBUTE_LIBRARY_VISIBLE; #if !defined(SWIG) /*! @@ -2701,7 +2711,8 @@ struct __wt_event_handler { * @errors */ int wiredtiger_struct_pack(WT_SESSION *session, - void *buffer, size_t size, const char *format, ...); + void *buffer, size_t size, const char *format, ...) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Calculate the size required to pack a structure. @@ -2719,7 +2730,7 @@ int wiredtiger_struct_pack(WT_SESSION *session, * @errors */ int wiredtiger_struct_size(WT_SESSION *session, - size_t *sizep, const char *format, ...); + size_t *sizep, const char *format, ...) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a structure from a buffer. @@ -2736,7 +2747,8 @@ int wiredtiger_struct_size(WT_SESSION *session, * @errors */ int wiredtiger_struct_unpack(WT_SESSION *session, - const void *buffer, size_t size, const char *format, ...); + const void *buffer, size_t size, const char *format, ...) + WT_ATTRIBUTE_LIBRARY_VISIBLE; #if !defined(SWIG) @@ -2763,7 +2775,8 @@ typedef struct __wt_pack_stream WT_PACK_STREAM; * @errors */ int wiredtiger_pack_start(WT_SESSION *session, - const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); + const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Start an unpacking operation from a buffer with the given format string. @@ -2779,7 +2792,8 @@ int wiredtiger_pack_start(WT_SESSION *session, * @errors */ int wiredtiger_unpack_start(WT_SESSION *session, - const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); + const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Close a packing stream. @@ -2788,7 +2802,8 @@ int wiredtiger_unpack_start(WT_SESSION *session, * @param[out] usedp the number of bytes in the buffer used by the stream * @errors */ -int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp); +int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack an item into a packing stream. @@ -2797,7 +2812,8 @@ int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp); * @param item an item to pack * @errors */ -int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack a signed integer into a packing stream. @@ -2806,7 +2822,8 @@ int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); * @param i a signed integer to pack * @errors */ -int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i); +int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack a string into a packing stream. @@ -2815,7 +2832,8 @@ int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i); * @param s a string to pack * @errors */ -int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s); +int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack an unsigned integer into a packing stream. @@ -2824,7 +2842,8 @@ int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s); * @param u an unsigned integer to pack * @errors */ -int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u); +int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack an item from a packing stream. @@ -2833,7 +2852,8 @@ int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u); * @param item an item to unpack * @errors */ -int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a signed integer from a packing stream. @@ -2842,7 +2862,8 @@ int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); * @param[out] ip the unpacked signed integer * @errors */ -int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); +int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a string from a packing stream. @@ -2851,7 +2872,8 @@ int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); * @param[out] sp the unpacked string * @errors */ -int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp); +int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack an unsigned integer from a packing stream. @@ -2860,7 +2882,8 @@ int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp); * @param[out] up the unpacked unsigned integer * @errors */ -int wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up); +int wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! @} */ /*! @@ -2938,7 +2961,8 @@ struct __wt_config_item { * @snippet ex_all.c Validate a configuration string */ int wiredtiger_config_validate(WT_SESSION *session, - WT_EVENT_HANDLER *errhandler, const char *name, const char *config); + WT_EVENT_HANDLER *errhandler, const char *name, const char *config) + WT_ATTRIBUTE_LIBRARY_VISIBLE; #endif /*! @@ -2958,7 +2982,8 @@ int wiredtiger_config_validate(WT_SESSION *session, * @snippet ex_config_parse.c Create a configuration parser */ int wiredtiger_config_parser_open(WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * A handle that can be used to search and traverse configuration strings @@ -3047,7 +3072,8 @@ struct __wt_config_parser { * @param patchp a location where the patch version number is returned * @returns a string representation of the version */ -const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); +const char *wiredtiger_version(int *majorp, int *minorp, int *patchp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /******************************************* * Error returns @@ -4546,304 +4572,306 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1102 /*! connection: auto adjusting condition wait calls */ #define WT_STAT_CONN_COND_AUTO_WAIT 1103 +/*! connection: detected system time went backwards */ +#define WT_STAT_CONN_TIME_TRAVEL 1104 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1104 +#define WT_STAT_CONN_FILE_OPEN 1105 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1105 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1106 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1106 +#define WT_STAT_CONN_MEMORY_FREE 1107 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1107 +#define WT_STAT_CONN_MEMORY_GROW 1108 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1108 +#define WT_STAT_CONN_COND_WAIT 1109 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1109 +#define WT_STAT_CONN_RWLOCK_READ 1110 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1110 +#define WT_STAT_CONN_RWLOCK_WRITE 1111 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1111 +#define WT_STAT_CONN_FSYNC_IO 1112 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1112 +#define WT_STAT_CONN_READ_IO 1113 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1113 +#define WT_STAT_CONN_WRITE_IO 1114 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1114 +#define WT_STAT_CONN_CURSOR_CREATE 1115 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1115 +#define WT_STAT_CONN_CURSOR_INSERT 1116 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1116 +#define WT_STAT_CONN_CURSOR_NEXT 1117 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1117 +#define WT_STAT_CONN_CURSOR_PREV 1118 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1118 +#define WT_STAT_CONN_CURSOR_REMOVE 1119 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1119 +#define WT_STAT_CONN_CURSOR_RESET 1120 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1120 +#define WT_STAT_CONN_CURSOR_RESTART 1121 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1121 +#define WT_STAT_CONN_CURSOR_SEARCH 1122 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1122 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1123 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1123 +#define WT_STAT_CONN_CURSOR_UPDATE 1124 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1124 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1125 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1125 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1126 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1126 +#define WT_STAT_CONN_DH_SWEEP_REF 1127 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1127 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1128 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1128 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1129 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1129 +#define WT_STAT_CONN_DH_SWEEP_TOD 1130 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1130 +#define WT_STAT_CONN_DH_SWEEPS 1131 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1131 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1132 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1132 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1133 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1133 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1134 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1134 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1135 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1135 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1136 /*! lock: handle-list lock eviction thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1136 +#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1137 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1137 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1138 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1138 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1139 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1139 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1140 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1140 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1141 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1141 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1142 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1142 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1143 /*! lock: table lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_COUNT 1143 +#define WT_STAT_CONN_LOCK_TABLE_COUNT 1144 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1144 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1145 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1145 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1146 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1146 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1147 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1147 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1148 /*! log: consolidated slot join active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1148 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1149 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1149 +#define WT_STAT_CONN_LOG_SLOT_RACES 1150 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1150 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1151 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1151 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1152 /*! log: consolidated slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1152 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1153 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1153 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1154 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1154 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1155 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1155 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1156 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1156 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1157 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1157 +#define WT_STAT_CONN_LOG_FLUSH 1158 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1158 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1159 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1159 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1160 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1160 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1161 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1161 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1162 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1162 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1163 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1163 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1164 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1164 +#define WT_STAT_CONN_LOG_SCANS 1165 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1165 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1166 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1166 +#define WT_STAT_CONN_LOG_WRITE_LSN 1167 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1167 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1168 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1168 +#define WT_STAT_CONN_LOG_SYNC 1169 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1169 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1170 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1170 +#define WT_STAT_CONN_LOG_SYNC_DIR 1171 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1171 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1172 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1172 +#define WT_STAT_CONN_LOG_WRITES 1173 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1173 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1174 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1174 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1175 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1175 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1176 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1176 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1177 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1177 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1178 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1178 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1179 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1179 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1180 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1180 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1181 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1181 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1182 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1182 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1183 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1183 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1184 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1184 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1185 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1185 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1186 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1186 +#define WT_STAT_CONN_REC_PAGES 1187 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1187 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1188 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1188 +#define WT_STAT_CONN_REC_PAGE_DELETE 1189 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1189 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1190 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1190 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1191 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1191 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1192 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1192 +#define WT_STAT_CONN_SESSION_OPEN 1193 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1193 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1194 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1194 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1195 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1195 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1196 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1196 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1197 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1197 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1198 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1198 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1199 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1199 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1200 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1200 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1201 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1201 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1202 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1202 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1203 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1203 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1204 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1204 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1205 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1205 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1206 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1206 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1207 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1207 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1208 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1208 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1209 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1209 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1210 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1210 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1211 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1211 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1212 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1212 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1213 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1213 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1214 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1214 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1215 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1215 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1216 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1216 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1217 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1217 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1218 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1218 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1219 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1219 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1220 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1220 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1221 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1221 +#define WT_STAT_CONN_PAGE_SLEEP 1222 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1222 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1223 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1223 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1224 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1224 +#define WT_STAT_CONN_TXN_BEGIN 1225 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1225 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1226 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1227 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1227 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1228 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1229 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1230 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1230 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1231 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1231 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1232 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1232 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1233 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1233 +#define WT_STAT_CONN_TXN_CHECKPOINT 1234 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1234 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1235 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1235 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1236 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1236 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1237 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1237 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1238 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1238 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1239 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1239 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1240 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1240 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1241 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1241 +#define WT_STAT_CONN_TXN_SYNC 1242 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1242 +#define WT_STAT_CONN_TXN_COMMIT 1243 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1243 +#define WT_STAT_CONN_TXN_ROLLBACK 1244 /*! * @} diff --git a/src/third_party/wiredtiger/src/os_common/os_alloc.c b/src/third_party/wiredtiger/src/os_common/os_alloc.c index ef96ed09ea7..c54bcc718f2 100644 --- a/src/third_party/wiredtiger/src/os_common/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_common/os_alloc.c @@ -266,6 +266,8 @@ __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_RET(__wt_malloc(session, len + 1, &p)); + WT_ASSERT(session, p != NULL); /* quiet clang scan-build */ + /* * Don't change this to strncpy, we rely on this function to duplicate * "strings" that contain nul bytes. diff --git a/src/third_party/wiredtiger/src/os_common/os_getopt.c b/src/third_party/wiredtiger/src/os_common/os_getopt.c index 960776c3999..fa21123ba0e 100644 --- a/src/third_party/wiredtiger/src/os_common/os_getopt.c +++ b/src/third_party/wiredtiger/src/os_common/os_getopt.c @@ -59,13 +59,17 @@ #include "wt_internal.h" -extern int __wt_opterr, __wt_optind, __wt_optopt, __wt_optreset; +extern int __wt_opterr WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optind WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optopt WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optreset WT_ATTRIBUTE_LIBRARY_VISIBLE; + int __wt_opterr = 1, /* if error message should be printed */ __wt_optind = 1, /* index into parent argv vector */ __wt_optopt, /* character checked for validity */ __wt_optreset; /* reset getopt */ -extern char *__wt_optarg; +extern char *__wt_optarg WT_ATTRIBUTE_LIBRARY_VISIBLE; char *__wt_optarg; /* argument associated with option */ #define BADCH (int)'?' diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 627278540d1..b1b6571e4ba 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -37,7 +37,13 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, dirallocsz = 0; entries = NULL; + /* + * If opendir fails, we should have a NULL pointer with an error value, + * but various static analysis programs remain unconvinced, check both. + */ WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? -1 : 0), ret); + if (dirp == NULL && ret == 0) + ret = EINVAL; if (ret != 0) WT_RET_MSG(session, ret, "%s: directory-list: opendir", directory); diff --git a/src/third_party/wiredtiger/src/os_posix/os_time.c b/src/third_party/wiredtiger/src/os_posix/os_time.c index 6f150ee8ffe..fe337fea7cf 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_time.c +++ b/src/third_party/wiredtiger/src/os_posix/os_time.c @@ -16,6 +16,7 @@ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { + struct timespec tmp; WT_DECL_RET; /* @@ -27,21 +28,34 @@ __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) tsp->tv_sec = 0; tsp->tv_nsec = 0; + /* + * Read into a local variable so that we're comparing the correct + * value when we check for monotonic increasing time. There are + * many places we read into an unlocked global variable. + */ #if defined(HAVE_CLOCK_GETTIME) - WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, tsp), ret); - if (ret == 0) + WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, &tmp), ret); + if (ret == 0) { + __wt_time_check_monotonic(session, &tmp); + tsp->tv_sec = tmp.tv_sec; + tsp->tv_nsec = tmp.tv_nsec; return; + } WT_PANIC_MSG(session, ret, "clock_gettime"); #elif defined(HAVE_GETTIMEOFDAY) + { struct timeval v; WT_SYSCALL_RETRY(gettimeofday(&v, NULL), ret); if (ret == 0) { - tsp->tv_sec = v.tv_sec; - tsp->tv_nsec = v.tv_usec * WT_THOUSAND; + tmp.tv_sec = v.tv_sec; + tmp.tv_nsec = v.tv_usec * WT_THOUSAND; + __wt_time_check_monotonic(session, &tmp); + *tsp = tmp; return; } WT_PANIC_MSG(session, ret, "gettimeofday"); + } #else NO TIME-OF-DAY IMPLEMENTATION: see src/os_posix/os_time.c #endif diff --git a/src/third_party/wiredtiger/src/os_win/os_time.c b/src/third_party/wiredtiger/src/os_win/os_time.c index 6aa5b3719f6..ba71341ab22 100644 --- a/src/third_party/wiredtiger/src/os_win/os_time.c +++ b/src/third_party/wiredtiger/src/os_win/os_time.c @@ -15,17 +15,18 @@ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) { + struct timespec tmp; FILETIME time; uint64_t ns100; - WT_UNUSED(session); - GetSystemTimeAsFileTime(&time); ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) - 116444736000000000LL; - tsp->tv_sec = ns100 / 10000000; - tsp->tv_nsec = (long)((ns100 % 10000000) * 100); + tmp.tv_sec = ns100 / 10000000; + tmp.tv_nsec = (long)((ns100 % 10000000) * 100); + __wt_time_check_monotonic(session, &tmp); + *tsp = tmp; } /* diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 74ef5135a4a..3dc51b6cb43 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -249,3 +249,34 @@ __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_TRET(__wt_schema_remove_table(session, table)); return (ret); } + +/* + * __wt_schema_sweep_tables -- + * Close all idle, obsolete tables in a session. + */ +int +__wt_schema_sweep_tables(WT_SESSION_IMPL *session) +{ + WT_TABLE *table, *next; + uint64_t schema_gen; + bool old_table_busy; + + WT_ORDERED_READ(schema_gen, S2C(session)->schema_gen); + if (schema_gen == session->table_sweep_gen) + return (0); + + old_table_busy = false; + TAILQ_FOREACH_SAFE(table, &session->tables, q, next) + if (table->schema_gen != schema_gen) { + if (table->refcnt == 0) + WT_RET(__wt_schema_remove_table( + session, table)); + else + old_table_busy = true; + } + + if (!old_table_busy) + session->table_sweep_gen = schema_gen; + + return (0); +} diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index b7daf0e2e02..5ce6135cfca 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -818,6 +818,8 @@ __session_reset(WT_SESSION *wt_session) WT_TRET(__wt_session_reset_cursors(session, true)); + WT_TRET(__wt_schema_sweep_tables(session)); + /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); @@ -1105,7 +1107,6 @@ int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) { - WT_CURSOR *cursor; WT_DECL_RET; int cmp; bool local_start; @@ -1134,12 +1135,13 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, } /* - * Cursor truncate is only supported for some objects, check for the - * supporting methods we need, range_truncate and compare. + * Cursor truncate is only supported for some objects, check for a + * supporting compare method. */ - cursor = start == NULL ? stop : start; - if (cursor->compare == NULL) - WT_ERR(__wt_bad_object_type(session, cursor->uri)); + if (start != NULL && start->compare == NULL) + WT_ERR(__wt_bad_object_type(session, start->uri)); + if (stop != NULL && stop->compare == NULL) + WT_ERR(__wt_bad_object_type(session, stop->uri)); /* * If both cursors set, check they're correctly ordered with respect to @@ -1150,6 +1152,9 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, * reference the same object and the keys are set. */ if (start != NULL && stop != NULL) { + /* quiet clang scan-build */ + WT_ASSERT(session, start->compare != NULL); + WT_ERR(start->compare(start, stop, &cmp)); if (cmp > 0) WT_ERR_MSG(session, EINVAL, diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 2c2217f8c20..8b72e653658 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -728,6 +728,7 @@ static const char * const __stats_connection_desc[] = { "cache: unmodified pages evicted", "connection: auto adjusting condition resets", "connection: auto adjusting condition wait calls", + "connection: detected system time went backwards", "connection: files currently open", "connection: memory allocations", "connection: memory frees", @@ -1014,6 +1015,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_clean = 0; stats->cond_auto_wait_reset = 0; stats->cond_auto_wait = 0; + stats->time_travel = 0; /* not clearing file_open */ stats->memory_allocation = 0; stats->memory_free = 0; @@ -1320,6 +1322,7 @@ __wt_stat_connection_aggregate( to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); to->cond_auto_wait_reset += WT_STAT_READ(from, cond_auto_wait_reset); to->cond_auto_wait += WT_STAT_READ(from, cond_auto_wait); + to->time_travel += WT_STAT_READ(from, time_travel); to->file_open += WT_STAT_READ(from, file_open); to->memory_allocation += WT_STAT_READ(from, memory_allocation); to->memory_free += WT_STAT_READ(from, memory_free); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index ea7faa2e966..76fdf71e715 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -503,13 +503,17 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; WT_TXN_OP *op; u_int i; - bool did_update; + bool did_update, locked; txn = &session->txn; conn = S2C(session); + txn_global = &conn->txn_global; did_update = txn->mod_count != 0; + locked = false; + WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || !did_update); if (!F_ISSET(txn, WT_TXN_RUNNING)) @@ -580,6 +584,14 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * This is particularly important for checkpoints. */ __wt_txn_release_snapshot(session); + /* + * We hold the visibility lock for reading from the time + * we write our log record until the time we release our + * transaction so that the LSN any checkpoint gets will + * always reflect visible data. + */ + __wt_readlock(session, &txn_global->visibility_rwlock); + locked = true; ret = __wt_txn_log_commit(session, cfg); } @@ -590,8 +602,12 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * Nothing can fail after this point. */ if (ret != 0) { + if (locked) + __wt_readunlock(session, + &txn_global->visibility_rwlock); WT_TRET(__wt_txn_rollback(session, cfg)); return (ret); + } /* Free memory associated with updates. */ @@ -600,6 +616,8 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) txn->mod_count = 0; __wt_txn_release(session); + if (locked) + __wt_readunlock(session, &txn_global->visibility_rwlock); return (0); } @@ -770,6 +788,7 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) &txn_global->id_lock, "transaction id lock")); WT_RET(__wt_rwlock_init(session, &txn_global->scan_rwlock)); WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->visibility_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); @@ -801,6 +820,7 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &txn_global->id_lock); __wt_rwlock_destroy(session, &txn_global->scan_rwlock); __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); + __wt_rwlock_destroy(session, &txn_global->visibility_rwlock); __wt_free(session, txn_global->states); } diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 2931dc1ce82..cb3b3436786 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -294,11 +294,13 @@ __wt_txn_checkpoint_log( WT_ITEM *ckpt_snapshot, empty; WT_LSN *ckpt_lsn; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; uint8_t *end, *p; size_t recsize; uint32_t i, rectype = WT_LOGREC_CHECKPOINT; const char *fmt = WT_UNCHECKED_STRING(IIIIu); + txn_global = &S2C(session)->txn_global; txn = &session->txn; ckpt_lsn = &txn->ckpt_lsn; @@ -319,6 +321,15 @@ __wt_txn_checkpoint_log( case WT_TXN_LOG_CKPT_PREPARE: txn->full_ckpt = true; WT_ERR(__wt_log_flush_lsn(session, ckpt_lsn, true)); + /* + * We take and immediately release the visibility lock. + * Acquiring the write lock guarantees that any transaction + * that has written to the log has also made its transaction + * visible at this time. + */ + __wt_writelock(session, &txn_global->visibility_rwlock); + __wt_writeunlock(session, &txn_global->visibility_rwlock); + /* * We need to make sure that the log records in the checkpoint * LSN are on disk. In particular to make sure that the diff --git a/version.json b/version.json index ad5f47b99da..84ff3f9e40e 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "056bf45128114e44c5358c7a8776fb582363e094", - "version": "3.2.16" + "githash": "186656d79574f7dfe0831a7e7821292ab380f667", + "version": "3.2.17" } \ No newline at end of file -- cgit v1.2.3